lua/lexers: update to scintillua 6.5

This is an amalgamation of the following upstream commits:

- Overhauled API documentation for lexer.lua.
- Fixed Markdown to allow code fence blocks to be indented.
- Use GitHub Pages' Primer theme for documentation.
  Build static pages with Jekyll, like GitHub Pages does.
- Migrated systemd lexer.
  Thanks to Matěj Cepl.
- Migrated Lisp lexer and highlight character escapes.
  Thanks to Matěj Cepl.
- Migrated rpmspec lexer and made some improvements.
  Thanks to Matěj Cepl.
- Modernized reST lexer.
  Thanks to Matěj Cepl.
- Markdown lexer should just tag the start of a blockquote.
  The quote's contents may contain markdown.
- Output lexer can highlight CSI color sequences.
- Allow lexers to define their own fold functions.
- Added custom folder for Markdown headers.
- Added `lexer.line_start`, `lexer.line_end` and `lexer.text_range()`.
- Fixed Markdown lexer to not lex some continuation lines as code.
- Fixed SciTE not using Scintillua's markdown lexer.
- Markdown lexer should not highlight secondary paragraphs in list items as code blocks.
- Have SciTE recognize CMakeLists.txt.

1 files changed, 73 insertions, 78 deletions

diff --git a/lua/lexers/rest.lua b/lua/lexers/rest.lua
index 74337e7..46742a4 100644
--- a/lua/lexers/rest.lua
+++ b/lua/lexers/rest.lua
@@ -1,11 +1,11 @@
 -- Copyright 2006-2025 Mitchell. See LICENSE.
 -- reStructuredText LPeg lexer.
 
-local lexer = require('lexer')
-local token, word_match, starts_line = lexer.token, lexer.word_match, lexer.starts_line
+local lexer = lexer
+local starts_line = lexer.starts_line
 local P, S = lpeg.P, lpeg.S
 
-local lex = lexer.new('rest')
+local lex = lexer.new(...)
 
 -- Literal block.
 local block = '::' * (lexer.newline + -1) * function(input, index)
@@ -19,8 +19,7 @@ local block = '::' * (lexer.newline + -1) * function(input, index)
 	end
 	return #input + 1
 end
-lex:add_rule('literal_block', token('literal_block', block))
-lex:add_style('literal_block', lexer.styles.embedded .. {eolfilled = true})
+lex:add_rule('literal_block', lex:tag(lexer.LABEL .. '.literal', block))
 
 -- Lists.
 local option_word = lexer.alnum * (lexer.alnum + '-')^0
@@ -30,9 +29,9 @@ local option_list = option * (',' * lexer.space^1 * option)^-1
 local bullet_list = S('*+-') -- TODO: '•‣⁃', as lpeg does not support UTF-8
 local enum_list = P('(')^-1 * (lexer.digit^1 + S('ivxlcmIVXLCM')^1 + lexer.alnum + '#') * S('.)')
 local field_list = ':' * (lexer.any - ':')^1 * P(':')^-1
-lex:add_rule('list', #(lexer.space^0 * (S('*+-:/') + enum_list)) *
-	starts_line(token(lexer.LIST,
-		lexer.space^0 * (option_list + bullet_list + enum_list + field_list) * lexer.space)))
+lex:add_rule('list',
+	#(lexer.space^0 * (S('*+-:/') + enum_list)) * starts_line(lex:tag(lexer.LIST, lexer.space^0 *
+		(option_list + bullet_list + enum_list + field_list) * lexer.space)))
 
 local any_indent = S(' \t')^0
 local word = lexer.alpha * (lexer.alnum + S('-.+'))^0
@@ -40,15 +39,12 @@ local prefix = any_indent * '.. '
 
 -- Explicit markup blocks.
 local footnote_label = '[' * (lexer.digit^1 + '#' * word^-1 + '*') * ']'
-local footnote = token('footnote_block', prefix * footnote_label * lexer.space)
+local footnote = lex:tag(lexer.LABEL .. '.footnote', prefix * footnote_label * lexer.space)
 local citation_label = '[' * word * ']'
-local citation = token('citation_block', prefix * citation_label * lexer.space)
-local link = token('link_block', prefix * '_' *
+local citation = lex:tag(lexer.LABEL .. '.citation', prefix * citation_label * lexer.space)
+local link = lex:tag(lexer.LABEL .. '.link', prefix * '_' *
 	(lexer.range('`') + (P('\\') * 1 + lexer.nonnewline - ':')^1) * ':' * lexer.space)
 lex:add_rule('markup_block', #prefix * starts_line(footnote + citation + link))
-lex:add_style('footnote_block', lexer.styles.label)
-lex:add_style('citation_block', lexer.styles.label)
-lex:add_style('link_block', lexer.styles.label)
 
 -- Sphinx code block.
 local indented_block = function(input, index)
@@ -61,60 +57,27 @@ local indented_block = function(input, index)
 end
 local code_block =
 	prefix * 'code-block::' * S(' \t')^1 * lexer.nonnewline^0 * (lexer.newline + -1) * indented_block
-lex:add_rule('code_block', #prefix * token('code_block', starts_line(code_block)))
-lex:add_style('code_block', lexer.styles.embedded .. {eolfilled = true})
+lex:add_rule('code_block', #prefix * lex:tag(lexer.LABEL .. '.code', starts_line(code_block)))
 
 -- Directives.
-local known_directive = token('directive', prefix * word_match{
-	-- Admonitions
-	'attention', 'caution', 'danger', 'error', 'hint', 'important', 'note', 'tip', 'warning',
-	'admonition',
-	-- Images
-	'image', 'figure',
-	-- Body elements
-	'topic', 'sidebar', 'line-block', 'parsed-literal', 'code', 'math', 'rubric', 'epigraph',
-	'highlights', 'pull-quote', 'compound', 'container',
-	-- Table
-	'table', 'csv-table', 'list-table',
-	-- Document parts
-	'contents', 'sectnum', 'section-autonumbering', 'header', 'footer',
-	-- References
-	'target-notes', 'footnotes', 'citations',
-	-- HTML-specific
-	'meta',
-	-- Directives for substitution definitions
-	'replace', 'unicode', 'date',
-	-- Miscellaneous
-	'include', 'raw', 'class', 'role', 'default-role', 'title', 'restructuredtext-test-directive'
-} * '::' * lexer.space)
-local sphinx_directive = token('sphinx_directive', prefix * word_match{
-	-- The TOC tree.
-	'toctree',
-	-- Paragraph-level markup.
-	'note', 'warning', 'versionadded', 'versionchanged', 'deprecated', 'seealso', 'rubric',
-	'centered', 'hlist', 'glossary', 'productionlist',
-	-- Showing code examples.
-	'highlight', 'literalinclude',
-	-- Miscellaneous
-	'sectionauthor', 'index', 'only', 'tabularcolumns'
-} * '::' * lexer.space)
-local unknown_directive = token('unknown_directive', prefix * word * '::' * lexer.space)
+local known_directive = lex:tag(lexer.KEYWORD,
+	prefix * lex:word_match(lexer.KEYWORD) * '::' * lexer.space)
+local sphinx_directive = lex:tag(lexer.KEYWORD .. '.sphinx', prefix *
+	lex:word_match(lexer.KEYWORD .. '.sphinx') * '::' * lexer.space)
+local unknown_directive = lex:tag(lexer.KEYWORD .. '.unknown', prefix * word * '::' * lexer.space)
 lex:add_rule('directive',
 	#prefix * starts_line(known_directive + sphinx_directive + unknown_directive))
-lex:add_style('directive', lexer.styles.keyword)
-lex:add_style('sphinx_directive', lexer.styles.keyword .. {bold = true})
-lex:add_style('unknown_directive', lexer.styles.keyword .. {italics = true})
 
 -- Substitution definitions.
-lex:add_rule('substitution', #prefix * token('substitution', starts_line(prefix * lexer.range('|') *
-	lexer.space^1 * word * '::' * lexer.space)))
-lex:add_style('substitution', lexer.styles.variable)
+lex:add_rule('substitution',
+	#prefix * lex:tag(lexer.FUNCTION, starts_line(prefix * lexer.range('|') * lexer.space^1 * word *
+		'::' * lexer.space)))
 
 -- Comments.
 local line_comment = lexer.to_eol(prefix)
 local bprefix = any_indent * '..'
 local block_comment = bprefix * lexer.newline * indented_block
-lex:add_rule('comment', #bprefix * token(lexer.COMMENT, starts_line(line_comment + block_comment)))
+lex:add_rule('comment', #bprefix * lex:tag(lexer.COMMENT, starts_line(line_comment + block_comment)))
 
 -- Section titles (2 or more characters).
 local adornment_chars = lpeg.C(S('!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~'))
@@ -135,38 +98,35 @@ local underline = lpeg.Cmt(starts_line(adornment), function(_, index, adm, c)
 	return pos and index - #adm + pos - 1 or nil
 end)
 -- Token needs to be a predefined one in order for folder to work.
-lex:add_rule('title', token(lexer.HEADING, overline + underline))
+lex:add_rule('title', lex:tag(lexer.HEADING, overline + underline))
 
 -- Line block.
-lex:add_rule('line_block_char', token(lexer.OPERATOR, starts_line(any_indent * '|')))
+lex:add_rule('line_block_char', lex:tag(lexer.OPERATOR, starts_line(any_indent * '|')))
 
 -- Whitespace.
-lex:add_rule('whitespace', token(lexer.WHITESPACE, S(' \t')^1 + lexer.newline^1))
+lex:add_rule('whitespace', lex:tag(lexer.WHITESPACE, S(' \t')^1 + lexer.newline^1))
 
 -- Inline markup.
-local strong = token(lexer.BOLD, lexer.range('**'))
-local em = token(lexer.ITALIC, lexer.range('*'))
-local inline_literal = token('inline_literal', lexer.range('``'))
+local strong = lex:tag(lexer.BOLD, lexer.range('**'))
+local em = lex:tag(lexer.ITALIC, lexer.range('*'))
+local inline_literal = lex:tag(lexer.CODE .. '.inline', lexer.range('``'))
 local postfix_link = (word + lexer.range('`')) * '_' * P('_')^-1
 local prefix_link = '_' * lexer.range('`')
-local link_ref = token(lexer.LINK, postfix_link + prefix_link)
-local role = token('role', ':' * word * ':' * (word * ':')^-1)
-local interpreted = role^-1 * token('interpreted', lexer.range('`')) * role^-1
-local footnote_ref = token(lexer.REFERENCE, footnote_label * '_')
-local citation_ref = token(lexer.REFERENCE, citation_label * '_')
-local substitution_ref = token('substitution', lexer.range('|', true) * ('_' * P('_')^-1)^-1)
-local link = token(lexer.LINK,
+local link_ref = lex:tag(lexer.LINK, postfix_link + prefix_link)
+local role = lex:tag(lexer.FUNCTION_BUILTIN, ':' * word * ':' * (word * ':')^-1)
+local interpreted = role^-1 * lex:tag(lexer.EMBEDDED, lexer.range('`')) * role^-1
+local footnote_ref = lex:tag(lexer.REFERENCE, footnote_label * '_')
+local citation_ref = lex:tag(lexer.REFERENCE, citation_label * '_')
+local substitution_ref = lex:tag(lexer.FUNCTION, lexer.range('|', true) * ('_' * P('_')^-1)^-1)
+local link = lex:tag(lexer.LINK,
 	lexer.alpha * (lexer.alnum + S('-.'))^1 * ':' * (lexer.alnum + S('/.+-%@'))^1)
 lex:add_rule('inline_markup',
 	(strong + em + inline_literal + link_ref + interpreted + footnote_ref + citation_ref +
 		substitution_ref + link) * -lexer.alnum)
-lex:add_style('inline_literal', lexer.styles.embedded)
-lex:add_style('role', lexer.styles.class)
-lex:add_style('interpreted', lexer.styles.string)
 
 -- Other.
-lex:add_rule('non_space', token(lexer.DEFAULT, lexer.alnum * (lexer.any - lexer.space)^0))
-lex:add_rule('escape', token(lexer.DEFAULT, '\\' * lexer.any))
+lex:add_rule('non_space', lex:tag(lexer.DEFAULT, lexer.alnum * (lexer.any - lexer.space)^0))
+lex:add_rule('escape', lex:tag(lexer.DEFAULT, '\\' * lexer.any))
 
 -- Section-based folding.
 local sphinx_levels = {
@@ -198,18 +158,53 @@ function lex:fold(text, start_line, start_level)
 	return folds
 end
 
--- lexer.property['fold.by.sphinx.convention'] = '0'
-
 --[[ Embedded languages.
 local bash = lexer.load('bash')
 local bash_indent_level
 local start_rule =
   #(prefix * 'code-block' * '::' * lexer.space^1 * 'bash' * (lexer.newline + -1)) *
-  sphinx_directive * token('bash_begin', P(function(input, index)
+  sphinx_directive * lex:tag(lexer.EMBEDDED, P(function(input, index)
     bash_indent_level = #input:match('^([ \t]*)', index)
     return index
   end))]]
 
+-- Word lists
+lex:set_word_list(lexer.KEYWORD, {
+	-- Admonitions
+	'attention', 'caution', 'danger', 'error', 'hint', 'important', 'note', 'tip', 'warning',
+	'admonition',
+	-- Images
+	'image', 'figure',
+	-- Body elements
+	'topic', 'sidebar', 'line-block', 'parsed-literal', 'code', 'math', 'rubric', 'epigraph',
+	'highlights', 'pull-quote', 'compound', 'container',
+	-- Table
+	'table', 'csv-table', 'list-table',
+	-- Document parts
+	'contents', 'sectnum', 'section-autonumbering', 'header', 'footer',
+	-- References
+	'target-notes', 'footnotes', 'citations',
+	-- HTML-specific
+	'meta',
+	-- Directives for substitution definitions
+	'replace', 'unicode', 'date',
+	-- Miscellaneous
+	'include', 'raw', 'class', 'role', 'default-role', 'title', 'restructuredtext-test-directive'
+})
+
+lex:set_word_list(lexer.KEYWORD .. '.sphinx', {
+	-- The TOC tree.
+	'toctree',
+	-- Paragraph-level markup.
+	'note', 'warning', 'versionadded', 'versionchanged', 'deprecated', 'seealso', 'rubric',
+	'centered', 'hlist', 'glossary', 'productionlist',
+	-- Showing code examples.
+	'highlight', 'literalinclude',
+	-- Miscellaneous
+	'sectionauthor', 'index', 'only', 'tabularcolumns'
+})
+
+-- lexer.property['fold.by.sphinx.convention'] = '0'
 lexer.property['scintillua.comment'] = '.. '
 
 return lex