Resync the lexers with Scintillua

- Resync the lexers with Scintillua

- Update the lexer readme

- Update `zenburn` theme to fix some highlighting issues

- lexers: redirect print function to vis:info()

- Fix support for custom style names

- As per error message "lexer.delimited_range() is deprecated, use lexer.range()".

- Remove remaining `lexer.delimited_range()` call

- Set syntax to `nil` if the file type has no matching lexer

- Updated Go lexer for Go 1.18.

- lexers/dsv: convert to new lexer format

  (cherry picked from commit 9edbc3cd9ea1d7142b1305840432a3d2739e755a)

- lexers/gemini: disable legacy gemini lexer

  This reverts commit 468f9ee1b027a7ce98b1a249fa1af5888feeb989.

  It is in legacy format and of questionable quality. Ideally it
  should be contributed upstream from where it will eventually
  trickle down to us.

- lexers/git-rebase: convert to new lexer format

  (cherry picked from commit 4000a4cc9ac4a4c2869dfae772b977a82aee8d8c)

- lexers/strace: convert to new lexer format

  (cherry picked from commit e420451320d97eb164f5629c1bcfab0b595be29d)

- lexers/typescript: add new upstream lexer revision 28e2b60

  (cherry picked from commit 7326e6deecdaa75fa94ae9ebdb653f9f907b33f2)

- use `package.searchpath` instead of a local `searchpath` function

- Restore `filetype: support filetype detection via hashbang`

- Remove redundant comment

- Restore gemini lexer

1 files changed, 122 insertions, 187 deletions

diff --git a/lua/lexers/texinfo.lua b/lua/lexers/texinfo.lua
index b19f131..68ddb69 100644
--- a/lua/lexers/texinfo.lua
+++ b/lua/lexers/texinfo.lua
@@ -1,15 +1,14 @@
--- Copyright 2014-2017 stef@ailleurs.land. See LICENSE.
+-- Copyright 2014-2022 stef@ailleurs.land. See LICENSE.
 -- Plain Texinfo version 5.2 LPeg lexer
 -- Freely inspired from Mitchell work and valuable help from him too !
 
--- Directives are processed (more or less) in the Reference Card Texinfo order
--- Reference Card page for each directive group is in comment for reference
+-- Directives are processed (more or less) in the Reference Card Texinfo order Reference Card
+-- page for each directive group is in comment for reference
 
 --[[
 Note: Improving Fold Points use with Texinfo
 
-At the very beginning of your Texinfo file, it could be wised to insert theses
-alias :
+At the very beginning of your Texinfo file, it could be wised to insert theses alias :
 
 @alias startchapter = comment
 @alias endchapter = comment
@@ -21,51 +20,94 @@ Then use this to begin each chapter :
 @startchapter ------------------------------------------------------------------
 
 With the use of Scintilla's `SCI_FOLDALL(SC_FOLDACTION_TOGGLE)` or Textadept's
-`buffer:fold_all(buffer.FOLDACTION_TOGGLE)`, you have then a nice chapter
-folding, useful with large documents.
+`buffer:fold_all(buffer.FOLDACTION_TOGGLE)`, you have then a nice chapter folding, useful with
+large documents.
 ]]
 
-local l = require('lexer')
-local token, word_match = l.token, l.word_match
-local P, R, S = lpeg.P, lpeg.R, lpeg.S
+local lexer = require('lexer')
+local token, word_match = lexer.token, lexer.word_match
+local P, S = lpeg.P, lpeg.S
 
-local M = {_NAME = 'texinfo'}
-
--------------------------------------------------------------------------------
--- Common processing
--------------------------------------------------------------------------------
+local lex = lexer.new('texinfo')
 
 -- Whitespace.
-local ws = token(l.WHITESPACE, l.space^1)
-
--- Comments.
-local line_comment = '@c' * l.nonnewline_esc^0
---local line_comment_long = '@comment' * l.nonnewline_esc^0
-local block_comment = '@ignore' * (l.any - '@end ignore')^0 *
-                      P('@end ignore')^-1
-local comment = token(l.COMMENT, line_comment + block_comment)
-
--- Strings.
-local string = token(l.STRING, l.delimited_range('{}', false, true, true))
-
--- Numbers.
-local number = token(l.NUMBER, l.float + l.integer)
+lex:add_rule('whitespace', token(lexer.WHITESPACE, lexer.space^1))
 
--- Identifiers
-local identifier = token(l.IDENTIFIER, l.word)
+-- Directives.
+local directives_base = word_match({
+  'end',
+  -- Custom keywords for chapter folding
+  'startchapter', 'endchapter',
+  -- List and tables (page 2, column 2)
+  'itemize', 'enumerate',
+  -- Beginning a Texinfo document (page 1, column 1)
+  'titlepage', 'copying',
+  -- Block environments (page 2, column 1)
+  'cartouche',
+  -- Block environments > Displays using fixed-width fonts (page 2, column 2)
+  'example', 'smallexample',
+  -- List and tables (page 2, column 2)
+  'multitable',
+  -- Floating Displays (page 2, column 3)
+  'float', 'listoffloats', 'caption', 'shortcaption', 'image',
+  -- Floating Displays > Footnotes (page 2, column 3)
+  'footnote', 'footnotestyle',
+  -- Conditionally (in)visible text > Output formats (page 3, column 3)
+  'ifdocbook', 'ifhtml', 'ifinfo', 'ifplaintext', 'iftex', 'ifxml', 'ifnotdocbook', 'ifnothtml',
+  'ifnotplaintext', 'ifnottex', 'ifnotxml', 'ifnotinfo', 'inlinefmt', 'inlinefmtifelse',
+  -- Conditionally (in)visible text > Raw formatter text (page 4, column 1)
+  'docbook', 'html', 'tex', 'xml', 'inlineraw',
+  -- Conditionally (in)visible text > Documents variables (page 4, column 1)
+  'set', 'clear', 'value', 'ifset', 'ifclear', 'inlineifset', 'inlineifclear',
+  -- Conditionally (in)visible text > Testing for commands (page 4, column 1)
+  'ifcommanddefined', 'ifcommandnotdefined', 'end',
+  -- Defining new Texinfo commands (page 4, column 1)
+  'alias', 'macro', 'unmacro', 'definfounclose',
+  -- File inclusion (page 4, column 1)
+  'include', 'verbatiminclude',
+  -- Formatting and headers footers for TeX (page 4, column 1)
+  'allowcodebreaks', 'finalout', 'fonttextsize',
+  -- Formatting and headers footers for TeX > paper size (page 4, column 2)
+  'smallbook', 'afourpaper', 'afivepaper', 'afourlatex', 'afourwide', 'pagesizes',
+  -- Formatting and headers footers for TeX > Page headers and footers (page 4, column 2)
+  -- not implemented
+  -- Document preferences (page 4, column 2)
+  -- not implemented
+  -- Ending a Texinfo document (page 4, column 2)
+  'bye'
+}, true)
+lex:add_rule('directive', token('directives', ('@end' * lexer.space^1 + '@') * directives_base))
+lex:add_style('directives', lexer.styles['function'])
 
--------------------------------------------------------------------------------
--- Common keywords
--------------------------------------------------------------------------------
+-- Chapters.
+local chapters_base = word_match({
+  -- Chapter structuring (page 1, column 2)
+  'lowersections', 'raisesections', 'part',
+  -- Chapter structuring > Numbered, included in contents (page 1, column 2)
+  'chapter', 'centerchap',
+  -- Chapter structuring > Context-dependent, included in contents (page 1, column 2)
+  'section', 'subsection', 'subsubsection',
+  -- Chapter structuring > Unumbered, included in contents (page 1, column 2)
+  'unnumbered', 'unnumberedsec', 'unnumberedsubsec', 'unnumberedsubsection', 'unnumberedsubsubsec',
+  'unnumberedsubsubsection',
+  -- Chapter structuring > Letter and numbered, included in contents (page 1, column 2)
+  'appendix', 'appendixsec', 'appendixsection', 'appendixsubsec', 'appendixsubsection',
+  'appendixsubsubsec', 'appendixsubsubsection',
+  -- Chapter structuring > Unumbered, not included in contents, no new page (page 1, column 3)
+  'chapheading', 'majorheading', 'heading', 'subheading', 'subsubheading'
+}, true)
+lex:add_rule('chapter', token('chapters', ('@end' * lexer.space^1 + '@') * chapters_base))
+lex:add_style('chapters', lexer.styles.class)
 
+-- Common keywords.
 local keyword_base = word_match({
   'end',
   -- Beginning a Texinfo document (page 1, column 1)
   'setfilename', 'settitle', 'insertcopying',
   -- Beginning a Texinfo document > Internationlization (page 1, column 1)
   'documentencoding', 'documentlanguage', 'frenchspacing',
-  -- Beginning a Texinfo document > Info directory specification and HTML
-  -- document description (page 1, column 1)
+  -- Beginning a Texinfo document > Info directory specification and HTML document description
+  -- (page 1, column 1)
   'dircategory', 'direntry', 'documentdescription',
   -- Beginning a Texinfo document > Titre pages (page 1, column 1)
   'shorttitlepage', 'center', 'titlefont', 'title', 'subtitle', 'author',
@@ -83,8 +125,7 @@ local keyword_base = word_match({
   -- Marking text > Markup for regular text (page 1, column 3)
   'var', 'dfn', 'acronym', 'abbr',
   -- Marking text > Markup for litteral text (page 1, column 3)
-  'code', 'file', 'command', 'env', 'option', 'kbd', 'key', 'email',
-  'indicateurl', 'samp', 'verb',
+  'code', 'file', 'command', 'env', 'option', 'kbd', 'key', 'email', 'indicateurl', 'samp', 'verb',
   -- Marking text > GUI sequences (page 2, column 1)
   'clicksequence', 'click', 'clickstyle', 'arrow',
   -- Marking text > Math (page 2, column 1)
@@ -93,33 +134,26 @@ local keyword_base = word_match({
   'sc', 'r', 'i', 'slanted', 'b', 'sansserif', 't',
   -- Block environments (page 2, column 1)
   'noindent', 'indent', 'exdent',
-  -- Block environments > Normally filled displays using regular text fonts
-  -- (page 2, column 1)
-  'quotation', 'smallquotation', 'indentedblock', 'smallindentedblock',
-  'raggedright',
-  -- Block environments > Line-for-line displays using regular test fonts (page
-  -- 2, column 2)
+  -- Block environments > Normally filled displays using regular text fonts (page 2, column 1)
+  'quotation', 'smallquotation', 'indentedblock', 'smallindentedblock', 'raggedright',
+  -- Block environments > Line-for-line displays using regular test fonts (page 2, column 2)
   'format', 'smallformat', 'display', 'smalldisplay', 'flushleft', 'flushright',
   -- Block environments > Displays using fixed-width fonts (page 2, column 2)
   'lisp', 'smalllisp', 'verbatim',
   -- List and tables (page 2, column 2)
-  'table', 'ftable', 'vtable', 'tab', 'item', 'itemx', 'headitem',
-  'headitemfont', 'asis',
+  'table', 'ftable', 'vtable', 'tab', 'item', 'itemx', 'headitem', 'headitemfont', 'asis',
   -- Indices (page 2, column 3)
-  'cindex', 'findex', 'vindex', 'kindex', 'pindex', 'tindex', 'defcodeindex',
-  'syncodeindex', 'synindex', 'printindex',
-  -- Insertions within a paragraph > Characters special to Texinfo (page 2,
-  -- column 3)
+  'cindex', 'findex', 'vindex', 'kindex', 'pindex', 'tindex', 'defcodeindex', 'syncodeindex',
+  'synindex', 'printindex',
+  -- Insertions within a paragraph > Characters special to Texinfo (page 2, column 3)
   '@', '{', '}', 'backslashcar', 'comma', 'hashcar', ':', '.', '?', '!', 'dmn',
   -- Insertions within a paragraph > Accents (page 3, column 1)
   -- not implemented
   -- Insertions within a paragraph > Non-English characters (page 3, column 1)
   -- not implemented
-  -- Insertions within a paragraph > Other text characters an logos (page 3,
-  -- column 1)
-  'bullet', 'dots', 'enddots', 'euro', 'pounds', 'textdegree', 'copyright',
-  'registeredsymbol', 'TeX', 'LaTeX', 'today',
-  'guillemetleft', 'guillementright', 'guillemotleft', 'guillemotright',
+  -- Insertions within a paragraph > Other text characters an logos (page 3, column 1)
+  'bullet', 'dots', 'enddots', 'euro', 'pounds', 'textdegree', 'copyright', 'registeredsymbol',
+  'TeX', 'LaTeX', 'today', 'guillemetleft', 'guillementright', 'guillemotleft', 'guillemotright',
   -- Insertions within a paragraph > Glyphs for code examples (page 3, column 2)
   'equiv', 'error', 'expansion', 'point', 'print', 'result',
   -- Making and preventing breaks (page 3, column 2)
@@ -128,143 +162,44 @@ local keyword_base = word_match({
   'sp', 'page', 'need', 'group', 'vskip'
   -- Definition commands (page 3, column 2)
   -- not implemented
-}, nil, true)
-
-local keyword = token(l.KEYWORD, ('@end' * l.space^1 + '@') * keyword_base)
-
--------------------------------------------------------------------------------
--- Chapter structuring Keywords
--------------------------------------------------------------------------------
-
-local chapters_base = word_match({
-  -- Chapter structuring (page 1, column 2)
-  'lowersections', 'raisesections', 'part',
-  -- Chapter structuring > Numbered, included in contents (page 1, column 2)
-  'chapter', 'centerchap',
-  -- Chapter structuring > Context-dependent, included in contents (page 1,
-  -- column 2)
-  'section', 'subsection', 'subsubsection',
-  -- Chapter structuring > Unumbered, included in contents (page 1, column 2)
-  'unnumbered', 'unnumberedsec', 'unnumberedsubsec', 'unnumberedsubsection',
-  'unnumberedsubsubsec', 'unnumberedsubsubsection',
-  -- Chapter structuring > Letter and numbered, included in contents (page 1,
-  -- column 2)
-  'appendix', 'appendixsec', 'appendixsection', 'appendixsubsec',
-  'appendixsubsection', 'appendixsubsubsec', 'appendixsubsubsection',
-  -- Chapter structuring > Unumbered, not included in contents, no new page
-  -- (page 1, column 3)
-  'chapheading', 'majorheading', 'heading', 'subheading', 'subsubheading'
-}, nil, true)
-
-local chapters = token('chapters', ('@end' * l.space^1 + '@') * chapters_base)
-
--------------------------------------------------------------------------------
--- Directives Keywords
--------------------------------------------------------------------------------
-
-local directives_base = word_match({
-  'end',
-  -- Custom keywords for chapter folding
-  'startchapter', 'endchapter',
-  -- List and tables (page 2, column 2)
-  'itemize', 'enumerate',
-  -- Beginning a Texinfo document (page 1, column 1)
-  'titlepage', 'copying',
-  -- Block environments (page 2, column 1)
-  'cartouche',
-  -- Block environments > Displays using fixed-width fonts (page 2, column 2)
-  'example', 'smallexample',
-  -- List and tables (page 2, column 2)
-  'multitable',
-  -- Floating Displays (page 2, column 3)
-  'float', 'listoffloats', 'caption', 'shortcaption', 'image',
-  -- Floating Displays > Footnotes (page 2, column 3)
-  'footnote', 'footnotestyle',
-  -- Conditionally (in)visible text > Output formats (page 3, column 3)
-  'ifdocbook', 'ifhtml', 'ifinfo', 'ifplaintext', 'iftex', 'ifxml',
-  'ifnotdocbook', 'ifnothtml', 'ifnotplaintext',
-  'ifnottex', 'ifnotxml', 'ifnotinfo', 'inlinefmt', 'inlinefmtifelse',
-  -- Conditionally (in)visible text > Raw formatter text (page 4, column 1)
-  'docbook', 'html', 'tex', 'xml', 'inlineraw',
-  -- Conditionally (in)visible text > Documents variables (page 4, column 1)
-  'set', 'clear', 'value', 'ifset', 'ifclear', 'inlineifset', 'inlineifclear',
-  -- Conditionally (in)visible text > Testing for commands (page 4, column 1)
-  'ifcommanddefined', 'ifcommandnotdefined', 'end',
-  -- Defining new Texinfo commands (page 4, column 1)
-  'alias', 'macro', 'unmacro', 'definfounclose',
-  -- File inclusion (page 4, column 1)
-  'include', 'verbatiminclude',
-  -- Formatting and headers footers for TeX (page 4, column 1)
-  'allowcodebreaks', 'finalout', 'fonttextsize',
-  -- Formatting and headers footers for TeX > paper size (page 4, column 2)
-  'smallbook', 'afourpaper', 'afivepaper', 'afourlatex', 'afourwide',
-  'pagesizes',
-  -- Formatting and headers footers for TeX > Page headers and footers (page 4,
-  -- column 2)
-  -- not implemented
-  -- Document preferences (page 4, column 2)
-  -- not implemented
-  -- Ending a Texinfo document (page 4, column 2)
-  'bye'
-}, nil, case_insensitive_tags)
-
-local directives = token('directives',
-                         ('@end' * l.space^1 + '@') * directives_base)
-
--------------------------------------------------------------------------------
--- Special keywords
--------------------------------------------------------------------------------
+}, true)
+lex:add_rule('keyword', token(lexer.KEYWORD, ('@end' * lexer.space^1 + '@') * keyword_base))
 
 -- Italics
-local emph = token('emph', '@emph' * l.delimited_range('{}', false, true, true))
+local nested_braces = lexer.range('{', '}', false, false, true)
+lex:add_rule('emph', token('emph', '@emph' * nested_braces))
+lex:add_style('emph', lexer.styles.string .. {italics = true})
 
 -- Bold
-local strong = token('strong',
-                     '@strong' * l.delimited_range('{}', false, true, true))
-
--------------------------------------------------------------------------------
--- Rules, Tokenstyles and Folds points
--------------------------------------------------------------------------------
-
-M._rules = {
-  {'whitespace', ws},
-  {'directives', directives},
-  {'chapters', chapters},
-  {'keyword', keyword},
-  {'emph', emph},
-  {'strong', strong},
-  {'identifier', identifier},
-  {'string', string},
-  {'number', number},
-  {'comment', comment},
-}
+lex:add_rule('strong', token('strong', '@strong' * nested_braces))
+lex:add_style('strong', lexer.styles.string .. {bold = true})
 
-M._tokenstyles = {
-  directives = l.STYLE_FUNCTION,
-  chapters = l.STYLE_CLASS,
-  emph = l.STYLE_STRING..',italics',
-  strong = l.STYLE_STRING..',bold'
-}
-
-M._foldsymbols = {
-  _patterns = {'@end %l+', '@%l+'},
-  directives = {
-    ['@titlepage'] = 1, ['@end titlepage'] = -1,
-    ['@copying'] = 1, ['@end copying'] = -1,
-
-    ['@ifset'] = 1, ['@end ifset'] = -1,
-    ['@tex'] = 1, ['@end tex'] = -1,
-
-    ['@itemize'] = 1, ['@end itemize'] = -1,
-    ['@enumerate'] = 1, ['@end enumerate'] = -1,
-    ['@multitable'] = 1, ['@end multitable'] = -1,
+-- Identifiers
+lex:add_rule('identifier', token(lexer.IDENTIFIER, lexer.word))
 
-    ['@example'] = 1, ['@end example'] = -1,
-    ['@smallexample'] = 1, ['@end smallexample'] = -1,
-    ['@cartouche'] = 1, ['@end cartouche'] = -1,
+-- Strings.
+lex:add_rule('string', token(lexer.STRING, nested_braces))
 
-    ['@startchapter'] = 1, ['@endchapter'] = -1,
-  }
-}
+-- Numbers.
+lex:add_rule('number', token(lexer.NUMBER, lexer.number))
 
-return M
+-- Comments.
+local line_comment = lexer.to_eol('@c', true)
+-- local line_comment_long = lexer.to_eol('@comment', true)
+local block_comment = lexer.range('@ignore', '@end ignore')
+lex:add_rule('comment', token(lexer.COMMENT, line_comment + block_comment))
+
+-- Fold points.
+lex:add_fold_point('directives', '@titlepage', '@end titlepage')
+lex:add_fold_point('directives', '@copying', '@end copying')
+lex:add_fold_point('directives', '@ifset', '@end ifset')
+lex:add_fold_point('directives', '@tex', '@end tex')
+lex:add_fold_point('directives', '@itemize', '@end itemize')
+lex:add_fold_point('directives', '@enumerate', '@end enumerate')
+lex:add_fold_point('directives', '@multitable', '@end multitable')
+lex:add_fold_point('directives', '@example', '@end example')
+lex:add_fold_point('directives', '@smallexample', '@end smallexample')
+lex:add_fold_point('directives', '@cartouche', '@end cartouche')
+lex:add_fold_point('directives', '@startchapter', '@end startchapter')
+
+return lex