From 4c4392d29df777ff702dfe99b4f3c23142976e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C4=9Bj=20Cepl?= Date: Fri, 11 Aug 2023 01:27:32 +0200 Subject: update lexers to orbitalquark/scintillua@b789dde Rather than cherry pick patches from after 6.2 we will just grab everything as is. --- lua/lexers/rust.lua | 102 ++++++++++++++++++++++++++++------------------------ 1 file changed, 55 insertions(+), 47 deletions(-) (limited to 'lua/lexers/rust.lua') diff --git a/lua/lexers/rust.lua b/lua/lexers/rust.lua index 25555d6..9b5b34a 100644 --- a/lua/lexers/rust.lua +++ b/lua/lexers/rust.lua @@ -1,54 +1,23 @@ --- Copyright 2015-2022 Alejandro Baez (https://keybase.io/baez). See LICENSE. +-- Copyright 2015-2024 Alejandro Baez (https://keybase.io/baez). See LICENSE. -- Rust LPeg lexer. -local lexer = require("lexer") -local token, word_match = lexer.token, lexer.word_match +local lexer = lexer local P, S = lpeg.P, lpeg.S local C, Cmt = lpeg.C, lpeg.Cmt -local lex = lexer.new('rust') - --- Whitespace. -lex:add_rule('whitespace', token(lexer.WHITESPACE, lexer.space^1)) +local lex = lexer.new(...) -- Keywords. --- https://github.com/rust-lang/rust/blob/stable/src/libsyntax_pos/symbol.rs -lex:add_rule('keyword', token(lexer.KEYWORD, word_match{ - 'Self', 'abstract', 'as', 'async', 'auto', 'await', 'become', 'box', 'break', 'catch', 'const', - 'continue', 'crate', 'default', 'do', 'dyn', 'else', 'enum', 'extern', 'false', 'final', 'fn', - 'for', 'if', 'impl', 'in', 'let', 'loop', 'macro', 'match', 'mod', 'move', 'mut', 'override', - 'priv', 'pub', 'ref', 'return', 'self', 'static', 'struct', 'super', 'trait', 'true', 'try', - 'type', 'typeof', 'union', 'unsafe', 'unsized', 'use', 'virtual', 'where', 'while', 'yield' -})) - --- Macro names. -lex:add_rule('macro', token(lexer.FUNCTION, lexer.word * S("!"))) +lex:add_rule('keyword', lex:tag(lexer.KEYWORD, lex:word_match(lexer.KEYWORD))) --- Library types -lex:add_rule('library', token(lexer.LABEL, lexer.upper * (lexer.lower + lexer.dec_num)^1)) - --- Numbers. -local identifier = P('r#')^-1 * lexer.word -local digit = lexer.digit -local decimal_literal = digit * (digit + '_')^0 -local function integer_suffix(digit) return P('_')^0 * digit * (digit + '_')^0 end -local function opt_cap(patt) return C(patt^-1) end -local float = decimal_literal * - (Cmt(opt_cap('.' * decimal_literal) * opt_cap(S('eE') * S('+-')^-1 * integer_suffix(digit)) * - opt_cap(P('f32') + 'f64'), function(input, index, decimals, exponent, type) - return decimals ~= "" or exponent ~= "" or type ~= "" - end) + '.' * -(S('._') + identifier)) -local function prefixed_integer(prefix, digit) return P(prefix) * integer_suffix(digit) end -local bin = prefixed_integer('0b', S('01')) -local oct = prefixed_integer('0o', lpeg.R('07')) -local hex = prefixed_integer('0x', lexer.xdigit) -local integer = (bin + oct + hex + decimal_literal) * - (S('iu') * (P('8') + '16' + '32' + '64' + '128' + 'size'))^-1 -lex:add_rule('number', token(lexer.NUMBER, float + integer)) +-- Library types. +lex:add_rule('library', lex:tag(lexer.TYPE, lexer.upper * (lexer.lower + lexer.dec_num)^1)) -- Types. -lex:add_rule('type', token(lexer.TYPE, word_match( - '() bool isize usize char str u8 u16 u32 u64 u128 i8 i16 i32 i64 i128 f32 f64'))) +lex:add_rule('type', lex:tag(lexer.TYPE, lex:word_match(lexer.TYPE))) + +-- Lifetime annotation. +lex:add_rule('lifetime', lex:tag(lexer.OPERATOR, S('<&') * P("'"))) -- Strings. local sq_str = P('b')^-1 * lexer.range("'", true) @@ -57,26 +26,65 @@ local raw_str = Cmt(P('b')^-1 * P('r') * C(P('#')^0) * '"', function(input, inde local _, e = input:find('"' .. hashes, index, true) return (e or #input) + 1 end) -lex:add_rule('string', token(lexer.STRING, sq_str + dq_str + raw_str)) +lex:add_rule('string', lex:tag(lexer.STRING, sq_str + dq_str + raw_str)) + +-- Functions. +local builtin_macros = lex:tag(lexer.FUNCTION_BUILTIN, lex:word_match(lexer.FUNCTION_BUILTIN) * '!') +local macros = lex:tag(lexer.FUNCTION, lexer.word * '!') +local func = lex:tag(lexer.FUNCTION, lexer.word) +lex:add_rule('function', (builtin_macros + macros + func) * #(lexer.space^0 * '(')) -- Identifiers. -lex:add_rule('identifier', token(lexer.IDENTIFIER, identifier)) +local identifier = P('r#')^-1 * lexer.word +lex:add_rule('identifier', lex:tag(lexer.IDENTIFIER, identifier)) -- Comments. local line_comment = lexer.to_eol('//', true) local block_comment = lexer.range('/*', '*/', false, false, true) -lex:add_rule('comment', token(lexer.COMMENT, line_comment + block_comment)) +lex:add_rule('comment', lex:tag(lexer.COMMENT, line_comment + block_comment)) + +-- Numbers. +lex:add_rule('number', lex:tag(lexer.NUMBER, lexer.number_('_'))) -- Attributes. -lex:add_rule('preprocessor', token(lexer.PREPROCESSOR, '#' * lexer.range('[', ']', true))) +lex:add_rule('preprocessor', lex:tag(lexer.PREPROCESSOR, '#' * lexer.range('[', ']', true))) -- Operators. -lex:add_rule('operator', token(lexer.OPERATOR, S('+-/*%<>!=`^~@&|?#~:;,.()[]{}'))) +lex:add_rule('operator', lex:tag(lexer.OPERATOR, S('+-/*%<>!=`^~@&|?#~:;,.()[]{}'))) -- Fold points. lex:add_fold_point(lexer.COMMENT, '/*', '*/') -lex:add_fold_point(lexer.COMMENT, lexer.fold_consecutive_lines('//')) lex:add_fold_point(lexer.OPERATOR, '(', ')') lex:add_fold_point(lexer.OPERATOR, '{', '}') +-- https://doc.rust-lang.org/std/#keywords +lex:set_word_list(lexer.KEYWORD, { + 'SelfTy', 'as', 'async', 'await', 'break', 'const', 'continue', 'crate', 'dyn', 'else', 'enum', + 'extern', 'false', 'fn', 'for', 'if', 'impl', 'in', 'let', 'loop', 'match', 'mod', 'move', 'mut', + 'pub', 'ref', 'return', 'self', 'static', 'struct', 'super', 'trait', 'true', 'type', 'union', + 'unsafe', 'use', 'where', 'while' +}) + +-- https://doc.rust-lang.org/std/#primitives +lex:set_word_list(lexer.TYPE, { + 'never', 'array', 'bool', 'char', 'f32', 'f64', 'fn', 'i8', 'i16', 'i32', 'i64', 'i128', 'isize', + 'pointer', 'reference', 'slice', 'str', 'tuple', 'u8', 'u16', 'u32', 'u64', 'u128', 'unit', + 'usize' +}) + +lex:set_word_list(lexer.FUNCTION_BUILTIN, { + 'assert', 'assert_eq', 'assert_ne', 'cfg', 'column', 'compile_error', 'concat', 'dbg', + 'debug_assert', 'debug_assert_eq', 'debug_assert_ne', 'env', 'eprint', 'eprintln', 'file', + 'format', 'format_args', 'include', 'include_bytes', 'include_str', 'line', 'matches', + 'module_path', 'option_env', 'panic', 'print', 'println', 'stringify', 'thread_local', 'todo', + 'unimplemented', 'unreachable', 'vec', 'write', 'writeln', + -- Experimental + 'concat_bytes', 'concat_idents', 'const_format_args', 'format_args_nl', 'log_syntax', + 'trace_macros', + -- Deprecated + 'try' +}) + +lexer.property['scintillua.comment'] = '//' + return lex -- cgit v1.2.3