vis: move syntax highlighting to pure Lua code

12 files changed, 152 insertions, 210 deletions

diff --git a/lexers/lexer.lua b/lexers/lexer.lua
index 345dabe..ae9cc9e 100644
--- a/lexers/lexer.lua
+++ b/lexers/lexer.lua
@@ -1590,8 +1590,4 @@ M.property_expanded = setmetatable({}, {
 local lexer
 ]]
 
-function M.get_style(lexer, lang, token_name)
-  return lexer['STYLE_'..string_upper(token_name)] or lang._EXTRASTYLES[token_name]
-end
-
 return M
diff --git a/main.c b/main.c
index 65f8493..2bc55de 100644
--- a/main.c
+++ b/main.c
@@ -2188,6 +2188,8 @@ int main(int argc, char *argv[]) {
 		.file_close = vis_lua_file_close,
 		.win_open = vis_lua_win_open,
 		.win_close = vis_lua_win_close,
+		.win_highlight = vis_lua_win_highlight,
+		.win_syntax = vis_lua_win_syntax,
 	};
 
 	vis = vis_new(ui_curses_new(), &event);
diff --git a/register.h b/register.h
index 0e7b925..a815a90 100644
--- a/register.h
+++ b/register.h
@@ -3,13 +3,10 @@
 
 #include <stddef.h>
 #include <stdbool.h>
+#include "vis.h"
 #include "buffer.h"
 #include "text-util.h"
 
-#ifndef VIS_H
-typedef struct Vis Vis;
-#endif
-
 typedef struct {
 	Buffer buf;
 	bool linewise; /* place register content on a new line when inserting? */
diff --git a/view.c b/view.c
index 50b7c8e..ec06b9e 100644
--- a/view.c
+++ b/view.c
@@ -5,7 +5,6 @@
 #include <errno.h>
 #include <regex.h>
 #include <limits.h>
-#include "vis-lua.h"
 #include "view.h"
 #include "text.h"
 #include "text-motions.h"
@@ -88,14 +87,13 @@ struct View {
 	int tabwidth;       /* how many spaces should be used to display a tab character */
 	Cursor *cursors;    /* all cursors currently active */
 	Selection *selections; /* all selected regions */
-	lua_State *lua;     /* lua state used for syntax highlighting */
 	int cursor_generation; /* used to filter out newly created cursors during iteration */
-	char *lexer_name;
 	size_t horizon; /* maximal number of bytes to consider for syntax highlighting
 	                 * before the visible area */
 	bool need_update;   /* whether view has been redrawn */
 	bool large_file;    /* optimize for displaying large files */
 	int colorcolumn;
+	ViewEvent *events;
 };
 
 static const SyntaxSymbol symbols_none[] = {
@@ -124,178 +122,6 @@ static void view_cursors_free(Cursor *c);
 /* set/move current cursor position to a given (line, column) pair */
 static size_t cursor_set(Cursor *cursor, Line *line, int col);
 
-#if !CONFIG_LUA
-
-static void view_syntax_color(View *view) { }
-bool view_syntax_set(View *view, const char *name) { return false; }
-
-#else
-
-static void view_syntax_color(View *view) {
-	lua_State *L = view->lua;
-	if (!L || !view->lexer_name)
-		return;
-	lua_getglobal(L, "vis");
-	lua_getfield(L, -1, "lexers");
-	if (lua_isnil(L, -1))
-		return;
-
-	/* absolute position to start syntax highlighting */
-	const size_t lexer_start = view->start >= view->horizon ?
-		view->start - view->horizon : 0;
-	/* number of bytes used for syntax highlighting before visible are */
-	size_t lexer_before = view->start - lexer_start;
-	/* number of bytes to read in one go */
-	const size_t text_size = lexer_before + (view->end - view->start);
-	/* current buffer to work with */
-	char text[text_size+1];
-	/* bytes to process */
-	const size_t text_len = text_bytes_get(view->text, lexer_start, text_size, text);
-	/* NUL terminate text section */
-	text[text_len] = '\0';
-	if (text_len < lexer_before)
-		lexer_before = text_len;
-
-	lua_getfield(L, -1, "load");
-	lua_pushstring(L, view->lexer_name);
-	lua_pcall(L, 1, 1, 0);
-
-	lua_getfield(L, -1, "_TOKENSTYLES");
-	lua_getfield(L, -2, "lex");
-
-	lua_pushvalue(L, -3); /* lexer obj */
-
-	const char *lex_text = text;
-	if (lexer_start > 0) {
-		/* try to start lexing at a line boundry */
-		/* TODO: start at known state, handle nested lexers */
-		const char *newline = memchr(text, '\n', lexer_before);
-		if (newline)
-			lex_text = newline;
-	}
-
-	lua_pushlstring(L, lex_text, text_len - (lex_text - text));
-	lua_pushinteger(L, 1 /* inital style: whitespace */);
-
-	int token_count;
-
-	if (lua_isfunction(L, -4) &&  !lua_pcall(L, 3, 1, 0) && lua_istable(L, -1) &&
-	   (token_count = lua_objlen(L, -1)) > 0) {
-
-		size_t pos = lexer_before - (lex_text - text);
-		Line *line = view->topline;
-		int col = 0;
-
-		for (int i = 1; i < token_count; i += 2) {
-			lua_rawgeti(L, -1, i);
-			//const char *name = lua_tostring(L, -1);
-			lua_gettable(L, -3); /* _TOKENSTYLES[token] */
-			size_t token_style = lua_tointeger(L, -1);
-			lua_pop(L, 1); /* style */
-			lua_rawgeti(L, -1, i + 1);
-			size_t token_end = lua_tointeger(L, -1) - 1;
-			lua_pop(L, 1); /* pos */
-
-			for (bool token_next = false; line; line = line->next, col = 0) {
-				for (; col < line->width; col++) {
-					if (pos < token_end) {
-						line->cells[col].style = token_style;
-						pos += line->cells[col].len;
-					} else {
-						token_next = true;
-						break;
-					}
-				}
-				if (token_next)
-					break;
-			}
-		}
-		lua_pop(L, 1);
-	}
-
-	lua_pop(L, 3); /* _TOKENSTYLES, language specific lexer, lexers global */
-}
-
-bool view_syntax_set(View *view, const char *name) {
-	lua_State *L = view->lua;
-	if (!L)
-		return name == NULL;
-
-	lua_getglobal(L, "vis");
-	lua_getfield(L, -1, "lexers");
-
-	static const struct {
-		enum UiStyle id;
-		const char *name;
-	} styles[] = {
-		{ UI_STYLE_DEFAULT,        "STYLE_DEFAULT"        },
-		{ UI_STYLE_CURSOR,         "STYLE_CURSOR"         },
-		{ UI_STYLE_CURSOR_PRIMARY, "STYLE_CURSOR_PRIMARY" },
-		{ UI_STYLE_CURSOR_LINE,    "STYLE_CURSOR_LINE"    },
-		{ UI_STYLE_SELECTION,      "STYLE_SELECTION"      },
-		{ UI_STYLE_LINENUMBER,     "STYLE_LINENUMBER"     },
-		{ UI_STYLE_COLOR_COLUMN,   "STYLE_COLOR_COLUMN"   },
-	};
-
-	for (size_t i = 0; i < LENGTH(styles); i++) {
-		lua_getfield(L, -1, styles[i].name);
-		view->ui->syntax_style(view->ui, styles[i].id, lua_tostring(L, -1));
-		lua_pop(L, 1);
-	}
-
-	if (!name) {
-		free(view->lexer_name);
-		view->lexer_name = NULL;
-		return true;
-	}
-
-	/* Try to load the specified lexer and parse its token styles.
-	 * Roughly equivalent to the following lua code:
-	 *
-	 * lang = vis.lexers.load(name)
-	 * for token_name, id in pairs(lang._TOKENSTYLES) do
-	 * 	ui->syntax_style(id, vis.lexers:get_style(lang, token_name);
-	 */
-	lua_getfield(L, -1, "load");
-	lua_pushstring(L, name);
-
-	if (lua_pcall(L, 1, 1, 0))
-		return false;
-
-	if (!lua_istable(L, -1)) {
-		lua_pop(L, 2);
-		return false;
-	}
-
-	view->lexer_name = strdup(name);
-	/* loop through all _TOKENSTYLES and parse them */
-	lua_getfield(L, -1, "_TOKENSTYLES");
-	lua_pushnil(L); /* first key */
-
-	while (lua_next(L, -2)) {
-		size_t id = lua_tointeger(L, -1);
-		//const char *name = lua_tostring(L, -2);
-		lua_pop(L, 1); /* remove value (=id), keep key (=name) */
-		lua_getfield(L, -4, "get_style");
-		lua_pushvalue(L, -5); /* lexer */
-		lua_pushvalue(L, -5); /* lang */
-		lua_pushvalue(L, -4); /* token_name */
-		if (lua_pcall(L, 3, 1, 0))
-			return false;
-		const char *style = lua_tostring(L, -1);
-		//printf("%s\t%d\t%s\n", name, id, style);
-		view->ui->syntax_style(view->ui, id, style);
-		lua_pop(L, 1); /* style */
-	}
-
-	lua_pop(L, 4); /* _TOKENSTYLES, grammar, lexers, vis */
-
-	return true;
-}
-
-#endif
-
-
 void view_tabwidth_set(View *view, int tabwidth) {
 	view->tabwidth = tabwidth;
 	view_draw(view);
@@ -615,7 +441,8 @@ void view_update(View *view) {
 	if (!view->need_update)
 		return;
 
-	view_syntax_color(view);
+	if (view->events->highlight)
+		view->events->highlight(view->events->data);
 
 	if (view->colorcolumn > 0) {
 		size_t lineno = 0;
@@ -719,7 +546,6 @@ void view_free(View *view) {
 	while (view->selections)
 		view_selections_free(view->selections);
 	free(view->lines);
-	free(view->lexer_name);
 	free(view);
 }
 
@@ -729,7 +555,7 @@ void view_reload(View *view, Text *text) {
 	view_cursor_to(view, 0);
 }
 
-View *view_new(Text *text, lua_State *lua) {
+View *view_new(Text *text, ViewEvent *events) {
 	if (!text)
 		return NULL;
 	View *view = calloc(1, sizeof(View));
@@ -741,7 +567,6 @@ View *view_new(Text *text, lua_State *lua) {
 	}
 
 	view->text = text;
-	view->lua = lua;
 	view->tabwidth = 8;
 	view->horizon = 1 << 15;
 	view_options_set(view, 0);
@@ -752,6 +577,7 @@ View *view_new(Text *text, lua_State *lua) {
 	}
 
 	view_cursor_to(view, 0);
+	view->events = events;
 
 	return view;
 }
@@ -1003,10 +829,6 @@ void view_scroll_to(View *view, size_t pos) {
 	view_cursors_scroll_to(view->cursor, pos);
 }
 
-const char *view_syntax_get(View *view) {
-	return view->lexer_name;
-}
-
 void view_options_set(View *view, enum UiOption options) {
 	const int mapping[] = {
 		[SYNTAX_SYMBOL_SPACE]    = UI_OPTION_SYMBOL_SPACE,
diff --git a/view.h b/view.h
index 1b2ea4e..ac036fb 100644
--- a/view.h
+++ b/view.h
@@ -3,19 +3,20 @@
 
 #include <stddef.h>
 #include <stdbool.h>
-#if CONFIG_LUA
-#include <lua.h>
-#else
-typedef struct lua_State lua_State;
-#endif
-#include "register.h"
-#include "text.h"
-#include "ui.h"
 
 typedef struct View View;
 typedef struct Cursor Cursor;
 typedef struct Selection Selection;
 
+#include "text.h"
+#include "ui.h"
+#include "register.h"
+
+typedef struct {
+	void *data;
+	void (*highlight)(void *data);
+} ViewEvent;
+
 typedef struct {
 	int width;          /* display width i.e. number of columns occupied by this character */
 	size_t len;         /* number of bytes the character displayed in this cell uses, for
@@ -40,7 +41,7 @@ struct Line {               /* a line on the screen, *not* in the file */
 	Cell cells[];       /* win->width cells storing information about the displayed characters */
 };
 
-View *view_new(Text*, lua_State*);
+View *view_new(Text*, ViewEvent*);
 void view_ui(View*, UiWin*);
 /* change associated text displayed in this window */
 void view_reload(View*, Text*);
@@ -89,9 +90,6 @@ Filerange view_viewport_get(View*);
  */
 bool view_viewport_up(View *view, int n);
 bool view_viewport_down(View *view, int n);
-/* associate a set of syntax highlighting rules to this window. */
-bool view_syntax_set(View*, const char *name);
-const char *view_syntax_get(View*);
 
 void view_options_set(View*, enum UiOption options);
 enum UiOption view_options_get(View*);
diff --git a/vis-cmds.c b/vis-cmds.c
index 3c67643..4c68d28 100644
--- a/vis-cmds.c
+++ b/vis-cmds.c
@@ -205,7 +205,7 @@ static bool cmd_set(Vis *vis, Win *win, Command *cmd, const char *argv[], Cursor
 		break;
 	case OPTION_SYNTAX:
 		if (!argv[2]) {
-			const char *syntax = view_syntax_get(win->view);
+			const char *syntax = vis_window_syntax_get(win);
 			if (syntax)
 				vis_info_show(vis, "Syntax definition in use: `%s'", syntax);
 			else
@@ -214,8 +214,8 @@ static bool cmd_set(Vis *vis, Win *win, Command *cmd, const char *argv[], Cursor
 		}
 
 		if (parse_bool(argv[2], &arg.b) && !arg.b)
-			return view_syntax_set(win->view, NULL);
-		if (!view_syntax_set(win->view, argv[2])) {
+			return vis_window_syntax_set(win, NULL);
+		if (!vis_window_syntax_set(win, argv[2])) {
 			vis_info_show(vis, "Unknown syntax definition: `%s'", argv[2]);
 			return false;
 		}
diff --git a/vis-core.h b/vis-core.h
index 6d3b35f..6d463df 100644
--- a/vis-core.h
+++ b/vis-core.h
@@ -3,6 +3,7 @@
 
 #include <setjmp.h>
 #include "vis.h"
+#include "vis-lua.h"
 #include "register.h"
 #include "text.h"
 #include "text-regex.h"
@@ -132,6 +133,8 @@ struct Win {
 	Mode modes[VIS_MODE_INVALID]; /* overlay mods used for per window key bindings */
 	Win *parent;            /* window which was active when showing the command prompt */
 	Mode *parent_mode;      /* mode which was active when showing the command prompt */
+	ViewEvent event;        /* callbacks from view.[ch] */
+	char *lexer_name;       /* corresponds to filename in lexers/ subdirectory */
 	Win *prev, *next;       /* neighbouring windows */
 };
 
diff --git a/vis-lua.c b/vis-lua.c
index 1988075..c239f63 100644
--- a/vis-lua.c
+++ b/vis-lua.c
@@ -28,6 +28,8 @@ void vis_lua_file_save(Vis *vis, File *file) { }
 void vis_lua_file_close(Vis *vis, File *file) { }
 void vis_lua_win_open(Vis *vis, Win *win) { }
 void vis_lua_win_close(Vis *vis, Win *win) { }
+void vis_lua_win_highlight(Vis *vis, Win *win) { }
+bool vis_lua_win_syntax(Vis *vis, Win *win, const char *syntax) { return true; }
 bool vis_theme_load(Vis *vis, const char *name) { return true; }
 
 #else
@@ -618,7 +620,7 @@ static int window_index(lua_State *L) {
 		}
 
 		if (strcmp(key, "syntax") == 0) {
-			const char *syntax = view_syntax_get(win->view);
+			const char *syntax = vis_window_syntax_get(win);
 			if (syntax)
 				lua_pushstring(L, syntax);
 			else
@@ -640,7 +642,7 @@ static int window_newindex(lua_State *L) {
 			const char *syntax = NULL;
 			if (!lua_isnil(L, 3))
 				syntax = luaL_checkstring(L, 3);
-			view_syntax_set(win->view, syntax);
+			vis_window_syntax_set(win, syntax);
 			return 0;
 		}
 	}
@@ -1274,6 +1276,34 @@ void vis_lua_win_close(Vis *vis, Win *win) {
 	lua_pop(L, 1);
 }
 
+void vis_lua_win_highlight(Vis *vis, Win *win) {
+	lua_State *L = vis->lua;
+	vis_lua_event_get(L, "win_highlight");
+	if (lua_isfunction(L, -1)) {
+		obj_ref_new(L, win, "vis.window");
+		pcall(vis, L, 1, 0);
+	}
+	lua_pop(L, 1);
+}
+
+bool vis_lua_win_syntax(Vis *vis, Win *win, const char *syntax) {
+	lua_State *L = vis->lua;
+	bool ret = false;
+	vis_lua_event_get(L, "win_syntax");
+	if (lua_isfunction(L, -1)) {
+		obj_ref_new(L, win, "vis.window");
+		if (syntax)
+			lua_pushstring(L, syntax);
+		else
+			lua_pushnil(L);
+		pcall(vis, L, 2, 1);
+		ret = lua_toboolean(L, -1);
+		lua_pop(L, 1);
+	}
+	lua_pop(L, 1);
+	return ret;
+}
+
 bool vis_theme_load(Vis *vis, const char *name) {
 	lua_State *L = vis->lua;
 	vis_lua_event_get(L, "theme_change");
diff --git a/vis-lua.h b/vis-lua.h
index 2b06e0a..7a37402 100644
--- a/vis-lua.h
+++ b/vis-lua.h
@@ -25,5 +25,7 @@ void vis_lua_file_save(Vis*, File*);
 void vis_lua_file_close(Vis*, File*);
 void vis_lua_win_open(Vis*, Win*);
 void vis_lua_win_close(Vis*, Win*);
+void vis_lua_win_highlight(Vis*, Win*);
+bool vis_lua_win_syntax(Vis*, Win*, const char *syntax);
 
 #endif
diff --git a/vis.c b/vis.c
index f35c50b..f23deba 100644
--- a/vis.c
+++ b/vis.c
@@ -149,9 +149,17 @@ static void window_free(Win *win) {
 	for (size_t i = 0; i < LENGTH(win->modes); i++)
 		map_free(win->modes[i].bindings);
 	ringbuf_free(win->jumplist);
+	free(win->lexer_name);
 	free(win);
 }
 
+static void window_highlight(void *ctx) {
+	Win *win = ctx;
+	Vis *vis = win->vis;
+	if (!win->file->internal && vis->event && vis->event->win_highlight)
+		vis->event->win_highlight(vis, win);
+}
+
 Win *window_new_file(Vis *vis, File *file) {
 	Win *win = calloc(1, sizeof(Win));
 	if (!win)
@@ -159,7 +167,8 @@ Win *window_new_file(Vis *vis, File *file) {
 	win->vis = vis;
 	win->file = file;
 	win->jumplist = ringbuf_alloc(31);
-	win->view = view_new(file->text, vis->lua);
+	win->event.data = win;
+	win->view = view_new(file->text, &win->event);
 	win->ui = vis->ui->window_new(vis->ui, win->view, file, UI_OPTION_STATUSBAR);
 	if (!win->jumplist || !win->view || !win->ui) {
 		window_free(win);
@@ -210,7 +219,7 @@ bool vis_window_split(Win *original) {
 	}
 	win->file = original->file;
 	win->file->refcount++;
-	view_syntax_set(win->view, view_syntax_get(original->view));
+	vis_window_syntax_set(win, vis_window_syntax_get(original));
 	view_options_set(win->view, view_options_get(original->view));
 	view_cursor_to(win->view, view_cursor_get(original->view));
 	vis_draw(win->vis);
@@ -242,6 +251,22 @@ void vis_window_prev(Vis *vis) {
 	vis_window_focus(sel);
 }
 
+const char *vis_window_syntax_get(Win *win) {
+	return win->lexer_name;
+}
+
+bool vis_window_syntax_set(Win *win, const char *syntax) {
+	Vis *vis = win->vis;
+	if (!win->file->internal && vis->event && vis->event->win_syntax) {
+		if (!vis->event->win_syntax(vis, win, syntax))
+			return false;
+	}
+	free(win->lexer_name);
+	win->lexer_name = syntax ? strdup(syntax) : NULL;
+	win->event.highlight = syntax ? window_highlight : NULL;
+	return !syntax || win->lexer_name;
+}
+
 void vis_draw(Vis *vis) {
 	vis->ui->draw(vis->ui);
 }
diff --git a/vis.h b/vis.h
index 32c1612..5e80ce8 100644
--- a/vis.h
+++ b/vis.h
@@ -33,6 +33,8 @@ typedef struct {
 	void (*file_close)(Vis*, File*);
 	void (*win_open)(Vis*, Win*);
 	void (*win_close)(Vis*, Win*);
+	void (*win_highlight)(Vis*, Win*);
+	bool (*win_syntax)(Vis*, Win*, const char *syntax);
 } VisEvent;
 
 typedef union { /* various types of arguments passed to key action functions */
@@ -98,6 +100,10 @@ void vis_window_prev(Vis*);
 void vis_window_focus(Win*);
 /* swap location of two windows */
 void vis_window_swap(Win*, Win*);
+
+const char *vis_window_syntax_get(Win*);
+bool vis_window_syntax_set(Win*, const char *name);
+
 /* display a user prompt with a certain title and default text */
 void vis_prompt_show(Vis*, const char *title);
 
diff --git a/vis.lua b/vis.lua
index 0ef715a..59d0590 100644
--- a/vis.lua
+++ b/vis.lua
@@ -222,3 +222,64 @@ vis.events.theme_change = function(name)
 	end
 end
 
+vis.events.win_syntax = function(win, name)
+	if name == nil then
+		return true
+	end
+	local lexers = vis.lexers
+	if lexers == nil then
+		return false
+	end
+	local lexer = lexers.load(name)
+	if not lexer then
+		return false
+	end
+
+	win:style_define(win.STYLE_DEFAULT, lexers.STYLE_DEFAULT)
+	win:style_define(win.STYLE_CURSOR, lexers.STYLE_CURSOR)
+	win:style_define(win.STYLE_CURSOR_PRIMARY, lexers.STYLE_CURSOR_PRIMARY)
+	win:style_define(win.STYLE_CURSOR_LINE, lexers.STYLE_CURSOR_LINE)
+	win:style_define(win.STYLE_SELECTION, lexers.STYLE_SELECTION)
+	win:style_define(win.STYLE_LINENUMBER, lexers.STYLE_LINENUMBER)
+	win:style_define(win.STYLE_COLOR_COLUMN, lexers.STYLE_COLOR_COLUMN)
+
+	for token_name, id in pairs(lexer._TOKENSTYLES) do
+		local style = lexers['STYLE_'..string.upper(token_name)] or lexer._EXTRASTYLES[token_name]
+		win:style_define(id, style)
+	end
+	return true
+end
+
+vis.events.win_highlight = function(win)
+	if win.syntax == nil or vis.lexers == nil then
+		return
+	end
+	local lexer = vis.lexers.load(win.syntax)
+	if lexer == nil then
+		return
+	end
+
+	-- TODO: improve heuristic for initial style
+	local viewport = win.viewport
+	local horizon_max = 32768
+	local horizon = viewport.start < horizon_max and viewport.start or horizon_max
+	local view_start = viewport.start
+	local lex_start = viewport.start - horizon
+	local token_start = lex_start
+	viewport.start = token_start
+	local data = win.file:content(viewport)
+	local token_styles = lexer._TOKENSTYLES
+	local tokens = lexer:lex(data, 1)
+
+	for i = 1, #tokens, 2 do
+		local token_end = lex_start + tokens[i+1] - 1
+		if token_end >= view_start then
+			local name = tokens[i]
+			local style = token_styles[name]
+			if style ~= nil then
+				win:style(style, token_start, token_end)
+			end
+		end
+		token_start = token_end
+	end
+end