Implementation of the non-blocking process running Lua API

Rationale
A modern text editor usually includes tools for helping user
to avoid mistakes in texts. Those tools include spell checkers and
programming language integrations. Though vis explicitly states
that the full featured IDE is not a goal, implementing some of
the tools might be achieved using its Lua API. Unfortunatelly
the API misses the ability to start a process and to perform
a communication with it without completely blocking the editor UI,
which is crucial for any tool that performs background tracking of
the inserted text (e. g. language servers).

Implementation details
New feature introduces new API method: communicate. The method
start a new process and returns a handle to communicate with
the process instantly. The patch inserts stderr and stdout
file descriptors of the process to the pselect call of the main loop
used for reading user input to track the process state without
blocking the main loop until the process is finished.
Any changes in the process state cause the iteration of the main loop
and are being exposed to the Lua API as new event: PROCESS_RESPONSE.

7 files changed, 350 insertions, 1 deletions

diff --git a/Makefile b/Makefile
index da8de29..5969f8b 100644
--- a/Makefile
+++ b/Makefile
@@ -26,6 +26,7 @@ SRC = array.c \
 	vis-prompt.c \
 	vis-registers.c \
 	vis-text-objects.c \
+	vis-subprocess.c \
 	$(REGEX_SRC)
 OBJ = $(SRC:%.c=obj/%.o)
 
diff --git a/lua/vis.lua b/lua/vis.lua
index 39649c1..f2f9421 100644
--- a/lua/vis.lua
+++ b/lua/vis.lua
@@ -152,6 +152,7 @@ local events = {
 	WIN_OPEN = "Event::WIN_OPEN", -- see @{win_open}
 	WIN_STATUS = "Event::WIN_STATUS", -- see @{win_status}
 	TERM_CSI = "Event::TERM_CSI", -- see @{term_csi}
+	PROCESS_RESPONSE = "Event::PROCESS_RESPONSE", -- see @{process_response}
 }
 
 events.file_close = function(...) events.emit(events.FILE_CLOSE, ...) end
@@ -167,6 +168,7 @@ events.win_highlight = function(...) events.emit(events.WIN_HIGHLIGHT, ...) end
 events.win_open = function(...) events.emit(events.WIN_OPEN, ...) end
 events.win_status = function(...) events.emit(events.WIN_STATUS, ...) end
 events.term_csi = function(...) events.emit(events.TERM_CSI, ...) end
+events.process_response = function(...) events.emit(events.PROCESS_RESPONSE, ...) end
 
 local handlers = {}
 
diff --git a/vis-lua.c b/vis-lua.c
index c3a164a..164a967 100644
--- a/vis-lua.c
+++ b/vis-lua.c
@@ -164,6 +164,9 @@ void vis_lua_win_close(Vis *vis, Win *win) { }
 void vis_lua_win_highlight(Vis *vis, Win *win) { }
 void vis_lua_win_status(Vis *vis, Win *win) { window_status_update(vis, win); }
 void vis_lua_term_csi(Vis *vis, const long *csi) { }
+void vis_lua_process_response(Vis *vis, const char *name,
+                              char *buffer, size_t len, ResponseType rtype) { }
+
 
 #else
 
@@ -1378,6 +1381,56 @@ static int redraw(lua_State *L) {
 	return 0;
 }
 /***
+ * Closes a stream returned by @{Vis:communicate}.
+ *
+ * @function close
+ * @tparam io.file inputfd the stream to be closed
+ * @treturn bool identical to @{io.close}
+ */
+static int close_subprocess(lua_State *L) {
+	luaL_Stream *file = luaL_checkudata(L, -1, "FILE*");
+	int result = fclose(file->f);
+	if (result == 0) {
+		file->f = NULL;
+		file->closef = NULL;
+	}
+	return luaL_fileresult(L, result == 0, NULL);
+}
+/***
+ * Open new process and return its input stream (stdin).
+ * If the stream is closed (by calling the close method or by being removed by a garbage collector)
+ * the spawned process will be killed by SIGTERM.
+ * When the process will quit or will output anything to stdout or stderr,
+ * the @{process_response} event will be fired.
+ *
+ * The editor core won't be blocked while the external process is running.
+ *
+ * @function communicate
+ * @tparam string name the name of subprocess (to distinguish processes in the @{process_response} event)
+ * @tparam string command the command to execute
+ * @return the file handle to write data to the process, in case of error the return values are equivalent to @{io.open} error values.
+ */
+static int communicate_func(lua_State *L) {
+
+	typedef struct {
+		/* Lua stream structure for the process input stream */
+		luaL_Stream stream;
+		Process *handler;
+	} ProcessStream;
+
+	Vis *vis = obj_ref_check(L, 1, "vis");
+	const char *name = luaL_checkstring(L, 2);
+	const char *cmd = luaL_checkstring(L, 3);
+	ProcessStream *inputfd = (ProcessStream *)lua_newuserdata(L, sizeof(ProcessStream));
+	luaL_setmetatable(L, LUA_FILEHANDLE);
+	inputfd->handler = vis_process_communicate(vis, name, cmd, &(inputfd->stream.closef));
+	if (inputfd->handler) {
+		inputfd->stream.f = fdopen(inputfd->handler->inpfd, "w");
+		inputfd->stream.closef = &close_subprocess;
+	}
+	return inputfd->stream.f ? 1 : luaL_fileresult(L, 0, name);
+}
+/***
  * Currently active window.
  * @tfield Window win
  * @see windows
@@ -1590,6 +1643,7 @@ static const struct luaL_Reg vis_lua[] = {
 	{ "exit", exit_func },
 	{ "pipe", pipe_func },
 	{ "redraw", redraw },
+	{ "communicate", communicate_func },
 	{ "__index", vis_index },
 	{ "__newindex", vis_newindex },
 	{ NULL, NULL },
@@ -3538,5 +3592,42 @@ void vis_lua_term_csi(Vis *vis, const long *csi) {
 	}
 	lua_pop(L, 1);
 }
+/***
+ * The response received from the process started via @{Vis:communicate}.
+ * @function process_response
+ * @tparam string name the name of process given to @{Vis:communicate}
+ * @tparam string response_type can be "STDOUT" or "STDERR" if new output was received in corresponding channel, "SIGNAL" if the process was terminated by a signal or "EXIT" when the process terminated normally
+ * @tparam int the exit code number if response\_type is "EXIT", or the signal number if response\_type is "SIGNAL"
+ * @tparam string buffer the available content sent by process
+ */
+void vis_lua_process_response(Vis *vis, const char *name,
+                              char *buffer, size_t len, ResponseType rtype) {
+	lua_State *L = vis->lua;
+	if (!L) {
+		return;
+	}
+	vis_lua_event_get(L, "process_response");
+	if (lua_isfunction(L, -1)) {
+		lua_pushstring(L, name);
+		switch (rtype) {
+		case STDOUT: lua_pushstring(L, "STDOUT"); break;
+		case STDERR: lua_pushstring(L, "STDERR"); break;
+		case SIGNAL: lua_pushstring(L, "SIGNAL"); break;
+		case EXIT: lua_pushstring(L, "EXIT"); break;
+		}
+		switch (rtype) {
+		case EXIT:
+		case SIGNAL:
+			lua_pushinteger(L, len);
+			lua_pushnil(L);
+			break;
+		default:
+			lua_pushnil(L);
+			lua_pushlstring(L, buffer, len);
+		}
+		pcall(vis, L, 4, 0);
+	}
+	lua_pop(L, 1);
+}
 
 #endif
diff --git a/vis-lua.h b/vis-lua.h
index da64233..d622ff5 100644
--- a/vis-lua.h
+++ b/vis-lua.h
@@ -5,11 +5,14 @@
 #include <lua.h>
 #include <lualib.h>
 #include <lauxlib.h>
+
 #else
 typedef struct lua_State lua_State;
+typedef void* lua_CFunction;
 #endif
 
 #include "vis.h"
+#include "vis-subprocess.h"
 
 /* add a directory to consider when loading lua files */
 bool vis_lua_path_add(Vis*, const char *path);
@@ -38,5 +41,6 @@ void vis_lua_win_close(Vis*, Win*);
 void vis_lua_win_highlight(Vis*, Win*);
 void vis_lua_win_status(Vis*, Win*);
 void vis_lua_term_csi(Vis*, const long *);
+void vis_lua_process_response(Vis *, const char *, char *, size_t, ResponseType);
 
 #endif
diff --git a/vis-subprocess.c b/vis-subprocess.c
new file mode 100644
index 0000000..5359820
--- /dev/null
+++ b/vis-subprocess.c
@@ -0,0 +1,218 @@
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdbool.h>
+#include <errno.h>
+#include <string.h>
+#include <sys/wait.h>
+#include "vis-lua.h"
+#include "vis-subprocess.h"
+#include "util.h"
+
+/* Pool of information about currently running subprocesses */
+static Process *process_pool;
+
+/**
+ * Adds new empty process information structure to the process pool and
+ * returns it
+ * @return a new Process instance
+ */
+Process *new_process_in_pool() {
+	Process *newprocess = malloc(sizeof(Process));
+	if (!newprocess) {
+		return NULL;
+	}
+	newprocess->next = process_pool;
+	process_pool = newprocess;
+	return newprocess;
+}
+
+/**
+ * Removes the subprocess information from the pool, sets invalidator to NULL
+ * and frees resources.
+ * @param a reference to a reference to the process to be removed
+ * @return the next process in the pool
+ */
+static void destroy_process(Process **pointer) {
+	Process *target = *pointer;
+	if (target->outfd != -1) {
+		close(target->outfd);
+	}
+	if (target->errfd != -1) {
+		close(target->errfd);
+	}
+	if (target->inpfd != -1) {
+		close(target->inpfd);
+	}
+	/* marking stream as closed for lua */
+	if (target->invalidator) {
+		*(target->invalidator) = NULL;
+	}
+	if (target->name) {
+		free(target->name);
+	}
+	*pointer = target->next;
+	free(target);
+}
+
+/**
+ * Starts new subprocess by passing the `command` to the shell and
+ * returns the subprocess information structure, containing file descriptors
+ * of the process.
+ * Also stores the subprocess information to the internal pool to track
+ * its status and responses.
+ * @param name a string that contains a unique name for the subprocess.
+ * This name will be passed to the PROCESS_RESPONSE event handler
+ * to distinguish running subprocesses.
+ * @param command a command to be executed to spawn a process
+ * @param invalidator a pointer to the pointer which shows that the subprocess
+ * is invalid when set to NULL. When the subprocess dies, it is set to NULL.
+ * If a caller sets the pointer to NULL the subprocess will be killed on the
+ * next main loop iteration.
+ */
+Process *vis_process_communicate(Vis *vis, const char *name,
+                                 const char *command, Invalidator **invalidator) {
+	int pin[2], pout[2], perr[2];
+	pid_t pid = (pid_t)-1;
+	if (pipe(perr) == -1) {
+		goto closeerr;
+	}
+	if (pipe(pout) == -1) {
+		goto closeouterr;
+	}
+	if (pipe(pin) == -1) {
+		goto closeall;
+	}
+	pid = fork();
+	if (pid == -1) {
+		vis_info_show(vis, "fork failed: %s", strerror(errno));
+	} else if (pid == 0) { /* child process */
+		sigset_t sigterm_mask;
+		sigemptyset(&sigterm_mask);
+		sigaddset(&sigterm_mask, SIGTERM);
+		if (sigprocmask(SIG_UNBLOCK, &sigterm_mask, NULL) == -1) {
+			fprintf(stderr, "failed to reset signal mask");
+			exit(EXIT_FAILURE);
+		}
+		dup2(pin[0], STDIN_FILENO);
+		dup2(pout[1], STDOUT_FILENO);
+		dup2(perr[1], STDERR_FILENO);
+	} else { /* main process */
+		Process *new = new_process_in_pool();
+		if (!new) {
+			vis_info_show(vis, "Cannot create process: %s", strerror(errno));
+			goto closeall;
+		}
+		new->name = strdup(name);
+		if (!new->name) {
+			vis_info_show(vis, "Cannot copy process name: %s", strerror(errno));
+			/* pop top element (which is `new`) from the pool */
+			destroy_process(&process_pool);
+			goto closeall;
+		}
+		new->outfd = pout[0];
+		new->errfd = perr[0];
+		new->inpfd = pin[1];
+		new->pid = pid;
+		new->invalidator = invalidator;
+		close(pin[0]);
+		close(pout[1]);
+		close(perr[1]);
+		return new;
+	}
+closeall:
+	close(pin[0]);
+	close(pin[1]);
+closeouterr:
+	close(pout[0]);
+	close(pout[1]);
+closeerr:
+	close(perr[0]);
+	close(perr[1]);
+	if (pid == 0) { /* start command in child process */
+		execlp(vis->shell, vis->shell, "-c", command, (char*)NULL);
+		fprintf(stderr, "exec failed: %s(%d)\n", strerror(errno), errno);
+		exit(1);
+	} else {
+		vis_info_show(vis, "process creation failed: %s", strerror(errno));
+	}
+	return NULL;
+}
+
+/**
+ * Adds file descriptors of currently running subprocesses to the `readfds`
+ * to track their readiness and returns maximum file descriptor value
+ * to pass it to the `pselect` call
+ * @param readfds the structure for `pselect` call to fill
+ * @return maximum file descriptor number in the readfds structure
+ */
+int vis_process_before_tick(fd_set *readfds) {
+	int maxfd = 0;
+	for (Process **pointer = &process_pool; *pointer; pointer = &((*pointer)->next)) {
+		Process *current = *pointer;
+		if (current->outfd != -1) {
+			FD_SET(current->outfd, readfds);
+			maxfd = maxfd < current->outfd ? current->outfd : maxfd;
+		}
+		if (current->errfd != -1) {
+			FD_SET(current->errfd, readfds);
+			maxfd = maxfd < current->errfd ? current->errfd : maxfd;
+		}
+	}
+	return maxfd;
+}
+
+/**
+ * Reads data from the given subprocess file descriptor `fd` and fires
+ * the PROCESS_RESPONSE event in Lua with given subprocess `name`,
+ * `rtype` and the read data as arguments.
+ * @param fd the file descriptor to read data from
+ * @param name a name of the subprocess
+ * @param rtype a type of file descriptor where the new data is found
+ */
+static void read_and_fire(Vis* vis, int fd, const char *name, ResponseType rtype) {
+	static char buffer[PIPE_BUF];
+	size_t obtained = read(fd, &buffer, PIPE_BUF-1);
+	if (obtained > 0) {
+		vis_lua_process_response(vis, name, buffer, obtained, rtype);
+	}
+}
+
+/**
+ * Checks if `readfds` contains file descriptors of subprocesses from
+ * the pool. If so, it reads their data and fires corresponding events.
+ * Also checks if each subprocess from the pool is dead or needs to be
+ * killed then raises an event or kills it if necessary.
+ * @param readfds the structure for `pselect` call with file descriptors
+ */
+void vis_process_tick(Vis *vis, fd_set *readfds) {
+	for (Process **pointer = &process_pool; *pointer; ) {
+		Process *current = *pointer;
+		if (current->outfd != -1 && FD_ISSET(current->outfd, readfds)) {
+			read_and_fire(vis, current->outfd, current->name, STDOUT);
+		}
+		if (current->errfd != -1 && FD_ISSET(current->errfd, readfds)) {
+			read_and_fire(vis, current->errfd, current->name, STDERR);
+		}
+		int status;
+		pid_t wpid = waitpid(current->pid, &status, WNOHANG);
+		if (wpid == -1)	{
+			vis_message_show(vis, strerror(errno));
+		} else if (wpid == current->pid) {
+			goto just_destroy;
+		} else if (!*(current->invalidator)) {
+			goto kill_and_destroy;
+		}
+		pointer = &current->next;
+		continue;
+kill_and_destroy:
+		kill(current->pid, SIGTERM);
+		waitpid(current->pid, &status, 0);
+just_destroy:
+		if (WIFSIGNALED(status)) {
+			vis_lua_process_response(vis, current->name, NULL, WTERMSIG(status), SIGNAL);
+		} else {
+			vis_lua_process_response(vis, current->name, NULL, WEXITSTATUS(status), EXIT);
+		}
+		destroy_process(pointer);
+	}
+}
diff --git a/vis-subprocess.h b/vis-subprocess.h
new file mode 100644
index 0000000..2e4c222
--- /dev/null
+++ b/vis-subprocess.h
@@ -0,0 +1,30 @@
+#ifndef VIS_SUBPROCESS_H
+#define VIS_SUBPROCESS_H
+#include "vis-core.h"
+#include "vis-lua.h"
+#include <sys/select.h>
+
+typedef struct Process Process;
+#if CONFIG_LUA
+typedef int Invalidator(lua_State*);
+#else
+typedef void Invalidator;
+#endif
+
+struct Process {
+	char *name;
+	int outfd;
+	int errfd;
+	int inpfd;
+	pid_t pid;
+	Invalidator** invalidator;
+	Process *next;
+};
+
+typedef enum { STDOUT, STDERR, SIGNAL, EXIT } ResponseType;
+
+Process *vis_process_communicate(Vis *, const char *command, const char *name,
+                                 Invalidator **invalidator);
+int vis_process_before_tick(fd_set *);
+void vis_process_tick(Vis *, fd_set *);
+#endif
diff --git a/vis.c b/vis.c
index d374d39..739e5c6 100644
--- a/vis.c
+++ b/vis.c
@@ -28,6 +28,7 @@
 #include "vis-core.h"
 #include "sam.h"
 #include "ui.h"
+#include "vis-subprocess.h"
 
 
 static void macro_replay(Vis *vis, const Macro *macro);
@@ -1437,7 +1438,8 @@ int vis_run(Vis *vis) {
 
 		vis_update(vis);
 		idle.tv_sec = vis->mode->idle_timeout;
-		int r = pselect(1, &fds, NULL, NULL, timeout, &emptyset);
+		int r = pselect(vis_process_before_tick(&fds) + 1, &fds, NULL, NULL,
+		                timeout, &emptyset);
 		if (r == -1 && errno == EINTR)
 			continue;
 
@@ -1445,6 +1447,7 @@ int vis_run(Vis *vis) {
 			/* TODO save all pending changes to a ~suffixed file */
 			vis_die(vis, "Error in mainloop: %s\n", strerror(errno));
 		}
+		vis_process_tick(vis, &fds);
 
 		if (!FD_ISSET(STDIN_FILENO, &fds)) {
 			if (vis->mode->idle)