feat: improve input processing and add extra channel in zencode-exec

.editorconfig

@@ -1,8 +1,11 @@
-# Copyright 2017-2023 Dyne.org foundation
-# SPDX-FileCopyrightText: 2017-2021 Dyne.org foundation
+# Copyright 2023 Dyne.org foundation
+# SPDX-FileCopyrightText: 2023 Dyne.org foundation
 #
 # SPDX-License-Identifier: AGPL-3.0-or-later
 
+# stop search for .editorconfig in parent directories
+root = true
+
 [*]
 end_of_line = lf
 insert_final_newline = true

bindings/README.md

@@ -40,8 +40,12 @@ The `zencode-data-keys-conf` is a file or stream with a newline separated list o
 2. zencode script (string -> base64) *newline*
 3. keys (json -> base64) *newline*
 4. data (json -> base64) *newline*
+5. extra (json -> base64) *newline*
+6. context (json -> base64) *newline*
 
-Each line should start directly with the base64 string without any prefix and should end with a newline. Anything else will likely be rejected.
+Each line should start directly with the base64 string without any prefix and should end with a newline.
+
+Empty lines can be a newline (or CRLF) and will be skipped, but should never be omitted: the zencode-exec expects 6 input lines in total, not less, including empty ones, all newline terminated. 
 
 This executes and returns two streams:
 1. `stdout` with the `json` formatted results of the execution

src/lua/statemachine.lua

@@ -118,25 +118,6 @@ function machine:cannot(e)
   return not self:can(e)
 end
 
-function machine:todot(filename)
-  local dotfile = io.open(filename,'w')
-  dotfile:write('digraph {\n')
-  local transition = function(event,from,to)
-    dotfile:write(string.format('%s -> %s [label=%s];\n',from,to,event))
-  end
-  for _, event in pairs(self.options.events) do
-    if type(event.from) == 'table' then
-      for _, from in ipairs(event.from) do
-        transition(event.name,from,event.to)
-      end
-    else
-      transition(event.name,event.from,event.to)
-    end
-  end
-  dotfile:write('}\n')
-  dotfile:close()
-end
-
 function machine:transition(event)
   if self.currentTransitioningEvent == event then
     return self[self.currentTransitioningEvent](self)

src/lua/zencode.lua

@@ -364,7 +364,6 @@ end
 
 -- Zencode HEAP globals
 IN = {} -- Given processing, import global DATA from json
-KIN = {} -- Given processing, import global KEYS from json
 TMP = TMP or {} -- Given processing, temp buffer for ack*->validate->push*
 ACK = ACK or {} -- When processing,  destination for push*
 OUT = OUT or {} -- print out
@@ -538,7 +537,6 @@ function zencode:begin()
 	-- Reset HEAP
 	self.machine = {}
 	IN = {} -- Given processing, import global DATA from json
-	KIN = {} -- Given processing, import global KEYS from json
 	TMP = {} -- Given processing, temp buffer for ack*->validate->push*
 	ACK = {} -- When processing,  destination for push*
 	OUT = {} -- print out
@@ -736,35 +734,45 @@ function zencode:run()
 	end
 	-- HEAP setup
 	IN = {} -- import global DATA from json
+	local tmp
+	if EXTRA then
+		tmp  = CONF.input.format.fun(EXTRA) or {}
+		for k, v in pairs(tmp) do
+			IN[k] = v
+		end
+		EXTRA = nil
+	end
 	if DATA then
-		-- if plain array conjoin into associative
-		IN = CONF.input.format.fun(DATA) or {}
+		tmp  = CONF.input.format.fun(DATA) or {}
+		for k, v in pairs(tmp) do
+			if IN[k] then
+				error("Object name collision in input: "..k)
+			end
+			IN[k] = v
+		end
 		DATA = nil
 	end
-	KIN = {} -- import global KEYS from json
 	if KEYS then
-		KIN = CONF.input.format.fun(KEYS) or {}
+		tmp  = CONF.input.format.fun(KEYS) or {}
+		for k, v in pairs(tmp) do
+			if IN[k] then
+				error("Object name collision in input: "..k)
+			end
+			IN[k] = v
+		end
 		KEYS = nil
 	end
+	tmp = nil
 	collectgarbage 'collect'
 
 	-- convert all spaces in keys to underscore
 	IN = IN_uscore(IN)
-	KIN = IN_uscore(KIN)
-
-	-- check name collisions between DATA and KEYS
-	if CONF.heap.check_collision then
-	   for k in pairs(IN) do
-	      if KIN[k] then
-		 error("Object name collision in input: "..k)
-	      end
-	   end
-	end
 
 	-- EXEC zencode
 	-- TODO: for optimization, to develop a lua iterator, which would save lookup time
 	-- https://www.lua.org/pil/7.1.html
-	while ZEN.next_instruction <= #self.AST do
+	local AST_size = #self.AST
+	while ZEN.next_instruction <= AST_size do
 		ZEN.current_instruction = ZEN.next_instruction
 		local x = self.AST[ZEN.current_instruction]
 		ZEN.next_instruction = ZEN.next_instruction + 1
@@ -778,7 +786,6 @@ function zencode:run()
 		-- trigger upon switch to when or then section
 		if x.from == 'given' and x.to ~= 'given' then
 			-- delete IN memory
-			KIN = {}
 			IN = {}
 			collectgarbage 'collect'
 		end
@@ -843,7 +850,6 @@ end
 function zencode.heap()
 	return ({
 		IN = IN,
-		KIN = KIN,
 		TMP = TMP,
 		ACK = ACK,
 		OUT = OUT

src/lua/zencode_debug.lua

@@ -61,7 +61,6 @@ local function debug_heap_dump()
 			  OCTET.from_string(
 				 JSON.encode(
 					{GIVEN_data = HEAP.IN,
-					 GIVEN_keys = HEAP.KIN,
 					 CODEC = ZEN.CODEC,
 					 WHEN = ack,
 					 THEN = HEAP.OUT}))))
@@ -72,7 +71,6 @@ local function debug_heap_dump()
 		 ack.keyring = '(hidden)'
 	  end
 	  I.warn({a_GIVEN_in = HEAP.IN,
-			  b_GIVEN_in = HEAP.KIN,
 			  c_WHEN_ack = ack,
 			  c_CODEC_ack = ZEN.CODEC,
 			  d_THEN_out = HEAP.OUT})

src/lua/zencode_dictionary.lua

@@ -231,7 +231,7 @@ When("create the copy of object named by '' from dictionary ''", function(name,
 end)
 
 local function take_out_f(path, dest, format)
-	local parr = strtok(uscore(path), '([^.]+)')
+	local parr = strtok(uscore(path), '.')
 	local root = parr[1] -- first
 	table.remove(parr, 1)
 	if not dest then

src/lua/zencode_given.lua

@@ -60,7 +60,7 @@ local function pick(what, conv)
    local data
    local raw
    local name = _index_to_string(what)
-   raw = KIN[name] or IN[name]
+   raw = IN[name]
    if not raw then error("Cannot find '" .. name .. "' anywhere (null value?)", 2) end
    if raw == '' then error("Found empty string in '" .. name) end
    -- if not conv and ZEN.schemas[what] then conv = what end
@@ -94,13 +94,7 @@ local function pickin(section, what, conv, fail)
    local bail  -- fail
    local name = _index_to_string(what)
 
-   if KIN[section] then
-      root = KIN[section]
-   elseif IN[section] then
-      root = IN[section]
-   else
-      root = nil
-   end
+   root = IN[section]
    if not root then
       error("Cannot find '"..section.."'", 2)
    end
@@ -253,7 +247,7 @@ Given(
    'nothing',
    function()
       ZEN.assert(
-         (next(IN) == nil) and (next(KIN) == nil),
+         (next(IN) == nil),
          'Undesired data passed as input'
       )
    end
@@ -354,7 +348,7 @@ Given(
    "a '' named by ''",
    function(s, n)
       -- local name = have(n)
-      local name = _index_to_string(KIN[n] or IN[n])
+      local name = _index_to_string(IN[n])
       -- ZEN.assert(encoder, "Invalid input encoding for '"..n.."': "..s)
       pick(name, s)
       ack(name)
@@ -374,7 +368,7 @@ Given(
 Given(
    "a '' named by '' in ''",
    function(s, n, t)
-      local name = _index_to_string(KIN[n] or IN[n])
+      local name = _index_to_string(IN[n])
       pickin(t, name, s)
       ack(name) -- save it in ACK.name
       gc()
@@ -404,7 +398,7 @@ Given(
    "my '' named by ''",
    function(s, n)
       -- ZEN.assert(encoder, "Invalid input encoding for '"..n.."': "..s)
-      local name = _index_to_string(KIN[n] or IN[n])
+      local name = _index_to_string(IN[n])
       pickin(WHO, name, s)
       ack(name)
       gc()
@@ -441,7 +435,7 @@ Given(
 )
 
 Given("a '' part of '' after string prefix ''", function(enc, src, pfx)
-		 local whole = KIN[src] or IN[src]
+		 local whole = IN[src]
 		 ZEN.assert(whole, "Cannot find '" .. src .. "' anywhere (null value?)")
 		 local plen = #pfx
 		 local wlen = #whole
@@ -457,7 +451,7 @@ Given("a '' part of '' after string prefix ''", function(enc, src, pfx)
 end)
 
 Given("a '' part of '' before string suffix ''", function(enc, src, sfx)
-		 local whole = KIN[src] or IN[src]
+		 local whole = IN[src]
 		 ZEN.assert(whole, "Cannot find '" .. src .. "' anywhere (null value?)")
 		 local slen = #sfx
 		 local wlen = #whole
@@ -473,11 +467,11 @@ Given("a '' part of '' before string suffix ''", function(enc, src, sfx)
 end)
 
 Given("a '' in path ''", function(enc, path)
-    local path_array = strtok(uscore(path), '([^.]+)')
+    local path_array = strtok(uscore(path), '.')
     local root = path_array[1]
     table.remove(path_array, 1)
     local dest = path_array[#path_array]
-    local res = KIN[root] or IN[root]
+    local res = IN[root]
     for k,v in pairs(path_array) do
         ZEN.assert(luatype(res) == 'table', "Object is not a table: "..root)
         ZEN.assert(res[v] ~= nil, "Key "..v.." not found in "..root)

src/lua/zenroom_common.lua

@@ -72,6 +72,27 @@ function xxx(s, n)
    end
 end
 
+function strtok(str, delimiter)
+    delimiter = delimiter or ' '
+    local result = {}
+    local start = 1
+    local delimiterPos = string.find(str, delimiter, start, true)
+
+    while delimiterPos do
+        local token = string.sub(str, start, delimiterPos - 1)
+        table.insert(result, token)
+        start = delimiterPos + 1
+        delimiterPos = string.find(str, delimiter, start, true)
+    end
+
+    local lastToken = string.sub(str, start)
+    if lastToken ~= "" then
+        table.insert(result, lastToken)
+    end
+
+    return result
+end
+
 -- sorted iterator for deterministic ordering of tables
 -- from: https://www.lua.org/pil/19.3.html
 _G["lua_pairs"]  = _G["pairs"]

src/zen_parse.c

@@ -191,6 +191,11 @@ static int lua_unserialize_json(lua_State* L) {
 	return 0;
 }
 
+// removed because of unexplained segfault when used inside pcall to
+// parse zencode: set_rule and set_scenario will explode, also seems
+// to perform worse than pure Lua (see PR #709)
+
+#if 0
 char* strtok_single(char* str, char const* delims)
 {
     static char* src = NULL;
@@ -218,15 +223,18 @@ static int lua_strtok(lua_State* L) {
 	const char DEFAULT_SEP[] = " ";
 
 	char copy[MAX_FILE];
-	char *sep = DEFAULT_SEP;
+	const char *sep = DEFAULT_SEP;
 
 	const char *in;
 	size_t size;
 	register int i = 1;
 	register char *token;
 
 	in = luaL_checklstring(L, 1, &size);
-
+	if(!in) {
+		lua_pushnil(L);
+		return 1;
+	}
 	if (lua_gettop(L) > 1) {
 		sep = luaL_checklstring(L, 2, NULL);
 	}
@@ -246,6 +254,7 @@ static int lua_strtok(lua_State* L) {
 	}
 	return 1;
 }
+#endif
 
 void zen_add_parse(lua_State *L) {
 	// override print() and io.write()
@@ -255,7 +264,6 @@ void zen_add_parse(lua_State *L) {
 		  {"trim", lua_trim_spaces},
 		  {"trimq", lua_trim_quotes},
 		  {"jsontok", lua_unserialize_json},
-		  {"strtok", lua_strtok},
 		  {NULL, NULL} };
 	lua_getglobal(L, "_G");
 	luaL_setfuncs(L, custom_parser, 0);  // for Lua versions 5.2 or greater