fix(cmd): kong vault get doesn't work in dbless mode (#11127)

* fix(cmd): `kong vault get` doesn't work in dbless mode The cli `kong vault get <reference>` doesn't work in DBless mode if <reference> uses vaults entity. It doesn't affect the normal use of vault in kong instance though. The reason is in DBless mode the vaults entity is stored in LMDB which is implemented by a Nginx C module. However Everytime `resty` cli (which is relied on by `kong` cli) runs it creates a temporary `nginx.conf` which doesn't contain the lmdb-related directives. This PR is fixing this by starting another `resty` call with lmdb-related directives inserted via the `--main-conf` option. Note we only try this after detecting the `no LMDB environment defined` error in order to avoid infinite loop. And because `resty` will create a temmporary nginx instance so we need to convert the relative paths in the nginx.conf to the absolute path under kong instance prefix. [FTI-4937](https://konghq.atlassian.net/browse/FTI-4937) * add CHANGELOG * make it more robust * update comment * update comment * test the existence of LMDB rather than Kong instance * fixup * make the fix more generic * fix and add tests in 04-prefix_handler_spec * add lua_ssl_protocols and fix tests * rename the new configuration files to avoid conflict with the prefix of injected directives * add and fix tests of 14-vault_spec * fix test * rename template files to consistent with configuration file names * add unit tests for inject_directives.lua * change to absolute path * fixup * fix path * Update CHANGELOG.md Co-authored-by: Hans Hübner <[email protected]> * use return (...) syntax instead * don't expose the option and use a better name * pass paths instead of patterns and use better names * correctly handle the stdout/stderr/exit code * preserve original cli args for reusing * use env variable to terminate recursion * resty isn't necessarily in the position -1, so add it explicitly * update the lmdb_map_size to 2048m * fix(cmd): lack of necessary nginx directives in kong cli nginx.conf This is an alternative of (#10675)[#10675]. The primary logic keeps the same. The inject logic is further moved forward from `kong/cmd/init.lua` to `bin/kong` so that the execution flow won't enter `kong/cmd/init.lua` twice. We still keep the `bin/kong` a resty script because many files such as `kong.conf_loader`, `kong.cmd.utils.process_secrets` rely on `ngx`. If we change `bin/kong` into a pure lua or other language script, we need to rewrite the conf_loader and compile part logic. [FTI-4937](https://konghq.atlassian.net/browse/FTI-4937) * fix lint * fix test * fix test * use xpcall to catch exceptions and handle error message * add health to skip_inject_cmds * fix tests in 11-config_spec.lua * add hybrid into skip_inject_cmds * fix typo * remove CHANGELOG entry to the right place ("Unreleased") * extend load() to a subset of fields and these fields can't reference vault * add field `database` to CONF_NO_VAULT * fix test * fix test * keep `conf.nginx_http_lua_ssl_protocols` and `conf.nginx_stream_lua_ssl_protocols` so that we don't change the previous behavior * fixup * fix test * fix test * fix test * update CHANGELOG * Update CHANGELOG.md Co-authored-by: Qirui(Keery) Nie <[email protected]> * always call prepare_prefix as the prefix directory may not existed and the lua_ssl_trusted_certificate config may be updated --------- Co-authored-by: Hans Hübner <[email protected]> Co-authored-by: Qirui(Keery) Nie <[email protected]>
Kong · Jul 24, 2023 · 8a1ebba · 8a1ebba · khcp-gha-bot · Jul 24, 2023
1 parent e025bbd
commit 8a1ebba
Show file tree

Hide file tree

Showing 18 changed files with 765 additions and 240 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -97,6 +97,10 @@
 - Added new span attribute `net.peer.name` if balancer_data.hostname is available.
   Thanks [@backjo](https://github.com/backjo) for contributing this change.
   [#10723](https://github.com/Kong/kong/pull/10729)
+- Make `kong vault get` CLI command work in dbless mode by injecting the necessary directives into the kong cli nginx.conf.
+Meanwhile, the following Kong configurations cannot reference vaults as they are required for vaults initializing:
+`prefix`, `vaults`, `database`, `lmdb_environment_path`, `lmdb_map_size`, `lua_ssl_trusted_certificate`, `lua_ssl_protocols`, `nginx_http_lua_ssl_protocols`, `nginx_stream_lua_ssl_protocols`, `vault_*`.
+  [#10675](https://github.com/Kong/kong/pull/10675)
 
 #### Admin API
 

diff --git a/bin/kong b/bin/kong
@@ -1,11 +1,148 @@
 #!/usr/bin/env resty
 
+setmetatable(_G, nil)
+pcall(require, "luarocks.loader")
+package.path = (os.getenv("KONG_LUA_PATH_OVERRIDE") or "") .. "./?.lua;./?/init.lua;" .. package.path
+
+local pl_app = require "pl.lapp"
+local pl_utils = require "pl.utils"
+local pl_tablex = require "pl.tablex"
+local inject_confs = require "kong.cmd.utils.inject_confs"
+
+local options = [[
+ --v              verbose
+ --vv             debug
+]]
+
+local cmds_arr = {}
+local cmds = {
+  start = true,
+  stop = true,
+  quit = true,
+  restart = true,
+  reload = true,
+  health = true,
+  check = true,
+  prepare = true,
+  migrations = true,
+  version = true,
+  config = true,
+  roar = true,
+  hybrid = true,
+  vault = true,
+}
+
+-- unnecessary to inject nginx directives for these simple cmds
+local skip_inject_cmds = {
+  version = true,
+  roar = true,
+  check = true,
+  stop = true,
+  quit = true,
+  health = true,
+  hybrid = true,
+}
+
+for k in pairs(cmds) do
+  cmds_arr[#cmds_arr+1] = k
+end
+
+table.sort(cmds_arr)
+
+local help = string.format([[
+Usage: kong COMMAND [OPTIONS]
+
+The available commands are:
+ %s
+
+Options:
+%s]], table.concat(cmds_arr, "\n "), options)
+
+local cmd_name = table.remove(arg, 1)
+if not cmd_name then
+  pl_app(help)
+  pl_app.quit()
+elseif not cmds[cmd_name] then
+  pl_app(help)
+  pl_app.quit("No such command: " .. cmd_name)
+end
+
+local cmd = require("kong.cmd." .. cmd_name)
+local cmd_lapp = cmd.lapp
+
+if cmd_lapp then
+  cmd_lapp = cmd_lapp .. options -- append universal options
+  arg = pl_app(cmd_lapp)
+end
+
+-- check sub-commands
+if cmd.sub_commands then
+  local sub_cmd = table.remove(arg, 1)
+  if not sub_cmd then
+    pl_app.quit()
+  elseif not cmd.sub_commands[sub_cmd] then
+    pl_app.quit("No such command for " .. cmd_name .. ": " .. sub_cmd)
+  else
+    arg.command = sub_cmd
+  end
+end
+
+-- inject necessary nginx directives (e.g. lmdb_*, lua_ssl_*)
+-- into the temporary nginx.conf that `resty` will create
+local main_conf = ""
+local http_conf = ""
+local stream_conf = ""
+
+if not skip_inject_cmds[cmd_name] then
+  local pok, confs = xpcall(inject_confs.compile_confs, function(err)
+    if not (arg.v or arg.vv) then
+      err = err:match "^.-:.-:.(.*)$"
+      io.stderr:write("Error: " .. err .. "\n")
+      io.stderr:write("\n  Run with --v (verbose) or --vv (debug) for more details\n")
+    else
+      local trace = debug.traceback(err, 2)
+      io.stderr:write("Error: \n")
+      io.stderr:write(trace .. "\n")
+    end
+
+    pl_app.quit(nil, true)
+  end, arg)
+
+  main_conf = confs.main_conf
+  http_conf = confs.http_conf
+  stream_conf = confs.stream_conf
+end
+
+-- construct the args table
+local args_table = { "{" }
+for k, v in pairs(arg) do
+  if type(k) == "string" then
+    k = "\"" .. k .. "\""
+  end
+  if type(v) == "string" then
+    v = "\"" .. v .. "\""
+  end
+
+  table.insert(args_table, string.format("[%s] = %s,", k, v))
+end
+table.insert(args_table, "}")
+
+local args_str = table.concat(args_table, " ")
+
+local inline_code = string.format([[
 setmetatable(_G, nil)
 
 pcall(require, "luarocks.loader")
 
 package.path = (os.getenv("KONG_LUA_PATH_OVERRIDE") or "") .. "./?.lua;./?/init.lua;" .. package.path
 
-require("kong.cmd.init")(arg)
+require("kong.cmd.init")("%s", %s)
+]], cmd_name, args_str)
+
+local resty_cmd = string.format(
+  "resty --main-conf \"%s\" --http-conf \"%s\" --stream-conf \"%s\" -e '%s'",
+  main_conf, http_conf, stream_conf, inline_code)
 
+local _, code = pl_utils.execute(resty_cmd)
+os.exit(code)
 -- vim: set ft=lua ts=2 sw=2 sts=2 et :
diff --git a/kong-3.4.0-0.rockspec b/kong-3.4.0-0.rockspec
@@ -87,6 +87,9 @@ build = {
     ["kong.templates.nginx_kong_gui_include"] = "kong/templates/nginx_kong_gui_include.lua",
     ["kong.templates.nginx_kong_stream"] = "kong/templates/nginx_kong_stream.lua",
     ["kong.templates.kong_defaults"] = "kong/templates/kong_defaults.lua",
+    ["kong.templates.nginx_inject"] = "kong/templates/nginx_inject.lua",
+    ["kong.templates.nginx_kong_inject"] = "kong/templates/nginx_kong_inject.lua",
+    ["kong.templates.nginx_kong_stream_inject"] = "kong/templates/nginx_kong_stream_inject.lua",
     ["kong.templates.kong_yml"] = "kong/templates/kong_yml.lua",
 
     ["kong.resty.dns.client"] = "kong/resty/dns/client.lua",
@@ -119,6 +122,7 @@ build = {
     ["kong.cmd.utils.nginx_signals"] = "kong/cmd/utils/nginx_signals.lua",
     ["kong.cmd.utils.prefix_handler"] = "kong/cmd/utils/prefix_handler.lua",
     ["kong.cmd.utils.process_secrets"] = "kong/cmd/utils/process_secrets.lua",
+    ["kong.cmd.utils.inject_confs"] = "kong/cmd/utils/inject_confs.lua",
 
     ["kong.api"] = "kong/api/init.lua",
     ["kong.api.api_helpers"] = "kong/api/api_helpers.lua",

diff --git a/kong/cmd/init.lua b/kong/cmd/init.lua
@@ -12,75 +12,10 @@ local function stop_timers()
   end
 end
 
-local options = [[
- --v              verbose
- --vv             debug
-]]
-
-local cmds_arr = {}
-local cmds = {
-  start = true,
-  stop = true,
-  quit = true,
-  restart = true,
-  reload = true,
-  health = true,
-  check = true,
-  prepare = true,
-  migrations = true,
-  version = true,
-  config = true,
-  roar = true,
-  hybrid = true,
-  vault = true,
-}
-
-for k in pairs(cmds) do
-  cmds_arr[#cmds_arr+1] = k
-end
-
-table.sort(cmds_arr)
-
-local help = string.format([[
-Usage: kong COMMAND [OPTIONS]
-
-The available commands are:
- %s
-
-Options:
-%s]], table.concat(cmds_arr, "\n "), options)
-
-return function(args)
-  local cmd_name = table.remove(args, 1)
-  if not cmd_name then
-    pl_app(help)
-    pl_app.quit()
-  elseif not cmds[cmd_name] then
-    pl_app(help)
-    pl_app.quit("No such command: " .. cmd_name)
-  end
-
+return function(cmd_name, args)
   local cmd = require("kong.cmd." .. cmd_name)
-  local cmd_lapp = cmd.lapp
   local cmd_exec = cmd.execute
 
-  if cmd_lapp then
-    cmd_lapp = cmd_lapp .. options -- append universal options
-    args = pl_app(cmd_lapp)
-  end
-
-  -- check sub-commands
-  if cmd.sub_commands then
-    local sub_cmd = table.remove(args, 1)
-    if not sub_cmd then
-      pl_app.quit()
-    elseif not cmd.sub_commands[sub_cmd] then
-      pl_app.quit("No such command for " .. cmd_name .. ": " .. sub_cmd)
-    else
-      args.command = sub_cmd
-    end
-  end
-
   -- verbose mode
   if args.v then
     log.set_lvl(log.levels.verbose)

diff --git a/kong/cmd/utils/inject_confs.lua b/kong/cmd/utils/inject_confs.lua
@@ -0,0 +1,95 @@
+local conf_loader = require "kong.conf_loader"
+local pl_path = require "pl.path"
+local pl_stringx = require "pl.stringx"
+local prefix_handler = require "kong.cmd.utils.prefix_handler"
+local log = require "kong.cmd.utils.log"
+local fmt = string.format
+
+local compile_nginx_main_inject_conf = prefix_handler.compile_nginx_main_inject_conf
+local compile_nginx_http_inject_conf = prefix_handler.compile_nginx_http_inject_conf
+local compile_nginx_stream_inject_conf = prefix_handler.compile_nginx_stream_inject_conf
+local prepare_prefix = prefix_handler.prepare_prefix
+
+local function load_conf(args)
+  -- retrieve default prefix or use given one
+  log.disable()
+  local conf = assert(conf_loader(args.conf, {
+    prefix = args.prefix
+  }, { pre_cmd = true }))
+  log.enable()
+
+  if pl_path.exists(conf.kong_env) then
+    -- load <PREFIX>/kong.conf containing running node's config
+    conf = assert(conf_loader(conf.kong_env))
+  end
+
+  -- make sure necessary files like `.ca_combined` exist
+  -- but skip_write to avoid overwriting the existing nginx configurations
+  assert(prepare_prefix(conf, nil, true))
+
+  return conf
+end
+
+-- convert relative path to absolute path
+-- as resty will run a temporary nginx instance
+local function convert_directive_path_to_absolute(prefix, nginx_conf, paths)
+  local new_conf = nginx_conf
+
+  for _, path in ipairs(paths) do
+    local pattern = fmt("(%s) (.+);", path)
+    local m, err = ngx.re.match(new_conf, pattern)
+    if err then
+      return nil, err
+
+    elseif m then
+      local path = pl_stringx.strip(m[2])
+
+      if path:sub(1, 1) ~= '/' then
+        local absolute_path = prefix .. "/" .. path
+        local replace = "$1 " .. absolute_path .. ";"
+        local _, err
+        new_conf, _, err = ngx.re.sub(new_conf, pattern, replace)
+
+        if not new_conf then
+          return nil, err
+        end
+      end
+    end
+  end
+
+  return new_conf, nil
+end
+
+local function compile_main_inject(conf)
+  local nginx_main_inject_conf, err = compile_nginx_main_inject_conf(conf)
+  if not nginx_main_inject_conf then
+    return nil, err
+  end
+
+  -- path directives that needs to be converted
+  local paths = {
+    "lmdb_environment_path",
+  }
+  return convert_directive_path_to_absolute(conf.prefix, nginx_main_inject_conf, paths)
+end
+
+local function compile_http_inject(conf)
+  return compile_nginx_http_inject_conf(conf)
+end
+
+local function compile_stream_inject(conf)
+  return compile_nginx_stream_inject_conf(conf)
+end
+
+local function compile_confs(args)
+  local conf = load_conf(args)
+  local main_conf = assert(compile_main_inject(conf))
+  local http_conf = assert(compile_http_inject(conf))
+  local stream_conf = assert(compile_stream_inject(conf))
+
+  return { main_conf = main_conf, http_conf = http_conf, stream_conf = stream_conf, }
+end
+
+return {
+  compile_confs = compile_confs,
+}