From 22ac91ee730332a6890bfc613e180c62bc7d08fb Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 09:39:01 +0800
Subject: [PATCH 01/13] tests notify new version

---
 .../09-notify_new_version_spec.lua            | 78 +++++++++++++++++++
 1 file changed, 78 insertions(+)
 create mode 100644 spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
new file mode 100644
index 00000000000..dd4094c57bb
--- /dev/null
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -0,0 +1,78 @@
+local helpers = require "spec.helpers"
+
+
+-- register a rpc connected event in custom plugin rpc-notify-new-version-test
+-- DISABLE rpc sync on cp side
+-- ENABLE rpc sync on dp side
+for _, strategy in helpers.each_strategy() do
+  describe("Hybrid Mode RPC #" .. strategy, function()
+
+    lazy_setup(function()
+      helpers.get_db_utils(strategy, {
+        "clustering_data_planes",
+      }) -- runs migrations
+
+      assert(helpers.start_kong({
+        role = "control_plane",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        database = strategy,
+        cluster_listen = "127.0.0.1:9005",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        plugins = "bundled,rpc-notify-new-version-test",
+        nginx_worker_processes = 4, -- multiple workers
+        cluster_rpc = "on", -- enable rpc
+        cluster_rpc_sync = "off", -- disable rpc sync
+      }))
+
+      assert(helpers.start_kong({
+        role = "data_plane",
+        database = "off",
+        prefix = "servroot2",
+        cluster_cert = "spec/fixtures/kong_clustering.crt",
+        cluster_cert_key = "spec/fixtures/kong_clustering.key",
+        cluster_control_plane = "127.0.0.1:9005",
+        proxy_listen = "0.0.0.0:9002",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        plugins = "bundled,rpc-notify-new-version-test",
+        nginx_worker_processes = 4, -- multiple workers
+        cluster_rpc = "on", -- enable rpc
+        cluster_rpc_sync = "on", -- enable rpc sync
+      }))
+    end)
+
+    lazy_teardown(function()
+      helpers.stop_kong("servroot2")
+      helpers.stop_kong()
+    end)
+
+    describe("sync.v2.notify_new_version works", function()
+      it("on dp side", function()
+        local name = "servroot2/logs/error.log"
+
+        -- dp logs
+        helpers.pwait_until(function()
+          assert.logfile(name).has.line(
+            "kong.sync.v2.get_delta ok", true)
+          assert.logfile(name).has.no.line(
+            "assertion failed", true)
+          assert.logfile(name).has.no.line(
+            "[error]", true)
+          return true
+        end, 10)
+
+        local name = nil
+
+        -- cp logs
+        helpers.pwait_until(function()
+          assert.logfile(name).has.no.line(
+            "assertion failed", true)
+          assert.logfile(name).has.no.line(
+            "[error]", true)
+          return true
+        end, 10)
+
+      end)
+    end)
+  end)
+end -- for _, strategy

From b5de60cf60fda4977ec7718fb29a8e6d8312a891 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 10:05:50 +0800
Subject: [PATCH 02/13] plugins

---
 .../rpc-notify-new-version-test/handler.lua   | 39 +++++++++++++++++++
 .../rpc-notify-new-version-test/schema.lua    | 12 ++++++
 2 files changed, 51 insertions(+)
 create mode 100644 spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
 create mode 100644 spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/schema.lua

diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
new file mode 100644
index 00000000000..027c8c1b8c0
--- /dev/null
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -0,0 +1,39 @@
+local rep = string.rep
+local isempty = require("table.isempty")
+
+
+local RpcSyncV2NotifyNewVersioinTestHandler = {
+  VERSION = "1.0",
+  PRIORITY = 1000,
+}
+
+
+function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
+  -- mock function on cp side
+  kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
+    local latest_version = "v02_" .. string.rep("1", 28)
+
+    local deltas = {}
+
+    return { default = { deltas = deltas, wipe = true, }, }
+  end)
+
+  -- call dp's sync.v2.notify_new_version
+  kong.rpc.callbacks:register("kong.test.notify_new_version", function(node_id)
+  end)
+
+  local worker_events = assert(kong.worker_events)
+
+  -- if rpc is ready we will send test calls
+  worker_events.register(function(capabilities_list)
+    local node_id = "control_plane"
+
+    local res, err = kong.rpc:call(node_id, "kong.test.notify_new_version")
+
+    ngx.log(ngx.DEBUG, "kong.sync.v2.notify_new_version ok")
+
+  end, "clustering:jsonrpc", "connected")
+end
+
+
+return RpcSyncV2NotifyNewVersioinTestHandler
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/schema.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/schema.lua
new file mode 100644
index 00000000000..f920dcb599c
--- /dev/null
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/schema.lua
@@ -0,0 +1,12 @@
+return {
+  name = "rpc-notify-new-version-test",
+  fields = {
+    {
+      config = {
+        type = "record",
+        fields = {
+        },
+      },
+    },
+  },
+}

From 3eb07e7267c2430c1d41fba6adb4deac3e2c9953 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 10:21:57 +0800
Subject: [PATCH 03/13] mock get_delta

---
 .../rpc-notify-new-version-test/handler.lua   | 25 ++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 027c8c1b8c0..27c3543d3f0 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -13,7 +13,30 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
   kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
     local latest_version = "v02_" .. string.rep("1", 28)
 
-    local deltas = {}
+    local fake_uuid1 = "00000000-0000-0000-0000-111111111111"
+    local fake_uuid2 = "00000000-0000-0000-0000-222222222222"
+
+    -- a basic config data
+    local deltas = {
+      {
+        entity = {
+          id = fake_uuid1,
+          name = "default",
+        },
+        type = "workspaces",
+        version = latest_version,
+        ws_id = fake_uuid1,
+      },
+      {
+        entity = {
+          key = "cluster_id",
+          value = fake_uuid2,
+        },
+        type = "parameters",
+        version = latest_version,
+        ws_id = fake_uuid1,
+      }
+    }
 
     return { default = { deltas = deltas, wipe = true, }, }
   end)

From fedc9fc438d3caac98b805b6926d3d245d7a505b Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 12:47:49 +0800
Subject: [PATCH 04/13] basic tests

---
 .../09-notify_new_version_spec.lua            | 28 ++++++++-----------
 .../rpc-notify-new-version-test/handler.lua   |  7 ++++-
 2 files changed, 18 insertions(+), 17 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index dd4094c57bb..d798516172d 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -51,26 +51,22 @@ for _, strategy in helpers.each_strategy() do
         local name = "servroot2/logs/error.log"
 
         -- dp logs
-        helpers.pwait_until(function()
-          assert.logfile(name).has.line(
-            "kong.sync.v2.get_delta ok", true)
-          assert.logfile(name).has.no.line(
-            "assertion failed", true)
-          assert.logfile(name).has.no.line(
-            "[error]", true)
-          return true
-        end, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.notify_new_version ok", true, 10)
+        assert.logfile(name).has.no.line(
+          "assertion failed", true, 0)
+        assert.logfile(name).has.no.line(
+          "[error]", true, 0)
 
         local name = nil
 
         -- cp logs
-        helpers.pwait_until(function()
-          assert.logfile(name).has.no.line(
-            "assertion failed", true)
-          assert.logfile(name).has.no.line(
-            "[error]", true)
-          return true
-        end, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok", true, 10)
+        assert.logfile(name).has.no.line(
+          "assertion failed", true, 0)
+        assert.logfile(name).has.no.line(
+          "[error]", true, 0)
 
       end)
     end)
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 27c3543d3f0..d73ce087fe6 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -11,7 +11,7 @@ local RpcSyncV2NotifyNewVersioinTestHandler = {
 function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
   -- mock function on cp side
   kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
-    local latest_version = "v02_" .. string.rep("1", 28)
+    local latest_version = string.format("v02_%028d", 10)
 
     local fake_uuid1 = "00000000-0000-0000-0000-111111111111"
     local fake_uuid2 = "00000000-0000-0000-0000-222222222222"
@@ -38,11 +38,14 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
       }
     }
 
+    ngx.log(ngx.DEBUG, "kong.sync.v2.get_delta ok")
+
     return { default = { deltas = deltas, wipe = true, }, }
   end)
 
   -- call dp's sync.v2.notify_new_version
   kong.rpc.callbacks:register("kong.test.notify_new_version", function(node_id)
+    return true
   end)
 
   local worker_events = assert(kong.worker_events)
@@ -52,6 +55,8 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     local node_id = "control_plane"
 
     local res, err = kong.rpc:call(node_id, "kong.test.notify_new_version")
+    assert(res == true)
+    assert(not err)
 
     ngx.log(ngx.DEBUG, "kong.sync.v2.notify_new_version ok")
 

From 6d166368f8642cc0aa9ecd7ea55c13d5821dd9ce Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 12:59:16 +0800
Subject: [PATCH 05/13] more cases

---
 .../rpc-notify-new-version-test/handler.lua   | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index d73ce087fe6..a9ef492116f 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -43,8 +43,24 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     return { default = { deltas = deltas, wipe = true, }, }
   end)
 
-  -- call dp's sync.v2.notify_new_version
+  -- test dp's sync.v2.notify_new_version
   kong.rpc.callbacks:register("kong.test.notify_new_version", function(node_id)
+
+    local dp_node_id = next(kong.rpc.clients)
+    local method = "kong.sync.v2.notify_new_version"
+
+    -- no default
+    local msg = {}
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(not res)
+    assert(err == "default namespace does not exist inside params")
+
+    -- no default.new_version
+    local msg = { default = {}, }
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(not res)
+    assert(err == "'new_version' key does not exist")
+
     return true
   end)
 
@@ -54,6 +70,7 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
   worker_events.register(function(capabilities_list)
     local node_id = "control_plane"
 
+    -- trigger cp's test
     local res, err = kong.rpc:call(node_id, "kong.test.notify_new_version")
     assert(res == true)
     assert(not err)

From c1a77e71e70312c4a49c775bdb2bd4a71357ea78 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 13:08:39 +0800
Subject: [PATCH 06/13] lint fix

---
 .../09-notify_new_version_spec.lua            |  6 ++-
 .../rpc-notify-new-version-test/handler.lua   | 40 +++++++++++--------
 2 files changed, 28 insertions(+), 18 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index d798516172d..b034e6eb0bb 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -62,7 +62,11 @@ for _, strategy in helpers.each_strategy() do
 
         -- cp logs
         assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok", true, 10)
+          "kong.sync.v2.get_delta ok: 1", true, 10)
+        assert.logfile(name).has.line(
+          "kong.test.notify_new_version ok", true, 10)
+        assert.logfile(name).has.no.line(
+          "kong.sync.v2.get_delta ok: 2", true, 0)
         assert.logfile(name).has.no.line(
           "assertion failed", true, 0)
         assert.logfile(name).has.no.line(
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index a9ef492116f..e4321dec4ec 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -1,5 +1,4 @@
-local rep = string.rep
-local isempty = require("table.isempty")
+local fmt = string.format
 
 
 local RpcSyncV2NotifyNewVersioinTestHandler = {
@@ -10,35 +9,28 @@ local RpcSyncV2NotifyNewVersioinTestHandler = {
 
 function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
   -- mock function on cp side
+  local counter = 0
+
   kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
-    local latest_version = string.format("v02_%028d", 10)
+    local latest_version = fmt("v02_%028d", 10)
 
-    local fake_uuid1 = "00000000-0000-0000-0000-111111111111"
-    local fake_uuid2 = "00000000-0000-0000-0000-222222222222"
+    local fake_uuid = "00000000-0000-0000-0000-111111111111"
 
     -- a basic config data
     local deltas = {
       {
         entity = {
-          id = fake_uuid1,
+          id = fake_uuid,
           name = "default",
         },
         type = "workspaces",
         version = latest_version,
-        ws_id = fake_uuid1,
+        ws_id = fake_uuid,
       },
-      {
-        entity = {
-          key = "cluster_id",
-          value = fake_uuid2,
-        },
-        type = "parameters",
-        version = latest_version,
-        ws_id = fake_uuid1,
-      }
     }
 
-    ngx.log(ngx.DEBUG, "kong.sync.v2.get_delta ok")
+    counter = counter + 1
+    ngx.log(ngx.DEBUG, "kong.sync.v2.get_delta ok: ", counter)
 
     return { default = { deltas = deltas, wipe = true, }, }
   end)
@@ -61,6 +53,20 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(not res)
     assert(err == "'new_version' key does not exist")
 
+    -- same version number
+    local msg = { default = { new_version = fmt("v02_%028d", 10), }, }
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(res)
+    assert(not err)
+
+    -- less version number
+    local msg = { default = { new_version = fmt("v02_%028d", 5), }, }
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(res)
+    assert(not err)
+
+    ngx.log(ngx.DEBUG, "kong.test.notify_new_version ok")
+
     return true
   end)
 

From a7713e12c8f2493af0a8d0c2fffab0bc1451225c Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 13:19:19 +0800
Subject: [PATCH 07/13] more tests

---
 .../09-notify_new_version_spec.lua            | 20 ++++++++++++++++---
 .../rpc-notify-new-version-test/handler.lua   |  6 ++++++
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index b034e6eb0bb..722c7aced7c 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -53,20 +53,34 @@ for _, strategy in helpers.each_strategy() do
         -- dp logs
         assert.logfile(name).has.line(
           "kong.sync.v2.notify_new_version ok", true, 10)
+        assert.logfile(name).has.line(
+          "sync_once retry count exceeded. retry_count: 6", true, 10)
         assert.logfile(name).has.no.line(
           "assertion failed", true, 0)
-        assert.logfile(name).has.no.line(
-          "[error]", true, 0)
 
         local name = nil
 
         -- cp logs
         assert.logfile(name).has.line(
           "kong.sync.v2.get_delta ok: 1", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 2", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 3", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 4", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 5", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 6", true, 10)
+        assert.logfile(name).has.line(
+          "kong.sync.v2.get_delta ok: 7", true, 10)
+
         assert.logfile(name).has.line(
           "kong.test.notify_new_version ok", true, 10)
+
         assert.logfile(name).has.no.line(
-          "kong.sync.v2.get_delta ok: 2", true, 0)
+          "kong.sync.v2.get_delta ok: 8", true, 0)
         assert.logfile(name).has.no.line(
           "assertion failed", true, 0)
         assert.logfile(name).has.no.line(
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index e4321dec4ec..d1726c99018 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -65,6 +65,12 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(res)
     assert(not err)
 
+    -- greater version number
+    local msg = { default = { new_version = fmt("v02_%028d", 20), }, }
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(res)
+    assert(not err)
+
     ngx.log(ngx.DEBUG, "kong.test.notify_new_version ok")
 
     return true

From 3135a730eebd3f6b6983529ab7a5d89412af9e6b Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 13:24:34 +0800
Subject: [PATCH 08/13] clean

---
 .../09-notify_new_version_spec.lua            | 22 +++++--------------
 .../rpc-notify-new-version-test/handler.lua   |  7 +++---
 2 files changed, 8 insertions(+), 21 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index 722c7aced7c..0501ef42324 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -52,7 +52,7 @@ for _, strategy in helpers.each_strategy() do
 
         -- dp logs
         assert.logfile(name).has.line(
-          "kong.sync.v2.notify_new_version ok", true, 10)
+          "kong.test.notify_new_version ok", true, 10)
         assert.logfile(name).has.line(
           "sync_once retry count exceeded. retry_count: 6", true, 10)
         assert.logfile(name).has.no.line(
@@ -61,26 +61,14 @@ for _, strategy in helpers.each_strategy() do
         local name = nil
 
         -- cp logs
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 1", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 2", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 3", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 4", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 5", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 6", true, 10)
-        assert.logfile(name).has.line(
-          "kong.sync.v2.get_delta ok: 7", true, 10)
+        for i = 1, 7 do
+          assert.logfile(name).has.line(
+            "kong.sync.v2.get_delta ok: " .. i, true, 10)
+        end
 
         assert.logfile(name).has.line(
           "kong.test.notify_new_version ok", true, 10)
 
-        assert.logfile(name).has.no.line(
-          "kong.sync.v2.get_delta ok: 8", true, 0)
         assert.logfile(name).has.no.line(
           "assertion failed", true, 0)
         assert.logfile(name).has.no.line(
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index d1726c99018..377ef3f62f2 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -8,9 +8,9 @@ local RpcSyncV2NotifyNewVersioinTestHandler = {
 
 
 function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
-  -- mock function on cp side
   local counter = 0
 
+  -- mock function on cp side
   kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
     local latest_version = fmt("v02_%028d", 10)
 
@@ -35,9 +35,8 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     return { default = { deltas = deltas, wipe = true, }, }
   end)
 
-  -- test dp's sync.v2.notify_new_version
+  -- test dp's sync.v2.notify_new_version on cp side
   kong.rpc.callbacks:register("kong.test.notify_new_version", function(node_id)
-
     local dp_node_id = next(kong.rpc.clients)
     local method = "kong.sync.v2.notify_new_version"
 
@@ -87,7 +86,7 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(res == true)
     assert(not err)
 
-    ngx.log(ngx.DEBUG, "kong.sync.v2.notify_new_version ok")
+    ngx.log(ngx.DEBUG, "kong.test.notify_new_version ok")
 
   end, "clustering:jsonrpc", "connected")
 end

From a1074699526e5f4f0719f355dbe54656d0e22f72 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 13:34:38 +0800
Subject: [PATCH 09/13] tries

---
 kong/clustering/services/sync/rpc.lua                     | 2 ++
 .../18-hybrid_rpc/09-notify_new_version_spec.lua          | 5 +++++
 .../kong/plugins/rpc-notify-new-version-test/handler.lua  | 8 ++++----
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/kong/clustering/services/sync/rpc.lua b/kong/clustering/services/sync/rpc.lua
index 56e2a9f1d27..b05daccd221 100644
--- a/kong/clustering/services/sync/rpc.lua
+++ b/kong/clustering/services/sync/rpc.lua
@@ -141,6 +141,8 @@ function _M:init_dp(manager)
       return self:sync_once()
     end
 
+    ngx_log(ngx_DEBUG, "no sync runs, version is ", version)
+
     return true
   end)
 end
diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index 0501ef42324..8b134e07405 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -48,11 +48,16 @@ for _, strategy in helpers.each_strategy() do
 
     describe("sync.v2.notify_new_version works", function()
       it("on dp side", function()
+        --local fmt = string.format
         local name = "servroot2/logs/error.log"
 
         -- dp logs
         assert.logfile(name).has.line(
           "kong.test.notify_new_version ok", true, 10)
+        --assert.logfile(name).has.line(
+        --  "no sync runs, version is " .. fmt("v02_%028x", 10), true, 10)
+        --assert.logfile(name).has.line(
+        --  "no sync runs, version is " .. fmt("v02_%028x", 5), true, 10)
         assert.logfile(name).has.line(
           "sync_once retry count exceeded. retry_count: 6", true, 10)
         assert.logfile(name).has.no.line(
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 377ef3f62f2..8721f02158d 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -12,7 +12,7 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
 
   -- mock function on cp side
   kong.rpc.callbacks:register("kong.sync.v2.get_delta", function(node_id, current_versions)
-    local latest_version = fmt("v02_%028d", 10)
+    local latest_version = fmt("v02_%028x", 10)
 
     local fake_uuid = "00000000-0000-0000-0000-111111111111"
 
@@ -53,19 +53,19 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(err == "'new_version' key does not exist")
 
     -- same version number
-    local msg = { default = { new_version = fmt("v02_%028d", 10), }, }
+    local msg = { default = { new_version = fmt("v02_%028x", 10), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)
     assert(res)
     assert(not err)
 
     -- less version number
-    local msg = { default = { new_version = fmt("v02_%028d", 5), }, }
+    local msg = { default = { new_version = fmt("v02_%028x", 5), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)
     assert(res)
     assert(not err)
 
     -- greater version number
-    local msg = { default = { new_version = fmt("v02_%028d", 20), }, }
+    local msg = { default = { new_version = fmt("v02_%028x", 20), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)
     assert(res)
     assert(not err)

From 8c55c89c8f51b7eb4923d789121cf61702989aee Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 15:51:44 +0800
Subject: [PATCH 10/13] test for no sync

---
 .../18-hybrid_rpc/09-notify_new_version_spec.lua  | 12 +++++++-----
 .../rpc-notify-new-version-test/handler.lua       | 15 +++++++++++++++
 2 files changed, 22 insertions(+), 5 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index 8b134e07405..3e2d74d527c 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -1,3 +1,4 @@
+local fmt = string.format
 local helpers = require "spec.helpers"
 
 
@@ -48,16 +49,17 @@ for _, strategy in helpers.each_strategy() do
 
     describe("sync.v2.notify_new_version works", function()
       it("on dp side", function()
-        --local fmt = string.format
         local name = "servroot2/logs/error.log"
 
         -- dp logs
         assert.logfile(name).has.line(
           "kong.test.notify_new_version ok", true, 10)
-        --assert.logfile(name).has.line(
-        --  "no sync runs, version is " .. fmt("v02_%028x", 10), true, 10)
-        --assert.logfile(name).has.line(
-        --  "no sync runs, version is " .. fmt("v02_%028x", 5), true, 10)
+
+        assert.logfile(name).has.line(
+          "no sync runs, version is " .. fmt("v02_%028x", 10), true, 10)
+        assert.logfile(name).has.line(
+          "no sync runs, version is " .. fmt("v02_%028x", 5), true, 10)
+
         assert.logfile(name).has.line(
           "sync_once retry count exceeded. retry_count: 6", true, 10)
         assert.logfile(name).has.no.line(
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 8721f02158d..60716c66032 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -1,3 +1,7 @@
+local declarative = require("kong.db.declarative")
+local DECLARATIVE_EMPTY_CONFIG_HASH = require("kong.constants").DECLARATIVE_EMPTY_CONFIG_HASH
+
+
 local fmt = string.format
 
 
@@ -79,6 +83,17 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
 
   -- if rpc is ready we will send test calls
   worker_events.register(function(capabilities_list)
+    -- wait dp's first sync finish
+    for i = 1, 100 do
+      local ver = declarative.get_current_hash()
+      if ver ~= DECLARATIVE_EMPTY_CONFIG_HASH then
+        break
+      end
+      ngx.sleep(0.05)
+    end
+
+    -- now dp's version should be "v02_0000a"
+
     local node_id = "control_plane"
 
     -- trigger cp's test

From 148f85b70bf92aa63e641b425c053cae0d8feecd Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 15:54:04 +0800
Subject: [PATCH 11/13] clean

---
 .../rpc-notify-new-version-test/handler.lua   | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 60716c66032..c1586c97be9 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -1,8 +1,22 @@
-local declarative = require("kong.db.declarative")
-local DECLARATIVE_EMPTY_CONFIG_HASH = require("kong.constants").DECLARATIVE_EMPTY_CONFIG_HASH
+local fmt = string.format
 
 
-local fmt = string.format
+local wait_first_sync_ok
+do
+  local declarative = require("kong.db.declarative")
+  local DECLARATIVE_EMPTY_CONFIG_HASH = require("kong.constants").DECLARATIVE_EMPTY_CONFIG_HASH
+
+  -- wait dp's first sync finish
+  wait_first_sync_ok = function()
+    for i = 1, 100 do
+      local ver = declarative.get_current_hash()
+      if ver ~= DECLARATIVE_EMPTY_CONFIG_HASH then
+        return
+      end
+      ngx.sleep(0.05)
+    end
+  end
+end
 
 
 local RpcSyncV2NotifyNewVersioinTestHandler = {
@@ -83,14 +97,7 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
 
   -- if rpc is ready we will send test calls
   worker_events.register(function(capabilities_list)
-    -- wait dp's first sync finish
-    for i = 1, 100 do
-      local ver = declarative.get_current_hash()
-      if ver ~= DECLARATIVE_EMPTY_CONFIG_HASH then
-        break
-      end
-      ngx.sleep(0.05)
-    end
+    wait_first_sync_ok()
 
     -- now dp's version should be "v02_0000a"
 

From 452d8ef19260aac4861cc223eacb4127c2f8e4b7 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 16:04:30 +0800
Subject: [PATCH 12/13] test less version

---
 .../09-notify_new_version_spec.lua            |  6 ++--
 .../rpc-notify-new-version-test/handler.lua   | 36 +++----------------
 2 files changed, 7 insertions(+), 35 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index 3e2d74d527c..d432ea552ee 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -1,4 +1,4 @@
-local fmt = string.format
+local rep = string.rep
 local helpers = require "spec.helpers"
 
 
@@ -56,9 +56,7 @@ for _, strategy in helpers.each_strategy() do
           "kong.test.notify_new_version ok", true, 10)
 
         assert.logfile(name).has.line(
-          "no sync runs, version is " .. fmt("v02_%028x", 10), true, 10)
-        assert.logfile(name).has.line(
-          "no sync runs, version is " .. fmt("v02_%028x", 5), true, 10)
+          "no sync runs, version is " .. rep(".", 32), true, 10)
 
         assert.logfile(name).has.line(
           "sync_once retry count exceeded. retry_count: 6", true, 10)
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index c1586c97be9..154ad0906e6 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -1,22 +1,5 @@
 local fmt = string.format
-
-
-local wait_first_sync_ok
-do
-  local declarative = require("kong.db.declarative")
-  local DECLARATIVE_EMPTY_CONFIG_HASH = require("kong.constants").DECLARATIVE_EMPTY_CONFIG_HASH
-
-  -- wait dp's first sync finish
-  wait_first_sync_ok = function()
-    for i = 1, 100 do
-      local ver = declarative.get_current_hash()
-      if ver ~= DECLARATIVE_EMPTY_CONFIG_HASH then
-        return
-      end
-      ngx.sleep(0.05)
-    end
-  end
-end
+local rep = string.rep
 
 
 local RpcSyncV2NotifyNewVersioinTestHandler = {
@@ -70,19 +53,14 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(not res)
     assert(err == "'new_version' key does not exist")
 
-    -- same version number
-    local msg = { default = { new_version = fmt("v02_%028x", 10), }, }
+    -- less version string
+    -- "....." < "00000" < "v02_xx"
+    local msg = { default = { new_version = rep(".", 32), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)
     assert(res)
     assert(not err)
 
-    -- less version number
-    local msg = { default = { new_version = fmt("v02_%028x", 5), }, }
-    local res, err = kong.rpc:call(dp_node_id, method, msg)
-    assert(res)
-    assert(not err)
-
-    -- greater version number
+    -- greater version string
     local msg = { default = { new_version = fmt("v02_%028x", 20), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)
     assert(res)
@@ -97,10 +75,6 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
 
   -- if rpc is ready we will send test calls
   worker_events.register(function(capabilities_list)
-    wait_first_sync_ok()
-
-    -- now dp's version should be "v02_0000a"
-
     local node_id = "control_plane"
 
     -- trigger cp's test

From 293e9ccf4ad8b516797bd32f9cb84d710d861b72 Mon Sep 17 00:00:00 2001
From: chronolaw <chrono_cpp@me.com>
Date: Thu, 16 Jan 2025 16:14:20 +0800
Subject: [PATCH 13/13] more tests

---
 .../18-hybrid_rpc/09-notify_new_version_spec.lua         | 4 +++-
 .../kong/plugins/rpc-notify-new-version-test/handler.lua | 9 ++++++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
index d432ea552ee..b803420d476 100644
--- a/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
+++ b/spec/02-integration/18-hybrid_rpc/09-notify_new_version_spec.lua
@@ -57,6 +57,8 @@ for _, strategy in helpers.each_strategy() do
 
         assert.logfile(name).has.line(
           "no sync runs, version is " .. rep(".", 32), true, 10)
+        assert.logfile(name).has.line(
+          "no sync runs, version is " .. rep("0", 32), true, 10)
 
         assert.logfile(name).has.line(
           "sync_once retry count exceeded. retry_count: 6", true, 10)
@@ -66,7 +68,7 @@ for _, strategy in helpers.each_strategy() do
         local name = nil
 
         -- cp logs
-        for i = 1, 7 do
+        for i = 0, 6 do
           assert.logfile(name).has.line(
             "kong.sync.v2.get_delta ok: " .. i, true, 10)
         end
diff --git a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
index 154ad0906e6..7d6d7952804 100644
--- a/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
+++ b/spec/fixtures/custom_plugins/kong/plugins/rpc-notify-new-version-test/handler.lua
@@ -30,8 +30,8 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
       },
     }
 
-    counter = counter + 1
     ngx.log(ngx.DEBUG, "kong.sync.v2.get_delta ok: ", counter)
+    counter = counter + 1
 
     return { default = { deltas = deltas, wipe = true, }, }
   end)
@@ -60,6 +60,13 @@ function RpcSyncV2NotifyNewVersioinTestHandler:init_worker()
     assert(res)
     assert(not err)
 
+    -- less or equal version string
+    -- "00000" < "v02_xx"
+    local msg = { default = { new_version = rep("0", 32), }, }
+    local res, err = kong.rpc:call(dp_node_id, method, msg)
+    assert(res)
+    assert(not err)
+
     -- greater version string
     local msg = { default = { new_version = fmt("v02_%028x", 20), }, }
     local res, err = kong.rpc:call(dp_node_id, method, msg)