From 36601d8e441dae856066e27061ae8b6ade4c7790 Mon Sep 17 00:00:00 2001 From: Shiroko Date: Wed, 12 Feb 2025 12:05:31 +0800 Subject: [PATCH 1/9] fix(clustering/sync): fix delta not using absolute TTL --- kong/clustering/services/sync/hooks.lua | 14 ++++++++++ .../services/sync/strategies/postgres.lua | 9 +++++++ kong/db/strategies/postgres/init.lua | 26 +++++++++++++++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/kong/clustering/services/sync/hooks.lua b/kong/clustering/services/sync/hooks.lua index 96612d4927c..5b2c0a541ac 100644 --- a/kong/clustering/services/sync/hooks.lua +++ b/kong/clustering/services/sync/hooks.lua @@ -9,6 +9,7 @@ local EMPTY = require("kong.tools.table").EMPTY local ipairs = ipairs local ngx_log = ngx.log local ngx_ERR = ngx.ERR +local ngx_WARN = ngx.WARN local ngx_DEBUG = ngx.DEBUG @@ -76,6 +77,19 @@ end function _M:entity_delta_writer(entity, name, options, ws_id, is_delete) + if not is_delete and entity.ttl then + -- Replace relative TTL value to absolute TTL value + local exported_entity = self.strategy:export_entity(name, entity, options) + + if exported_entity then + ngx_log(ngx_DEBUG, "[kong.sync.v2] Update TTL from relative value to absolute value ", exported_entity.ttl, ".") + entity.ttl = exported_entity.ttl + + else + ngx_log(ngx_WARN, "[kong.sync.v2] Cannot update TTL of entity (", name, ") to absolute value.") + end + end + local res, err = self.strategy:insert_delta() if not res then self.strategy:cancel_txn() diff --git a/kong/clustering/services/sync/strategies/postgres.lua b/kong/clustering/services/sync/strategies/postgres.lua index 06da15cabdc..2c18a615626 100644 --- a/kong/clustering/services/sync/strategies/postgres.lua +++ b/kong/clustering/services/sync/strategies/postgres.lua @@ -1,6 +1,7 @@ local _M = {} local _MT = { __index = _M } +local kong_table = require("kong.tools.table") local sub = string.sub local fmt = string.format @@ -15,6 +16,7 @@ local VERSION_FMT = VER_PREFIX .. "%028x" function _M.new(db) local self = { + db = db, connector = db.connector, } @@ -64,6 +66,13 @@ function _M:is_valid_version(str) end +function _M:export_entity(name, entity, options) + local options = kong_table.cycle_aware_deep_copy(options, true) + options["export"] = true + return self.db[name]:select(entity, options) +end + + function _M:begin_txn() return self.connector:query("BEGIN;") end diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index c751b4b2072..3ebc8d4142c 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -570,8 +570,14 @@ end function _mt:select(primary_key, options) local statement_name = "select" - if self.schema.ttl and options and options.skip_ttl then - statement_name = "select_skip_ttl" + if options then + -- If we do select for export, there is no need to skip ttl. + if options.export then + statement_name = "select_for_export" + + elseif self.schema.ttl and options.skip_ttl then + statement_name = "select_skip_ttl" + end end local res, err = execute(self, statement_name, self.collapse(primary_key), options) @@ -1248,6 +1254,22 @@ function _M.new(connector, schema, errors) } }) + add_statement_for_export("select", { + operation = "read", + expr = select_expressions, + argn = primary_key_names, + argv = primary_key_args, + code = { + "SELECT ", select_expressions, "\n", + " FROM ", table_name_escaped, "\n", + where_clause( + " WHERE ", "(" .. pk_escaped .. ") = (" .. primary_key_placeholders .. ")", + ttl_select_where, + ws_id_select_where), + " LIMIT 1;" + } + }) + add_statement_for_export("page_first", { operation = "read", argn = { LIMIT }, From 752ffc8cf7c39bf972d67228ba1176e8365f4aab Mon Sep 17 00:00:00 2001 From: Haoxuan Date: Wed, 12 Feb 2025 15:20:52 +0800 Subject: [PATCH 2/9] Fix nil check for entity and TTL --- kong/clustering/services/sync/hooks.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kong/clustering/services/sync/hooks.lua b/kong/clustering/services/sync/hooks.lua index 5b2c0a541ac..4109336939e 100644 --- a/kong/clustering/services/sync/hooks.lua +++ b/kong/clustering/services/sync/hooks.lua @@ -77,11 +77,11 @@ end function _M:entity_delta_writer(entity, name, options, ws_id, is_delete) - if not is_delete and entity.ttl then + if not is_delete and entity and entity.ttl then -- Replace relative TTL value to absolute TTL value local exported_entity = self.strategy:export_entity(name, entity, options) - if exported_entity then + if exported_entity and exported_entity.ttl then ngx_log(ngx_DEBUG, "[kong.sync.v2] Update TTL from relative value to absolute value ", exported_entity.ttl, ".") entity.ttl = exported_entity.ttl From edbaebf3f8cf7ec4c3d4f46f64d65688ecff57ad Mon Sep 17 00:00:00 2001 From: Shiroko Date: Mon, 17 Feb 2025 14:09:35 +0800 Subject: [PATCH 3/9] simplify code --- kong/clustering/services/sync/hooks.lua | 11 ++++++++--- kong/clustering/services/sync/strategies/postgres.lua | 9 --------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/kong/clustering/services/sync/hooks.lua b/kong/clustering/services/sync/hooks.lua index 4109336939e..f23b2fc0b4f 100644 --- a/kong/clustering/services/sync/hooks.lua +++ b/kong/clustering/services/sync/hooks.lua @@ -3,7 +3,8 @@ local _MT = { __index = _M, } local hooks = require("kong.hooks") -local EMPTY = require("kong.tools.table").EMPTY +local kong_table = require("kong.tools.table") +local EMPTY = kong_table.EMPTY local ipairs = ipairs @@ -77,9 +78,13 @@ end function _M:entity_delta_writer(entity, name, options, ws_id, is_delete) - if not is_delete and entity and entity.ttl then + local dao = kong.db[name] + + if not is_delete and dao and entity and entity.ttl then -- Replace relative TTL value to absolute TTL value - local exported_entity = self.strategy:export_entity(name, entity, options) + local export_options = kong_table.cycle_aware_deep_copy(options, true) + export_options["export"] = true + local exported_entity = dao:select(entity, export_options) if exported_entity and exported_entity.ttl then ngx_log(ngx_DEBUG, "[kong.sync.v2] Update TTL from relative value to absolute value ", exported_entity.ttl, ".") diff --git a/kong/clustering/services/sync/strategies/postgres.lua b/kong/clustering/services/sync/strategies/postgres.lua index 2c18a615626..06da15cabdc 100644 --- a/kong/clustering/services/sync/strategies/postgres.lua +++ b/kong/clustering/services/sync/strategies/postgres.lua @@ -1,7 +1,6 @@ local _M = {} local _MT = { __index = _M } -local kong_table = require("kong.tools.table") local sub = string.sub local fmt = string.format @@ -16,7 +15,6 @@ local VERSION_FMT = VER_PREFIX .. "%028x" function _M.new(db) local self = { - db = db, connector = db.connector, } @@ -66,13 +64,6 @@ function _M:is_valid_version(str) end -function _M:export_entity(name, entity, options) - local options = kong_table.cycle_aware_deep_copy(options, true) - options["export"] = true - return self.db[name]:select(entity, options) -end - - function _M:begin_txn() return self.connector:query("BEGIN;") end From ddc08f993e6ba78d5ae41fe53bf09f373f1b107b Mon Sep 17 00:00:00 2001 From: Shiroko Date: Mon, 17 Feb 2025 14:12:27 +0800 Subject: [PATCH 4/9] partially remove flaky green on this PR --- spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua index 66f59e40b13..6a0a6432628 100644 --- a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua +++ b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua @@ -59,6 +59,8 @@ for _, strategy in helpers.each_strategy({"postgres"}) do cluster_rpc = rpc, cluster_rpc_sync = rpc_sync, })) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] full sync ends", true, 20) end) lazy_teardown(function() From aa0b4cba130860edc223137303465e4533325202 Mon Sep 17 00:00:00 2001 From: Shiroko Date: Mon, 17 Feb 2025 15:18:59 +0800 Subject: [PATCH 5/9] style: fix up --- kong/clustering/services/sync/hooks.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kong/clustering/services/sync/hooks.lua b/kong/clustering/services/sync/hooks.lua index f23b2fc0b4f..2debe66eb6d 100644 --- a/kong/clustering/services/sync/hooks.lua +++ b/kong/clustering/services/sync/hooks.lua @@ -83,7 +83,7 @@ function _M:entity_delta_writer(entity, name, options, ws_id, is_delete) if not is_delete and dao and entity and entity.ttl then -- Replace relative TTL value to absolute TTL value local export_options = kong_table.cycle_aware_deep_copy(options, true) - export_options["export"] = true + export_options.export = true local exported_entity = dao:select(entity, export_options) if exported_entity and exported_entity.ttl then From 169c3f8e69ba5488321fb6a086fa511e804163d5 Mon Sep 17 00:00:00 2001 From: Shiroko Date: Tue, 18 Feb 2025 10:06:52 +0800 Subject: [PATCH 6/9] tests(clustering/sync): increase wait time for dp full sync ends --- spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua index 6a0a6432628..34dd366836c 100644 --- a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua +++ b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua @@ -60,7 +60,9 @@ for _, strategy in helpers.each_strategy({"postgres"}) do cluster_rpc_sync = rpc_sync, })) - assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] full sync ends", true, 20) + if rpc_sync == "on" then + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] full sync ends", true, 20) + end end) lazy_teardown(function() From 86a9d16d97d18671da3f60eee298758d758e5493 Mon Sep 17 00:00:00 2001 From: Shiroko Date: Tue, 18 Feb 2025 10:55:50 +0800 Subject: [PATCH 7/9] tests(clustering/sync): decrease wait time for dp full sync ends --- spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua index 34dd366836c..55a68667821 100644 --- a/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua +++ b/spec/03-plugins/09-key-auth/04-hybrid_mode_spec.lua @@ -61,7 +61,7 @@ for _, strategy in helpers.each_strategy({"postgres"}) do })) if rpc_sync == "on" then - assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] full sync ends", true, 20) + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] full sync ends", true, 10) end end) From 6912dcec104c66f55224fa0878f0be0f95b69a49 Mon Sep 17 00:00:00 2001 From: Shiroko Date: Tue, 18 Feb 2025 11:35:33 +0800 Subject: [PATCH 8/9] style: add comment for select statement for export --- kong/db/strategies/postgres/init.lua | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 3ebc8d4142c..8ac65ca8e7b 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -3,6 +3,7 @@ local json = require "pgmoon.json" local cjson_safe = require "cjson.safe" local new_tab = require "table.new" local clear_tab = require "table.clear" +local access = require("kong.plugins.session.access") local kong = kong @@ -1254,6 +1255,7 @@ function _M.new(connector, schema, errors) } }) + -- Add statements for exporting database, avoiding exporting TTL in absolute value. add_statement_for_export("select", { operation = "read", expr = select_expressions, From 54f59e4702b3b8b30e8e8135fa523fac2d695bcd Mon Sep 17 00:00:00 2001 From: Shiroko Date: Tue, 18 Feb 2025 15:53:31 +0800 Subject: [PATCH 9/9] style: remove unused require --- kong/db/strategies/postgres/init.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/kong/db/strategies/postgres/init.lua b/kong/db/strategies/postgres/init.lua index 8ac65ca8e7b..dad681a7da8 100644 --- a/kong/db/strategies/postgres/init.lua +++ b/kong/db/strategies/postgres/init.lua @@ -3,7 +3,6 @@ local json = require "pgmoon.json" local cjson_safe = require "cjson.safe" local new_tab = require "table.new" local clear_tab = require "table.clear" -local access = require("kong.plugins.session.access") local kong = kong