From bc98ef08564f7b6905293be006e4191cb8e6e927 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 07:58:14 +0800 Subject: [PATCH 1/9] tests(clustering): basic tests for incremental sync --- .../19-incrmental_sync/01-sync_spec.lua | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 spec/02-integration/19-incrmental_sync/01-sync_spec.lua diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua new file mode 100644 index 00000000000..641ccf5bcae --- /dev/null +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -0,0 +1,195 @@ +local helpers = require "spec.helpers" +local cjson = require("cjson.safe") + +for _, strategy in helpers.each_strategy() do + +describe("Incremental Sync RPC #" .. strategy, function() + + lazy_setup(function() + helpers.get_db_utils(strategy, { + "clustering_data_planes", + }) -- runs migrations + + assert(helpers.start_kong({ + role = "control_plane", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + database = strategy, + cluster_listen = "127.0.0.1:9005", + nginx_conf = "spec/fixtures/custom_nginx.template", + cluster_incremental_sync = "on", -- incremental sync + })) + + assert(helpers.start_kong({ + role = "data_plane", + database = "off", + prefix = "servroot2", + cluster_cert = "spec/fixtures/kong_clustering.crt", + cluster_cert_key = "spec/fixtures/kong_clustering.key", + cluster_control_plane = "127.0.0.1:9005", + proxy_listen = "0.0.0.0:9002", + nginx_conf = "spec/fixtures/custom_nginx.template", + nginx_worker_processes = 4, -- multiple workers + cluster_incremental_sync = "on", -- incremental sync + })) + end) + + lazy_teardown(function() + helpers.stop_kong("servroot2") + helpers.stop_kong() + end) + + describe("sync works", function() + local route_id + + it("create route on CP", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + local res = assert(admin_client:post("/services", { + body = { name = "service-001", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-001/routes", { + body = { paths = { "/001" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + route_id = json.id + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/001", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + end) + + it("update route on CP", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + local res = assert(admin_client:post("/services", { + body = { name = "service-002", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-002/routes", { + body = { paths = { "/002-foo" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + route_id = json.id + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/002-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + res = assert(admin_client:put("/services/service-002/routes/" .. route_id, { + body = { paths = { "/002-bar" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(200, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/002-bar", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + end) + + it("delete route on CP", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + local res = assert(admin_client:post("/services", { + body = { name = "service-003", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-003/routes", { + body = { paths = { "/003-foo" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + route_id = json.id + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/003-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + res = assert(admin_client:delete("/services/service-003/routes/" .. route_id)) + local body = assert.res_status(204, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/003-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 404 then + return true + end + end, 10) + end) + end) + +end) + +end -- for _, strategy From c5167a3dd6aeb73c7cd3d5337fc4aae73a147e87 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 08:21:04 +0800 Subject: [PATCH 2/9] updaet/delete with name --- .../19-incrmental_sync/01-sync_spec.lua | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 641ccf5bcae..da02f91474f 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -188,6 +188,115 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) end) + + it("update route on CP with name", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + local res = assert(admin_client:post("/services", { + body = { name = "service-004", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-004/routes", { + body = { name = "route-004", paths = { "/004-foo" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/004-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + res = assert(admin_client:put("/services/service-004/routes/route-004", { + body = { paths = { "/004-bar" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(200, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/004-bar", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + end) + + it("delete route on CP with name", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + local res = assert(admin_client:post("/services", { + body = { name = "service-005", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-005/routes", { + body = { name = "route-005", paths = { "/005-foo" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/005-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + res = assert(admin_client:delete("/services/service-005/routes/route-005")) + local body = assert.res_status(204, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/005-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 404 then + return true + end + end, 10) + end) end) end) From cd86e1c00564e8f5a7b8ed2d28cf95e978904294 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 08:59:50 +0800 Subject: [PATCH 3/9] cascade delete on CP --- .../19-incrmental_sync/01-sync_spec.lua | 105 +++++++++++++++++- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index da02f91474f..ac5dcef2f5d 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -117,7 +117,7 @@ describe("Incremental Sync RPC #" .. strategy, function() body = { paths = { "/002-bar" }, }, headers = {["Content-Type"] = "application/json"} })) - local body = assert.res_status(200, res) + assert.res_status(200, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -171,7 +171,7 @@ describe("Incremental Sync RPC #" .. strategy, function() end, 10) res = assert(admin_client:delete("/services/service-003/routes/" .. route_id)) - local body = assert.res_status(204, res) + assert.res_status(204, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -227,7 +227,7 @@ describe("Incremental Sync RPC #" .. strategy, function() body = { paths = { "/004-bar" }, }, headers = {["Content-Type"] = "application/json"} })) - local body = assert.res_status(200, res) + assert.res_status(200, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -280,7 +280,7 @@ describe("Incremental Sync RPC #" .. strategy, function() end, 10) res = assert(admin_client:delete("/services/service-005/routes/route-005")) - local body = assert.res_status(204, res) + assert.res_status(204, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -297,6 +297,103 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) end) + + it("cascade delete on CP", function() + local admin_client = helpers.admin_client(10000) + finally(function() + admin_client:close() + end) + + -- create service and route + + local res = assert(admin_client:post("/services", { + body = { name = "service-006", url = "https://127.0.0.1:15556/request", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/services/service-006/routes", { + body = { paths = { "/006-foo" }, }, + headers = {["Content-Type"] = "application/json"} + })) + local body = assert.res_status(201, res) + local json = cjson.decode(body) + + route_id = json.id + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/006-foo", + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + -- create consumer and key-auth + + res = assert(admin_client:post("/consumers", { + body = { username = "foo", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + res = assert(admin_client:post("/consumers/foo/key-auth", { + body = { key = "my-key", }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + res = assert(admin_client:post("/plugins", { + body = { name = "key-auth", + config = { key_names = {"apikey"} }, + route = { id = route_id }, + }, + headers = {["Content-Type"] = "application/json"} + })) + assert.res_status(201, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/006-foo", + headers = {["apikey"] = "my-key"}, + }) + + local status = res and res.status + proxy_client:close() + if status == 200 then + return true + end + end, 10) + + -- delete consumer and key-auth + + res = assert(admin_client:delete("/consumers/foo")) + assert.res_status(204, res) + + helpers.wait_until(function() + local proxy_client = helpers.http_client("127.0.0.1", 9002) + + res = proxy_client:send({ + method = "GET", + path = "/006-foo", + headers = {["apikey"] = "my-key"}, + }) + + local status = res and res.status + proxy_client:close() + if status == 401 then + return true + end + end, 10) + end) end) end) From c6c22a469bbcc1cd76f19bb9f2c71981b68c698c Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 09:26:33 +0800 Subject: [PATCH 4/9] check log file --- .../19-incrmental_sync/01-sync_spec.lua | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index ac5dcef2f5d..3b1af4f02e2 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -76,6 +76,11 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) end) it("update route on CP", function() @@ -133,6 +138,11 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) end) it("delete route on CP", function() @@ -170,6 +180,11 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) + res = assert(admin_client:delete("/services/service-003/routes/" .. route_id)) assert.res_status(204, res) @@ -187,6 +202,8 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] delete entity", true) end) it("update route on CP with name", function() @@ -243,6 +260,11 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) end) it("delete route on CP with name", function() @@ -279,6 +301,11 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) + res = assert(admin_client:delete("/services/service-005/routes/route-005")) assert.res_status(204, res) @@ -296,6 +323,8 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] delete entity", true) end) it("cascade delete on CP", function() @@ -335,6 +364,11 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) + assert.logfile().has.line("[kong.sync.v2] config push (connected client)", true) + assert.logfile().has.no.line("unable to update clustering data plane status", true) + + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) + -- create consumer and key-auth res = assert(admin_client:post("/consumers", { @@ -393,6 +427,9 @@ describe("Incremental Sync RPC #" .. strategy, function() return true end end, 10) + + assert.logfile().has.line("[kong.sync.v2] new delta due to cascade deleting", true) + assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] delete entity", true) end) end) From 106d4f047f9536eba924e457228e303c3adf58a5 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 09:32:35 +0800 Subject: [PATCH 5/9] after_each --- spec/02-integration/19-incrmental_sync/01-sync_spec.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 3b1af4f02e2..2258232d72a 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -39,6 +39,11 @@ describe("Incremental Sync RPC #" .. strategy, function() helpers.stop_kong() end) + after_each(function() + helpers.clean_logfile("servroot2/logs/error.log") + helpers.clean_logfile() + end) + describe("sync works", function() local route_id From 54e213e2f8e6e2e389d6664ab3379ea4b8b4650f Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 09:36:37 +0800 Subject: [PATCH 6/9] more log check --- spec/02-integration/19-incrmental_sync/01-sync_spec.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 2258232d72a..5cd35d178a2 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -189,6 +189,7 @@ describe("Incremental Sync RPC #" .. strategy, function() assert.logfile().has.no.line("unable to update clustering data plane status", true) assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) + assert.logfile("servroot2/logs/error.log").has.no.line("[kong.sync.v2] delete entity", true) res = assert(admin_client:delete("/services/service-003/routes/" .. route_id)) assert.res_status(204, res) @@ -310,6 +311,7 @@ describe("Incremental Sync RPC #" .. strategy, function() assert.logfile().has.no.line("unable to update clustering data plane status", true) assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] update entity", true) + assert.logfile("servroot2/logs/error.log").has.no.line("[kong.sync.v2] delete entity", true) res = assert(admin_client:delete("/services/service-005/routes/route-005")) assert.res_status(204, res) @@ -412,6 +414,9 @@ describe("Incremental Sync RPC #" .. strategy, function() end end, 10) + assert.logfile().has.no.line("[kong.sync.v2] new delta due to cascade deleting", true) + assert.logfile("servroot2/logs/error.log").has.no.line("[kong.sync.v2] delete entity", true) + -- delete consumer and key-auth res = assert(admin_client:delete("/consumers/foo")) From 00601bf84559c8185d86fd05398baa3ca8971ebc Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 09:42:28 +0800 Subject: [PATCH 7/9] lint fix --- spec/02-integration/19-incrmental_sync/01-sync_spec.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 5cd35d178a2..5d0bf439bd6 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -229,7 +229,6 @@ describe("Incremental Sync RPC #" .. strategy, function() headers = {["Content-Type"] = "application/json"} })) local body = assert.res_status(201, res) - local json = cjson.decode(body) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -290,7 +289,6 @@ describe("Incremental Sync RPC #" .. strategy, function() headers = {["Content-Type"] = "application/json"} })) local body = assert.res_status(201, res) - local json = cjson.decode(body) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) From 0a4c228b8c9c6243948b069d2d0c7da2584f0ce2 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 12:04:13 +0800 Subject: [PATCH 8/9] lint fix --- spec/02-integration/19-incrmental_sync/01-sync_spec.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 5d0bf439bd6..9e5b3741e9a 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -228,7 +228,7 @@ describe("Incremental Sync RPC #" .. strategy, function() body = { name = "route-004", paths = { "/004-foo" }, }, headers = {["Content-Type"] = "application/json"} })) - local body = assert.res_status(201, res) + assert.res_status(201, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) @@ -288,7 +288,7 @@ describe("Incremental Sync RPC #" .. strategy, function() body = { name = "route-005", paths = { "/005-foo" }, }, headers = {["Content-Type"] = "application/json"} })) - local body = assert.res_status(201, res) + assert.res_status(201, res) helpers.wait_until(function() local proxy_client = helpers.http_client("127.0.0.1", 9002) From 6cf34c8e2e7b0a1ade9f8142cb4da18a65c58bf5 Mon Sep 17 00:00:00 2001 From: chronolaw Date: Sat, 9 Nov 2024 12:57:55 +0800 Subject: [PATCH 9/9] check cascade deletion version --- .../19-incrmental_sync/01-sync_spec.lua | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua index 9e5b3741e9a..baaa579f23a 100644 --- a/spec/02-integration/19-incrmental_sync/01-sync_spec.lua +++ b/spec/02-integration/19-incrmental_sync/01-sync_spec.lua @@ -438,6 +438,30 @@ describe("Incremental Sync RPC #" .. strategy, function() assert.logfile().has.line("[kong.sync.v2] new delta due to cascade deleting", true) assert.logfile("servroot2/logs/error.log").has.line("[kong.sync.v2] delete entity", true) + + -- cascade deletion should be the same version + + local ver + local count = 0 + local patt = "delete entity, version: %d+" + local f = io.open("servroot2/logs/error.log", "r") + while true do + local line = f:read("*l") + + if not line then + f:close() + break + end + + local found = line:match(patt) + if found then + ver = ver or found + assert.equal(ver, found) + count = count + 1 + end + end + assert(count > 1) + end) end)