From 8b5f0ffaa29223b6bf08e5f9d965c10ec2098f36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=82=9F=E7=A9=BA?= Date: Tue, 1 Apr 2025 22:18:30 +0800 Subject: [PATCH 1/3] feat: implement long-term stability benchmark test --- Makefile | 9 + test/e2e/gatewayapi/httproute.go | 14 +- test/e2e/scaffold/httpbin.go | 8 +- test/e2e/scaffold/locust.go | 174 ++++++++++++++++++ test/e2e/scaffold/scaffold.go | 14 +- test/long_term_stability/lts_test.go | 40 ++++ .../spec_subjects/spec_subjects.go | 170 +++++++++++++++++ 7 files changed, 418 insertions(+), 11 deletions(-) create mode 100644 test/e2e/scaffold/locust.go create mode 100644 test/long_term_stability/lts_test.go create mode 100644 test/long_term_stability/spec_subjects/spec_subjects.go diff --git a/Makefile b/Makefile index 2d1280edf..4f9419549 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,15 @@ e2e-test: @kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test ./test/e2e/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)" +.PHONY: kind-lts-test +kind-lts-test: kind-up build-image kind-load-images lts-test + +# lts-test is long-term-stability test +.PHONY: lts-test +lts-test: + @kind get kubeconfig --name $(KIND_NAME) > $$KUBECONFIG + DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test ./test/long_term_stability/ -test.timeout=$(TEST_TIMEOUT) -v -ginkgo.v -ginkgo.focus="$(TEST_FOCUS)" + .PHONY: conformance-test conformance-test: DASHBOARD_VERSION=$(DASHBOARD_VERSION) go test -v ./test/conformance -tags=conformance diff --git a/test/e2e/gatewayapi/httproute.go b/test/e2e/gatewayapi/httproute.go index db0204463..9b34c9714 100644 --- a/test/e2e/gatewayapi/httproute.go +++ b/test/e2e/gatewayapi/httproute.go @@ -16,7 +16,7 @@ import ( var _ = Describe("Test HTTPRoute", func() { s := scaffold.NewDefaultScaffold() - var defautlGatewayClass = ` + var defaultGatewayClass = ` apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: @@ -25,7 +25,7 @@ spec: controllerName: %s ` - var defautlGateway = ` + var defaultGateway = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -37,7 +37,7 @@ spec: protocol: HTTP port: 80 ` - var defautlGatewayHTTPS = ` + var defaultGatewayHTTPS = ` apiVersion: gateway.networking.k8s.io/v1 kind: Gateway metadata: @@ -78,7 +78,7 @@ spec: var beforeEachHTTP = func() { By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -89,7 +89,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGateway, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGateway, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) @@ -105,7 +105,7 @@ spec: createSecret(s, secretName) By("create GatewayClass") gatewayClassName := fmt.Sprintf("api7-%d", time.Now().Unix()) - err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defautlGatewayClass, gatewayClassName, s.GetControllerName()), "") + err := s.CreateResourceFromStringWithNamespace(fmt.Sprintf(defaultGatewayClass, gatewayClassName, s.GetControllerName()), "") Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") time.Sleep(5 * time.Second) @@ -116,7 +116,7 @@ spec: Expect(gcyaml).To(ContainSubstring("message: the gatewayclass has been accepted by the api7-ingress-controller"), "checking GatewayClass condition message") By("create Gateway") - err = s.CreateResourceFromString(fmt.Sprintf(defautlGatewayHTTPS, gatewayClassName)) + err = s.CreateResourceFromString(fmt.Sprintf(defaultGatewayHTTPS, gatewayClassName)) Expect(err).NotTo(HaveOccurred(), "creating Gateway") time.Sleep(5 * time.Second) diff --git a/test/e2e/scaffold/httpbin.go b/test/e2e/scaffold/httpbin.go index 6e67b4558..67118dcb8 100644 --- a/test/e2e/scaffold/httpbin.go +++ b/test/e2e/scaffold/httpbin.go @@ -19,7 +19,7 @@ import ( "time" "github.com/gruntwork-io/terratest/modules/k8s" - ginkgo "github.com/onsi/ginkgo/v2" + "github.com/onsi/ginkgo/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -70,6 +70,10 @@ spec: - containerPort: 80 name: "http" protocol: "TCP" + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "sleep 5"] ` _httpService = ` apiVersion: v1 @@ -112,7 +116,7 @@ func (s *Scaffold) NewHTTPBINWithNamespace(namespace string) (*corev1.Service, e return s.newHTTPBIN() } -// ScaleHTTPBIN scales the number of HTTPBIN pods to desired. +// ScaleHTTPBIN scales the number of HTTPBIN pods to desire. func (s *Scaffold) ScaleHTTPBIN(desired int) error { httpbinDeployment := fmt.Sprintf(s.FormatRegistry(_httpbinDeploymentTemplate), desired) if err := s.CreateResourceFromString(httpbinDeployment); err != nil { diff --git a/test/e2e/scaffold/locust.go b/test/e2e/scaffold/locust.go new file mode 100644 index 000000000..60e992b29 --- /dev/null +++ b/test/e2e/scaffold/locust.go @@ -0,0 +1,174 @@ +package scaffold + +import ( + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/gruntwork-io/terratest/modules/k8s" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" +) + +const ( + _locustConfigMapTemplate = ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: locust-config +data: + locustfile.py: |- + from locust import HttpUser, task, between + + class HttpbinRequester(HttpUser): + @task + def request_headers(self): + self.client.get("/headers", headers={"Host": "httpbin.example"}) + LOCUST_HOST: http://api7ee3-apisix-gateway-mtls:9080 + LOCUST_SPAWN_RATE: "50" + LOCUST_USERS: "500" + LOCUST_AUTOSTART: "true" +` + _locustDeploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: locust +spec: + selector: + matchLabels: + app: locust + template: + metadata: + labels: + app: locust + spec: + containers: + - name: locust + image: locustio/locust + ports: + - containerPort: 8089 + env: + - name: LOCUST_HOST + valueFrom: + configMapKeyRef: + name: locust-config + key: LOCUST_HOST + - name: LOCUST_SPAWN_RATE + valueFrom: + configMapKeyRef: + name: locust-config + key: LOCUST_SPAWN_RATE + - name: LOCUST_USERS + valueFrom: + configMapKeyRef: + name: locust-config + key: LOCUST_USERS + - name: LOCUST_AUTOSTART + valueFrom: + configMapKeyRef: + name: locust-config + key: LOCUST_AUTOSTART + volumeMounts: + - mountPath: /home/locust + name: locust-config + volumes: + - name: locust-config + configMap: + name: locust-config +` + _locustServiceTemplate = ` +apiVersion: v1 +kind: Service +metadata: + name: locust +spec: + selector: + app: locust + ports: + - name: web + port: 8089 + targetPort: 8089 + protocol: TCP + type: ClusterIP ` +) + +func (s *Scaffold) DeployLocust() *corev1.Service { + // create ConfigMap, Deployment, Service + for _, yaml_ := range []string{_locustConfigMapTemplate, _locustDeploymentTemplate, _locustServiceTemplate} { + err := s.CreateResourceFromString(yaml_) + Expect(err).NotTo(HaveOccurred(), "create resource: %s", yaml_) + } + + service, err := k8s.GetServiceE(s.t, s.kubectlOptions, "locust") + Expect(err).NotTo(HaveOccurred(), "get service: locust") + + s.EnsureNumEndpointsReady(s.t, service.Name, 1) + s.locustTunnel = k8s.NewTunnel(s.kubectlOptions, k8s.ResourceTypeService, "locust", 8089, 8089) + s.addFinalizers(s.locustTunnel.Close) + + err = s.locustTunnel.ForwardPortE(s.t) + Expect(err).NotTo(HaveOccurred(), "port-forward service: locust") + + return service +} + +// func (s *Scaffold) LocustClient() *httpexpect.Expect { +// u := url.URL{ +// Scheme: "http", +// Host: s.locustTunnel.Endpoint(), +// } +// return httpexpect.WithConfig(httpexpect.Config{ +// BaseURL: u.String(), +// Client: &http.Client{ +// Transport: &http.Transport{}, +// CheckRedirect: func(req *http.Request, via []*http.Request) error { +// return http.ErrUseLastResponse +// }, +// }, +// Reporter: httpexpect.NewAssertReporter( +// httpexpect.NewAssertReporter(s.GinkgoT), +// ), +// }) +// } + +func (s *Scaffold) ResetLocust() error { + if s.locustTunnel == nil { + return errors.New("locust is not deployed") + } + resp, err := http.Get("http://" + s.locustTunnel.Endpoint() + "/stats/reset") + if err != nil { + return errors.Wrap(err, "failed to request reset locust") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("request reset locust not OK, status: %s", resp.Status) + } + return nil +} + +func (s *Scaffold) DownloadLocustReport(filename string) error { + if s.locustTunnel == nil { + return errors.New("locust is not deployed") + } + if !strings.EqualFold(filepath.Ext(filename), ".html") { + filename += ".html" + } + _ = os.Remove(filename) + resp, err := http.Get("http://" + s.locustTunnel.Endpoint() + "/stats/report?download=1&theme=light") + if err != nil { + return errors.Wrap(err, "failed to request download report") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("request download report not OK, status: %s", resp.Status) + } + data, err := io.ReadAll(resp.Body) + if err != nil { + return errors.Wrap(err, "failed to read report") + } + return os.WriteFile(filename, data, 0644) +} diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 5699ce3dc..1cdb9c3da 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -71,6 +71,9 @@ type Options struct { NamespaceSelectorLabel map[string][]string DisableNamespaceSelector bool DisableNamespaceLabel bool + + GinkgoBeforeCallback func(...any) bool + GinkgoAfterCallback func(...any) bool } type Scaffold struct { @@ -96,6 +99,7 @@ type Scaffold struct { apisixTCPTunnel *k8s.Tunnel apisixTLSOverTCPTunnel *k8s.Tunnel apisixUDPTunnel *k8s.Tunnel + locustTunnel *k8s.Tunnel // apisixControlTunnel *k8s.Tunnel } @@ -158,8 +162,14 @@ func NewScaffold(o *Options) *Scaffold { t: GinkgoT(), } - BeforeEach(s.beforeEach) - AfterEach(s.afterEach) + if o.GinkgoBeforeCallback == nil { + o.GinkgoBeforeCallback = BeforeEach + } + if o.GinkgoAfterCallback == nil { + o.GinkgoAfterCallback = AfterEach + } + o.GinkgoBeforeCallback(s.beforeEach) + o.GinkgoAfterCallback(s.afterEach) return s } diff --git a/test/long_term_stability/lts_test.go b/test/long_term_stability/lts_test.go new file mode 100644 index 000000000..a646d2d16 --- /dev/null +++ b/test/long_term_stability/lts_test.go @@ -0,0 +1,40 @@ +/* +Copyright 2024. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package long_term_stability + +import ( + "fmt" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/api7/api7-ingress-controller/test/e2e/framework" + _ "github.com/api7/api7-ingress-controller/test/long_term_stability/spec_subjects" +) + +// Run long-term-stability tests using Ginkgo runner. +func TestLongTermStability(t *testing.T) { + RegisterFailHandler(Fail) + var f = framework.NewFramework() + + BeforeSuite(f.BeforeSuite) + AfterSuite(f.AfterSuite) + + _, _ = fmt.Fprintf(GinkgoWriter, "Starting long-term-stability suite\n") + RunSpecs(t, "long-term-stability suite") +} diff --git a/test/long_term_stability/spec_subjects/spec_subjects.go b/test/long_term_stability/spec_subjects/spec_subjects.go new file mode 100644 index 000000000..4c989fdfe --- /dev/null +++ b/test/long_term_stability/spec_subjects/spec_subjects.go @@ -0,0 +1,170 @@ +package spec_subjects + +import ( + "fmt" + "net/http" + "time" + + "github.com/api7/api7-ingress-controller/test/e2e/scaffold" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("API7 Ingress Controller Long Term Stability Tests", Ordered, func() { + var ( + s = scaffold.NewScaffold(&scaffold.Options{ + ControllerName: "gateway.api7.io/api7-ingress-controller", + GinkgoBeforeCallback: BeforeAll, + GinkgoAfterCallback: AfterAll, + }) + it int + ) + var ( + gatewayClassTemplate = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: api7 +spec: + controllerName: gateway.api7.io/api7-ingress-controller +` + gatewayTemplate = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: api7ee +spec: + gatewayClassName: api7 + listeners: + - name: http1 + protocol: HTTP + port: 80 +` + httpRouteTemplate = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httpbin +spec: + parentRefs: + - name: api7ee + hostnames: + - httpbin.example + rules: + - matches: + - path: + type: Exact + value: /headers + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + ) + + BeforeAll(func() { + By("apply GatewayClass") + err := s.CreateResourceFromString(gatewayClassTemplate) + Expect(err).NotTo(HaveOccurred(), "creating GatewayClass") + Eventually(func() string { + yaml_, err := s.GetResourceYaml("GatewayClass", "api7") + Expect(err).NotTo(HaveOccurred()) + return yaml_ + }).WithTimeout(8*time.Second).ProbeEvery(time.Second). + Should(ContainSubstring(`status: "True"`), "checking GatewayClass condition status") + + By("apply Gateway") + err = s.CreateResourceFromString(gatewayTemplate) + Expect(err).NotTo(HaveOccurred(), "creating Gateway") + Eventually(func() string { + yaml_, err := s.GetResourceYaml("Gateway", "api7ee") + Expect(err).NotTo(HaveOccurred()) + return yaml_ + }).WithTimeout(8*time.Second).ProbeEvery(time.Second). + Should(ContainSubstring(`status: "True"`), "checking Gateway condition status") + + By("deploy locust") + _ = s.DeployLocust() + }) + + BeforeEach(func() { + By("Create HTTPRoute") + err := s.CreateResourceFromString(httpRouteTemplate) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + Eventually(func() string { + yaml_, err := s.GetResourceYaml("HTTPRoute", "httpbin") + Expect(err).NotTo(HaveOccurred(), "getting yaml: %s", yaml_) + return yaml_ + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second). + Should(ContainSubstring(`status: "True"`)) + Eventually(func() int { + return s.NewAPISIXClient().GET("/headers").WithHost("httpbin.example").Expect().Raw().StatusCode + }).WithTimeout(8 * time.Second).ProbeEvery(time.Second). + Should(Equal(http.StatusOK)) + }) + + BeforeEach(func() { + it++ + }) + + JustBeforeEach(func() { + By("reset locust statistics") + err := s.ResetLocust() + Expect(err).NotTo(HaveOccurred(), "reset locust") + }) + + Context("Benchmark", func() { + It("benchmark", func() { + By("sleep and waiting for locust test") + time.Sleep(10 * time.Minute) + + err := s.DownloadLocustReport(fmt.Sprintf("%02d_benchmark", it)) + Expect(err).NotTo(HaveOccurred(), "getting locust report") + }) + }) + + Context("Service Discovery", func() { + BeforeEach(func() { + // scale backend pods replicas + err := s.ScaleHTTPBIN(5) + Expect(err).NotTo(HaveOccurred(), "scaling httpbin") + }) + + It("service discovery", func() { + var total = 5 + for i := 0; i < total; i++ { + By(fmt.Sprintf("rolling update deployment/httpbin-deployment-e2e-test [%02d/%02d]", i+1, total)) + now := time.Now().Format("2006_01_02_15_04_05") + _, err := s.RunKubectlAndGetOutput("set", "env", "deployment/httpbin-deployment-e2e-test", "ENV_NOW="+now) + Expect(err).NotTo(HaveOccurred(), "kubectl set env deployment/httpbin-deployment-e2e-test MOCK_ENV=%s", now) + time.Sleep(time.Minute) + } + + err := s.DownloadLocustReport(fmt.Sprintf("%02d_service_discovery", it)) + Expect(err).NotTo(HaveOccurred(), "getting locust report") + }) + }) + + Context("Large-scale HTTPRoute", func() { + It("it 0", func() { + Ω(true).Should(BeTrue()) + }) + }) + + Context("Ingress Controller is crashing", func() { + It("it 0", func() { + Ω(true).Should(BeTrue()) + }) + }) + + Context("Regression Tests", func() { + It("Under large-scale HTTPRoute, some HTTPRoute resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {}) + + It("Under large-scale CRDs, some CRDs resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {}) + + It("When a large number of CRDs are applied concurrently, the processing capacity of IngressController is linear (≤O(n), n is the number of resources applied simultaneously)", func() {}) + + It("Under large-scale CRDs, some CRDs are added/deleted/modified concurrently, and the processing capacity of IngressController has no obvious relationship with the number of existing CRDs", func() {}) + + It("When IngressController is unexpectedly unavailable, it does not affect the existing configuration of the data plane", func() {}) + }) +}) From 10ce4bce8ada7e80caee6b8e90aff3ab1a204830 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=82=9F=E7=A9=BA?= Date: Wed, 2 Apr 2025 08:59:15 +0800 Subject: [PATCH 2/3] large scale HTTPRoute --- test/e2e/scaffold/k8s.go | 8 + test/e2e/scaffold/locust.go | 42 +++-- test/e2e/scaffold/scaffold.go | 6 + .../spec_subjects/spec_subjects.go | 151 ++++++++++++++++-- 4 files changed, 172 insertions(+), 35 deletions(-) diff --git a/test/e2e/scaffold/k8s.go b/test/e2e/scaffold/k8s.go index 5a9aa9efb..7684bbe32 100644 --- a/test/e2e/scaffold/k8s.go +++ b/test/e2e/scaffold/k8s.go @@ -110,6 +110,14 @@ func (s *Scaffold) ListPodsByLabels(labels string) ([]corev1.Pod, error) { }) } +func (s *Scaffold) DeleteResourcesByLabels(resourceType, label string) error { + return k8s.RunKubectlE(s.t, s.kubectlOptions, "delete", resourceType, "-l", label) +} + +func (s *Scaffold) GetResourcesByLabelsOutput(resourceType, label string) (string, error) { + return k8s.RunKubectlAndGetOutputE(s.t, s.kubectlOptions, "get", resourceType, "-l", label) +} + // CreateResourceFromStringWithNamespace creates resource from a loaded yaml string // and sets its namespace to the specified one. func (s *Scaffold) CreateResourceFromStringWithNamespace(yaml, namespace string) error { diff --git a/test/e2e/scaffold/locust.go b/test/e2e/scaffold/locust.go index 60e992b29..4b94248d6 100644 --- a/test/e2e/scaffold/locust.go +++ b/test/e2e/scaffold/locust.go @@ -25,8 +25,21 @@ data: class HttpbinRequester(HttpUser): @task - def request_headers(self): + def headers(self): self.client.get("/headers", headers={"Host": "httpbin.example"}) + + @task + def get(self): + self.client.get("/get", headers={"Host": "httpbin.example"}) + + @task + def post(self): + self.client.post("/post", headers={"Host": "httpbin.example"}) + + @task + def image(self): + self.client.image("/image", headers={"Host": "httpbin.example"}) + LOCUST_HOST: http://api7ee3-apisix-gateway-mtls:9080 LOCUST_SPAWN_RATE: "50" LOCUST_USERS: "500" @@ -116,25 +129,6 @@ func (s *Scaffold) DeployLocust() *corev1.Service { return service } -// func (s *Scaffold) LocustClient() *httpexpect.Expect { -// u := url.URL{ -// Scheme: "http", -// Host: s.locustTunnel.Endpoint(), -// } -// return httpexpect.WithConfig(httpexpect.Config{ -// BaseURL: u.String(), -// Client: &http.Client{ -// Transport: &http.Transport{}, -// CheckRedirect: func(req *http.Request, via []*http.Request) error { -// return http.ErrUseLastResponse -// }, -// }, -// Reporter: httpexpect.NewAssertReporter( -// httpexpect.NewAssertReporter(s.GinkgoT), -// ), -// }) -// } - func (s *Scaffold) ResetLocust() error { if s.locustTunnel == nil { return errors.New("locust is not deployed") @@ -143,7 +137,9 @@ func (s *Scaffold) ResetLocust() error { if err != nil { return errors.Wrap(err, "failed to request reset locust") } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return errors.Errorf("request reset locust not OK, status: %s", resp.Status) } @@ -162,7 +158,9 @@ func (s *Scaffold) DownloadLocustReport(filename string) error { if err != nil { return errors.Wrap(err, "failed to request download report") } - defer resp.Body.Close() + defer func() { + _ = resp.Body.Close() + }() if resp.StatusCode != http.StatusOK { return errors.Errorf("request download report not OK, status: %s", resp.Status) } diff --git a/test/e2e/scaffold/scaffold.go b/test/e2e/scaffold/scaffold.go index 1cdb9c3da..986fbc086 100644 --- a/test/e2e/scaffold/scaffold.go +++ b/test/e2e/scaffold/scaffold.go @@ -31,6 +31,7 @@ import ( "github.com/gavv/httpexpect/v2" "github.com/gruntwork-io/terratest/modules/k8s" + "github.com/gruntwork-io/terratest/modules/logger" "github.com/gruntwork-io/terratest/modules/testing" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -74,6 +75,8 @@ type Options struct { GinkgoBeforeCallback func(...any) bool GinkgoAfterCallback func(...any) bool + + KubectlLogger *logger.Logger } type Scaffold struct { @@ -365,6 +368,9 @@ func (s *Scaffold) beforeEach() { ConfigPath: s.opts.Kubeconfig, Namespace: s.namespace, } + if s.opts.KubectlLogger != nil { + s.kubectlOptions.Logger = s.opts.KubectlLogger + } if s.opts.ControllerName == "" { s.opts.ControllerName = fmt.Sprintf("%s/%d", DefaultControllerName, time.Now().Nanosecond()) } diff --git a/test/long_term_stability/spec_subjects/spec_subjects.go b/test/long_term_stability/spec_subjects/spec_subjects.go index 4c989fdfe..f107403f2 100644 --- a/test/long_term_stability/spec_subjects/spec_subjects.go +++ b/test/long_term_stability/spec_subjects/spec_subjects.go @@ -3,11 +3,16 @@ package spec_subjects import ( "fmt" "net/http" + "strconv" + "strings" "time" "github.com/api7/api7-ingress-controller/test/e2e/scaffold" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/gruntwork-io/terratest/modules/retry" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pkg/errors" ) var _ = Describe("API7 Ingress Controller Long Term Stability Tests", Ordered, func() { @@ -16,6 +21,7 @@ var _ = Describe("API7 Ingress Controller Long Term Stability Tests", Ordered, f ControllerName: "gateway.api7.io/api7-ingress-controller", GinkgoBeforeCallback: BeforeAll, GinkgoAfterCallback: AfterAll, + KubectlLogger: logger.Discard, // too many logs in long-term stability test so discard kubectl apply logs }) it int ) @@ -55,6 +61,37 @@ spec: - path: type: Exact value: /headers + - path: + type: Exact + value: /get + - path: + type: Exact + value: /post + - path: + type: Exact + value: /image + + backendRefs: + - name: httpbin-service-e2e-test + port: 80 +` + httpRouteTemplate2 = ` +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: %s + labels: + template_name: httpRouteTemplate2 +spec: + parentRefs: + - name: api7ee + hostnames: + - httpbin.example + rules: + - matches: + - path: + type: Exact + value: %s backendRefs: - name: httpbin-service-e2e-test port: 80 @@ -124,12 +161,12 @@ spec: Context("Service Discovery", func() { BeforeEach(func() { - // scale backend pods replicas + By("scale backend pods replicas") err := s.ScaleHTTPBIN(5) Expect(err).NotTo(HaveOccurred(), "scaling httpbin") }) - It("service discovery", func() { + It("rolling update", func() { var total = 5 for i := 0; i < total; i++ { By(fmt.Sprintf("rolling update deployment/httpbin-deployment-e2e-test [%02d/%02d]", i+1, total)) @@ -139,32 +176,120 @@ spec: time.Sleep(time.Minute) } - err := s.DownloadLocustReport(fmt.Sprintf("%02d_service_discovery", it)) + err := s.DownloadLocustReport(fmt.Sprintf("%02d_rolling_update", it)) + Expect(err).NotTo(HaveOccurred(), "getting locust report") + }) + + It("scale replicas", func() { + var total = 5 + for i := 0; i < total; i++ { + By(fmt.Sprintf("scale replicas for deployment/httpbin-deployment-e2e-test [%02d/%02d]", i+1, total)) + err := s.ScaleHTTPBIN(4 + i%2*3) // scale to 4 if "i" is even, scale to 7 if "i" is odd. + Expect(err).NotTo(HaveOccurred(), "scale replicas for deployment/httpbin-deployment-e2e-test") + time.Sleep(time.Minute) + } + + err := s.DownloadLocustReport(fmt.Sprintf("%02d_scale_replicas", it)) Expect(err).NotTo(HaveOccurred(), "getting locust report") }) }) Context("Large-scale HTTPRoute", func() { - It("it 0", func() { - Ω(true).Should(BeTrue()) - }) + var ( + reconcileDurationPerHTTPRoute = 3 * time.Second + probeDuration = 100 * time.Millisecond + + resourceTypeHTTPRoute = "HTTPRoute" + label = "template_name=httpRouteTemplate2" + ) + + for _, total := range []int{500, 2000, 5000} { + var ( + reconcileDurationBatchProcess = time.Duration(total) * reconcileDurationPerHTTPRoute + title = strconv.FormatInt(int64(total), 10) + " HTTPRoute" + ) + + It(title, func() { + defer func() { + By("cleaning up HTTPRoutes") + err := s.DeleteResourcesByLabels(resourceTypeHTTPRoute, label) + Expect(err).NotTo(HaveOccurred(), "delete HTTPRoute by label") + + Eventually(func() string { + output, err := s.GetResourcesByLabelsOutput(resourceTypeHTTPRoute, label) + Expect(err).NotTo(HaveOccurred(), "getting HTTPRoute") + return output + }).WithTimeout(reconcileDurationBatchProcess).ProbeEvery(probeDuration). + Should(ContainSubstring("No resources found")) + }() + + By("prepare HTTPRoutes") + for i := 0; i < total+100; i++ { + By(fmt.Sprintf("prepare HTTPRoutes [%04d/%04d]", i+1, total)) + routeName := "httpbin-" + strconv.FormatInt(int64(i), 10) + pathValue := "/delay/" + strconv.FormatInt(int64(i), 10) + err := s.CreateResourceFromString(fmt.Sprintf(httpRouteTemplate2, routeName, pathValue)) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + + message := retry.DoWithRetry(s.GinkgoT, "Wait for HTTPRoute ok", 100, time.Second, func() (string, error) { + yaml_, err := s.GetResourceYaml(resourceTypeHTTPRoute, routeName) + if err != nil { + return "", err + } + if !strings.Contains(yaml_, `status: "True"`) { + return "", errors.New("HTTPRoute status is not True") + } + return "HTTPRoute is now available", nil + }, + ) + s.Logf(message) + } + + By("delete 100 HTTPRoutes") + for i := total; i < total+100; i++ { + By(fmt.Sprintf("prepare 1000 HTTPRoute [%04d/%04d]", i, total+100)) + routeName := "httpbin-" + strconv.FormatInt(int64(i), 10) + err := s.DeleteResource(resourceTypeHTTPRoute, routeName) + Expect(err).NotTo(HaveOccurred(), "creating HTTPRoute") + + Eventually(func() string { + _, err := s.GetResourceYaml(resourceTypeHTTPRoute, "") + return err.Error() + }).WithTimeout(reconcileDurationPerHTTPRoute).ProbeEvery(probeDuration). + Should(ContainSubstring("not found")) + } + + err := s.DownloadLocustReport(fmt.Sprintf("%02d_large_scale_httproute(1000)", it)) + Expect(err).NotTo(HaveOccurred(), "getting locust report") + }) + } }) - Context("Ingress Controller is crashing", func() { + PContext("Ingress Controller is crashing", func() { It("it 0", func() { Ω(true).Should(BeTrue()) }) }) - Context("Regression Tests", func() { - It("Under large-scale HTTPRoute, some HTTPRoute resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {}) + PContext("Regression Tests", func() { + // Under large-scale HTTPRoute, some HTTPRoute resources are abnormal, which cannot affect the synchronization + // efficient of other resources + It("", func() {}) - It("Under large-scale CRDs, some CRDs resources are abnormal, which cannot affect the synchronization efficient of other resources", func() {}) + // Under large-scale CRDs, some CRDs resources are abnormal, which cannot affect the synchronization efficient + // of other resources + It("", func() {}) - It("When a large number of CRDs are applied concurrently, the processing capacity of IngressController is linear (≤O(n), n is the number of resources applied simultaneously)", func() {}) + // When a large number of CRDs are applied concurrently, the processing capacity of IngressController is linear + // (≤O(n), n is the number of resources applied simultaneously) + It("", func() {}) - It("Under large-scale CRDs, some CRDs are added/deleted/modified concurrently, and the processing capacity of IngressController has no obvious relationship with the number of existing CRDs", func() {}) + // Under large-scale CRDs, some CRDs are added/deleted/modified concurrently, and the processing capacity of + // IngressController has no obvious relationship with the number of existing CRDs + It("", func() {}) - It("When IngressController is unexpectedly unavailable, it does not affect the existing configuration of the data plane", func() {}) + // When IngressController is unexpectedly unavailable, it does not affect the existing configuration of the + // data plane + It("", func() {}) }) }) From e1c23de5d336eb5c9e7633be950d3762ea41c785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=82=9F=E7=A9=BA?= Date: Wed, 2 Apr 2025 09:05:37 +0800 Subject: [PATCH 3/3] Ingress Controller is crashing --- test/long_term_stability/spec_subjects/spec_subjects.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/long_term_stability/spec_subjects/spec_subjects.go b/test/long_term_stability/spec_subjects/spec_subjects.go index f107403f2..0acf0be5d 100644 --- a/test/long_term_stability/spec_subjects/spec_subjects.go +++ b/test/long_term_stability/spec_subjects/spec_subjects.go @@ -267,7 +267,13 @@ spec: PContext("Ingress Controller is crashing", func() { It("it 0", func() { - Ω(true).Should(BeTrue()) + for i := 0; i < 5; i++ { + go s.RestartAPISIXDeploy() + time.Sleep(2 * time.Minute) + } + + err := s.DownloadLocustReport(fmt.Sprintf("%02d_ingress_controller_is_creshing", it)) + Expect(err).NotTo(HaveOccurred(), "getting locust report") }) })