From fe3296925b39d60e18663df309da88757776d694 Mon Sep 17 00:00:00 2001 From: Jason DeTiberus Date: Tue, 1 Oct 2019 15:50:18 -0400 Subject: [PATCH] Add helper utilities for testing - Add common kind utility for provider-based testing - Add flag helper - Add scheme helper - Add component helper --- Makefile | 16 +++ config/default/kustomization.yaml | 1 + config/default/manager_pull_policy.yaml | 11 ++ scripts/ci-integration.sh | 2 +- test/e2e/capi_test.go | 26 ++++ test/e2e/e2e_suite_test.go | 141 ++++++++++++++++++++++ test/helpers/components/common.go | 58 +++++++++ test/helpers/flag/flag.go | 30 +++++ test/helpers/kind/setup.go | 152 ++++++++++++++++++++++++ test/helpers/scheme/scheme.go | 31 +++++ 10 files changed, 467 insertions(+), 1 deletion(-) create mode 100644 config/default/manager_pull_policy.yaml create mode 100644 test/e2e/capi_test.go create mode 100644 test/e2e/e2e_suite_test.go create mode 100644 test/helpers/components/common.go create mode 100644 test/helpers/flag/flag.go create mode 100644 test/helpers/kind/setup.go create mode 100644 test/helpers/scheme/scheme.go diff --git a/Makefile b/Makefile index c6cde34956eb..33079e61aa3f 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ TAG ?= dev ARCH ?= amd64 ALL_ARCH = amd64 arm arm64 ppc64le s390x +# Allow overriding the imagePullPolicy +PULL_POLICY ?= Always + all: test manager clusterctl help: ## Display this help @@ -71,6 +74,11 @@ test: generate lint ## Run tests test-integration: ## Run integration tests go test -v -tags=integration ./test/integration/... +.PHONY: test-e2e +test-e2e: ## Run e2e tests + PULL_POLICY=IfNotPresent $(MAKE) docker-build + go test -v -tags=e2e -timeout=1h ./test/e2e/... -args --managerImage $(CONTROLLER_IMG)-$(ARCH):$(TAG) + ## -------------------------------------- ## Binaries ## -------------------------------------- @@ -147,6 +155,7 @@ modules: ## Runs go mod to ensure modules are up to date. docker-build: ## Build the docker image for controller-manager docker build --pull --build-arg ARCH=$(ARCH) . -t $(CONTROLLER_IMG)-$(ARCH):$(TAG) MANIFEST_IMG=$(CONTROLLER_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image + $(MAKE) set-manifest-pull-policy .PHONY: docker-push docker-push: ## Push the docker image @@ -176,12 +185,18 @@ docker-push-manifest: ## Push the fat manifest docker image. @for arch in $(ALL_ARCH); do docker manifest annotate --arch $${arch} ${CONTROLLER_IMG}:${TAG} ${CONTROLLER_IMG}-$${arch}:${TAG}; done docker manifest push --purge $(CONTROLLER_IMG):$(TAG) MANIFEST_IMG=$(CONTROLLER_IMG) MANIFEST_TAG=$(TAG) $(MAKE) set-manifest-image + $(MAKE) set-manifest-pull-policy .PHONY: set-manifest-image set-manifest-image: $(info Updating kustomize image patch file for manager resource) sed -i'' -e 's@image: .*@image: '"${MANIFEST_IMG}:$(MANIFEST_TAG)"'@' ./config/default/manager_image_patch.yaml +.PHONY: set-manifest-pull-policy +set-manifest-pull-policy: + $(info Updating kustomize pull policy file for manager resource) + sed -i'' -e 's@imagePullPolicy: .*@imagePullPolicy: '"$(PULL_POLICY)"'@' ./config/default/manager_pull_policy.yaml + ## -------------------------------------- ## Release ## -------------------------------------- @@ -203,6 +218,7 @@ release: clean-release ## Builds and push container images using the latest git # Set the manifest image to the production bucket. MANIFEST_IMG=$(PROD_REGISTRY)/$(IMAGE_NAME) MANIFEST_TAG=$(RELEASE_TAG) \ $(MAKE) set-manifest-image + PULL_POLICY=IfNotPresent $(MAKE) set-manifest-pull-policy $(MAKE) release-manifests $(MAKE) release-binaries diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index aedfa377808f..2d48f521f3c9 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -18,6 +18,7 @@ namePrefix: capi- patchesStrategicMerge: - manager_image_patch.yaml - manager_label_patch.yaml +- manager_pull_policy.yaml # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in crd/kustomization.yaml #- manager_webhook_patch.yaml diff --git a/config/default/manager_pull_policy.yaml b/config/default/manager_pull_policy.yaml new file mode 100644 index 000000000000..74a0879c604a --- /dev/null +++ b/config/default/manager_pull_policy.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + imagePullPolicy: Always diff --git a/scripts/ci-integration.sh b/scripts/ci-integration.sh index a14da3bf2da4..b54ecef68119 100755 --- a/scripts/ci-integration.sh +++ b/scripts/ci-integration.sh @@ -57,7 +57,7 @@ build_containers() { export CONTROLLER_IMG="${CONTROLLER_REPO}" export EXAMPLE_PROVIDER_IMG="${EXAMPLE_PROVIDER_REPO}" - "${MAKE}" docker-build TAG="${VERSION}" ARCH="${GOARCH}" + "${MAKE}" docker-build TAG="${VERSION}" ARCH="${GOARCH}" PULL_POLICY=IfNotPresent "${MAKE}" docker-build-example-provider TAG="${VERSION}" ARCH="${GOARCH}" } diff --git a/test/e2e/capi_test.go b/test/e2e/capi_test.go new file mode 100644 index 000000000000..f04fdba37b8e --- /dev/null +++ b/test/e2e/capi_test.go @@ -0,0 +1,26 @@ +// +build e2e + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + "github.com/onsi/ginkgo" +) + +var _ = ginkgo.Describe("functional tests", func() { +}) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go new file mode 100644 index 000000000000..b358490c42b3 --- /dev/null +++ b/test/e2e/e2e_suite_test.go @@ -0,0 +1,141 @@ +// +build e2e + +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e_test + +import ( + "context" + "flag" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "testing" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + apimachinerytypes "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/cluster-api/test/helpers/kind" + "sigs.k8s.io/cluster-api/test/helpers/scheme" + "sigs.k8s.io/cluster-api/util" + crclient "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestE2E(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "e2e Suite") +} + +const ( + capiNamespace = "capi-system" + capiDeploymentName = "capi-controller-manager" + setupTimeout = 10 * 60 +) + +var ( + managerImage = flag.String("managerImage", "", "Docker image to load into the kind cluster for testing") + capiComponents = flag.String("capiComponents", "", "URL to CAPI components to load") + kustomizeBinary = flag.String("kustomizeBinary", "kustomize", "path to the kustomize binary") + + kindCluster kind.Cluster + kindClient crclient.Client + suiteTmpDir string +) + +var _ = ginkgo.BeforeSuite(func() { + fmt.Fprintf(ginkgo.GinkgoWriter, "Setting up kind cluster\n") + + var err error + suiteTmpDir, err = ioutil.TempDir("", "capi-e2e-suite") + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + kindCluster = kind.Cluster{ + Name: "capi-test-" + util.RandomString(6), + } + kindCluster.Setup() + loadManagerImage(kindCluster) + + // Deploy the CAPA components + deployCAPIComponents(kindCluster) + + kindClient, err = crclient.New(kindCluster.RestConfig(), crclient.Options{Scheme: scheme.SetupScheme()}) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // Verify capi components are deployed + waitDeployment(capiNamespace, capiDeploymentName) +}, setupTimeout) + +var _ = ginkgo.AfterSuite(func() { + fmt.Fprintf(ginkgo.GinkgoWriter, "Tearing down kind cluster\n") + kindCluster.Teardown() + os.RemoveAll(suiteTmpDir) +}) + +func waitDeployment(namespace, name string) { + fmt.Fprintf(ginkgo.GinkgoWriter, "Ensuring %s/%s is deployed\n", namespace, name) + gomega.Eventually( + func() (int32, error) { + deployment := &appsv1.Deployment{} + if err := kindClient.Get(context.TODO(), apimachinerytypes.NamespacedName{Namespace: namespace, Name: name}, deployment); err != nil { + return 0, err + } + return deployment.Status.ReadyReplicas, nil + }, 5*time.Minute, 15*time.Second, + ).ShouldNot(gomega.BeZero()) +} + +func loadManagerImage(kindCluster kind.Cluster) { + if managerImage != nil && *managerImage != "" { + kindCluster.LoadImage(*managerImage) + } +} + +func applyManifests(kindCluster kind.Cluster, manifests *string) { + gomega.Expect(manifests).ToNot(gomega.BeNil()) + fmt.Fprintf(ginkgo.GinkgoWriter, "Applying manifests for %s\n", *manifests) + gomega.Expect(*manifests).ToNot(gomega.BeEmpty()) + kindCluster.ApplyYAML(*manifests) +} + +func deployCAPIComponents(kindCluster kind.Cluster) { + if capiComponents != nil && *capiComponents != "" { + applyManifests(kindCluster, capiComponents) + return + } + + fmt.Fprintf(ginkgo.GinkgoWriter, "Generating CAPI manifests\n") + + // Build the manifests using kustomize + capiManifests, err := exec.Command(*kustomizeBinary, "build", "../../config/default").Output() + if err != nil { + if exitError, ok := err.(*exec.ExitError); ok { + fmt.Fprintf(ginkgo.GinkgoWriter, "Error: %s\n", string(exitError.Stderr)) + } + } + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + + // write out the manifests + manifestFile := path.Join(suiteTmpDir, "cluster-api-components.yaml") + gomega.Expect(ioutil.WriteFile(manifestFile, capiManifests, 0644)).NotTo(gomega.HaveOccurred()) + + // apply generated manifests + applyManifests(kindCluster, &manifestFile) +} diff --git a/test/helpers/components/common.go b/test/helpers/components/common.go new file mode 100644 index 000000000000..4ac8622a8896 --- /dev/null +++ b/test/helpers/components/common.go @@ -0,0 +1,58 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/cluster-api/test/helpers/flag" + "sigs.k8s.io/cluster-api/test/helpers/kind" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + CAPIVersion = "v0.2.2" +) + +var ( + capiComponents = flag.DefineOrLookupStringFlag("capiComponents", "https://github.com/kubernetes-sigs/cluster-api/releases/download/"+CAPIVersion+"/cluster-api-components.yaml", "URL to CAPI components to load") +) + +func DeployCAPIComponents(kindCluster kind.Cluster) { + gomega.Expect(capiComponents).ToNot(gomega.BeNil()) + fmt.Fprintf(ginkgo.GinkgoWriter, "Applying cluster-api components\n") + gomega.Expect(*capiComponents).ToNot(gomega.BeEmpty()) + kindCluster.ApplyYAML(*capiComponents) +} + +func WaitDeployment(c client.Client, namespace, name string) { + fmt.Fprintf(ginkgo.GinkgoWriter, "Ensuring %s/%s is deployed\n", namespace, name) + gomega.Eventually( + func() (int32, error) { + deployment := &appsv1.Deployment{} + if err := c.Get(context.TODO(), client.ObjectKey{Namespace: namespace, Name: name}, deployment); err != nil { + return 0, err + } + return deployment.Status.ReadyReplicas, nil + }, 5*time.Minute, 15*time.Second, + ).ShouldNot(gomega.BeZero()) +} diff --git a/test/helpers/flag/flag.go b/test/helpers/flag/flag.go new file mode 100644 index 000000000000..6db24419c800 --- /dev/null +++ b/test/helpers/flag/flag.go @@ -0,0 +1,30 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package flag + +import ( + "flag" +) + +func DefineOrLookupStringFlag(name string, value string, usage string) *string { + f := flag.Lookup(name) + if f != nil { + v := f.Value.String() + return &v + } + return flag.String(name, value, usage) +} diff --git a/test/helpers/kind/setup.go b/test/helpers/kind/setup.go new file mode 100644 index 000000000000..830cfa0cba4e --- /dev/null +++ b/test/helpers/kind/setup.go @@ -0,0 +1,152 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kind + +import ( + "bufio" + "bytes" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "strings" + "sync" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +var ( + kindBinary = flag.String("kindBinary", "kind", "path to the kind binary") + kubectlBinary = flag.String("kubectlBinary", "kubectl", "path to the kubectl binary") +) + +// Cluster represents the running state of a KIND cluster. +// An empty struct is enough to call Setup() on. +type Cluster struct { + Name string + tmpDir string + kubepath string +} + +// Setup creates a kind cluster and returns a path to the kubeconfig +func (c *Cluster) Setup() { + var err error + c.tmpDir, err = ioutil.TempDir("", "kind-home") + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + fmt.Fprintf(ginkgo.GinkgoWriter, "creating Kind cluster named %q\n", c.Name) + c.run(exec.Command(*kindBinary, "create", "cluster", "--name", c.Name)) + path := c.runWithOutput(exec.Command(*kindBinary, "get", "kubeconfig-path", "--name", c.Name)) + c.kubepath = strings.TrimSpace(string(path)) + fmt.Fprintf(ginkgo.GinkgoWriter, "kubeconfig path: %q. Can use the following to access the cluster:\n", c.kubepath) + fmt.Fprintf(ginkgo.GinkgoWriter, "export KUBECONFIG=%s\n", c.kubepath) +} + +// Teardown attempts to delete the KIND cluster +func (c *Cluster) Teardown() { + c.run(exec.Command(*kindBinary, "delete", "cluster", "--name", c.Name)) + os.RemoveAll(c.tmpDir) +} + +// LoadImage loads the specified image archive into the kind cluster +func (c *Cluster) LoadImage(image string) { + fmt.Fprintf( + ginkgo.GinkgoWriter, + "loading image %q into Kind node\n", + image) + c.run(exec.Command(*kindBinary, "load", "docker-image", "--name", c.Name, image)) +} + +// ApplyYAML applies the provided manifest to the kind cluster +func (c *Cluster) ApplyYAML(manifestPath string) { + c.run(exec.Command( + *kubectlBinary, + "create", + "--kubeconfig="+c.kubepath, + "-f", manifestPath, + )) +} + +// RestConfig returns a rest configuration pointed at the provisioned cluster +func (c *Cluster) RestConfig() *restclient.Config { + cfg, err := clientcmd.BuildConfigFromFlags("", c.kubepath) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + return cfg +} + +// KubeClient returns a Kubernetes client pointing at the provisioned cluster +func (c *Cluster) KubeClient() kubernetes.Interface { + cfg := c.RestConfig() + client, err := kubernetes.NewForConfig(cfg) + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + return client +} + +func (c *Cluster) runWithOutput(cmd *exec.Cmd) []byte { + var stdout bytes.Buffer + cmd.Stdout = &stdout + c.run(cmd) + return stdout.Bytes() +} + +func (c *Cluster) run(cmd *exec.Cmd) { + var wg sync.WaitGroup + errPipe, err := cmd.StderrPipe() + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + + cmd.Env = append( + cmd.Env, + // KIND positions the configuration file relative to HOME. + // To prevent clobbering an existing KIND installation, override this + // n.b. HOME isn't always set inside BAZEL + fmt.Sprintf("HOME=%s", c.tmpDir), + //needed for Docker. TODO(EKF) Should be properly hermetic + fmt.Sprintf("PATH=%s", os.Getenv("PATH")), + ) + + // Log output + wg.Add(1) + go captureOutput(&wg, errPipe, "stderr") + if cmd.Stdout == nil { + outPipe, err := cmd.StdoutPipe() + gomega.ExpectWithOffset(1, err).NotTo(gomega.HaveOccurred()) + wg.Add(1) + go captureOutput(&wg, outPipe, "stdout") + } + + gomega.Expect(cmd.Start()).NotTo(gomega.HaveOccurred()) + wg.Wait() + gomega.Expect(cmd.Wait()).NotTo(gomega.HaveOccurred()) +} + +func captureOutput(wg *sync.WaitGroup, r io.Reader, label string) { + defer wg.Done() + reader := bufio.NewReader(r) + + for { + line, err := reader.ReadString('\n') + fmt.Fprintf(ginkgo.GinkgoWriter, "[%s] %s", label, line) + if err != nil { + return + } + } +} diff --git a/test/helpers/scheme/scheme.go b/test/helpers/scheme/scheme.go new file mode 100644 index 000000000000..2651520adab7 --- /dev/null +++ b/test/helpers/scheme/scheme.go @@ -0,0 +1,31 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scheme + +import ( + "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + clusterv1 "sigs.k8s.io/cluster-api/api/v1alpha2" +) + +func SetupScheme() *runtime.Scheme { + scheme := runtime.NewScheme() + gomega.Expect(clientgoscheme.AddToScheme(scheme)).NotTo(gomega.HaveOccurred()) + gomega.Expect(clusterv1.AddToScheme(scheme)).NotTo(gomega.HaveOccurred()) + return scheme +}