From 749be6330de5d44c7a97bd1ec61bd329c9e6e1b9 Mon Sep 17 00:00:00 2001 From: ricolin Date: Thu, 31 Oct 2024 21:55:07 +0800 Subject: [PATCH 01/13] CI: skip upgrade with kube_tag in functional test --- .../tests/functional/test_driver.py | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/magnum_cluster_api/tests/functional/test_driver.py b/magnum_cluster_api/tests/functional/test_driver.py index b9c56f89..6d9891f0 100644 --- a/magnum_cluster_api/tests/functional/test_driver.py +++ b/magnum_cluster_api/tests/functional/test_driver.py @@ -90,25 +90,6 @@ def _create_node_group(self, context, driver, node_group_name, cluster_template) return new_node_group - def test_upgrade_cluster(self, context, ubuntu_driver, cluster_template): - cluster_template.labels["kube_tag"] = "v1.26.3" - - cluster_resource = objects.Cluster.for_magnum_cluster(self.api, self.cluster) - current_observed_generation = cluster_resource.observed_generation - - ubuntu_driver.upgrade_cluster( - context, self.cluster, cluster_template, None, None - ) - - cluster_resource.wait_for_observed_generation_changed( - existing_observed_generation=current_observed_generation, - ) - - cluster_resource = objects.Cluster.for_magnum_cluster(self.api, self.cluster) - assert cluster_resource.observed_generation != current_observed_generation - - self.cluster.save.assert_not_called() - def test_upgrade_cluster_to_same_version( self, kube_tag, context, ubuntu_driver, cluster_template ): @@ -149,8 +130,6 @@ def test_upgrade_cluster_with_multiple_node_groups( ], ) - cluster_template.labels["kube_tag"] = "v1.26.3" - cluster_resource = objects.Cluster.for_magnum_cluster(self.api, self.cluster) current_observed_generation = cluster_resource.observed_generation From 91457fe71373cc3feef5c3c79afa5eda42415b12 Mon Sep 17 00:00:00 2001 From: ricolin Date: Fri, 1 Nov 2024 10:19:46 +0800 Subject: [PATCH 02/13] Add upgrade tests to integration test --- hack/run-integration-tests.sh | 37 +++++++++++++++++++++++++++++++ zuul.d/jobs-ubuntu-2204.yaml | 3 +++ zuul.d/playbooks/sonobuoy/run.yml | 26 ++++++++++++++++++++++ 3 files changed, 66 insertions(+) diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index aa669199..4060cf9b 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -27,6 +27,7 @@ NETWORK_DRIVER=${NETWORK_DRIVER:-calico} SONOBUOY_VERSION=${SONOBUOY_VERSION:-0.56.16} SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64} DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} +UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" # If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise @@ -60,6 +61,23 @@ openstack coe cluster template create \ --label fixed_subnet_cidr=192.168.24.0/24 \ k8s-${KUBE_TAG}; +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + # Create cluster template for upgrade + openstack coe cluster template create \ + --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${UPGRADE_KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${UPGRADE_KUBE_TAG}; +fi + # Create cluster openstack coe cluster create \ --cluster-template k8s-${KUBE_TAG} \ @@ -101,6 +119,25 @@ done # Get the cluster configuration file eval $(openstack coe cluster config k8s-cluster) +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to upgrade" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + break + else + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done +fi + # Download sonobuoy curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz tar -xzf sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index af35355c..63ed4ae6 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -11,6 +11,7 @@ parent: magnum-cluster-api-sonobuoy-ubuntu-2204 vars: kube_tag: v1.28.11 + upgrade_kube_tag: v1.29.6 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-28-11-1719601167.qcow2 - job: @@ -30,6 +31,7 @@ parent: magnum-cluster-api-sonobuoy-ubuntu-2204 vars: kube_tag: v1.29.6 + upgrade_kube_tag: v1.30.2 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 - job: @@ -49,6 +51,7 @@ parent: magnum-cluster-api-sonobuoy-ubuntu-2204 vars: kube_tag: v1.30.2 + upgrade_kube_tag: v1.31.1 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 - job: diff --git a/zuul.d/playbooks/sonobuoy/run.yml b/zuul.d/playbooks/sonobuoy/run.yml index 0e5089f2..6a08af5b 100644 --- a/zuul.d/playbooks/sonobuoy/run.yml +++ b/zuul.d/playbooks/sonobuoy/run.yml @@ -21,6 +21,31 @@ when: - image_url is defined + - name: Fetch artifact for upgrade + get_url: + url: "{{ item.url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact + loop: "{{ zuul.artifacts }}" + when: + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + - item.name is defined + - item.name == "{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + + - name: Download image for upgrade + get_url: + url: "{{ image_url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact + retries: 5 + delay: 10 + when: + - image_url is defined + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + + - shell: "./hack/stack.sh" args: chdir: "{{ zuul.project.src_dir }}" @@ -32,6 +57,7 @@ IMAGE_OS: "{{ image_operating_system }}" OS_DISTRO: "{{ image_os_distro }}" KUBE_TAG: "{{ kube_tag }}" + UPGRADE_KUBE_TAG: "{{ upgrade_kube_tag | default(kube_tag) }}" NODE_COUNT: 2 BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" NETWORK_DRIVER: "{{ network_driver }}" From d0d2f8c871c66c8effea7e9c6e6f6770b56e361a Mon Sep 17 00:00:00 2001 From: ricolin Date: Fri, 1 Nov 2024 12:10:29 +0800 Subject: [PATCH 03/13] tunning integration for upgrade --- hack/run-integration-tests.sh | 115 ++++++++++++++++++++++++------ zuul.d/jobs-ubuntu-2204.yaml | 4 ++ zuul.d/playbooks/sonobuoy/run.yml | 9 +-- 3 files changed, 104 insertions(+), 24 deletions(-) diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index 4060cf9b..096cd5d0 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -29,6 +29,7 @@ SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64} DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" +UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}" # If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise # we download the latest promoted image. @@ -46,6 +47,21 @@ openstack image create \ --file=${IMAGE_NAME}.qcow2 \ ${IMAGE_NAME} +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then + curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2 + else + test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1 + fi + # Upload Upgrade image to Glance + openstack image create \ + --disk-format=qcow2 \ + --container-format=bare \ + --property os_distro=${OS_DISTRO} \ + --file=${UPGRADE_IMAGE_NAME}.qcow2 \ + ${UPGRADE_IMAGE_NAME} +fi + # Create cluster template openstack coe cluster template create \ --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ @@ -64,7 +80,7 @@ openstack coe cluster template create \ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then # Create cluster template for upgrade openstack coe cluster template create \ - --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ + --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ --external-network public \ --dns-nameserver ${DNS_NAMESERVER} \ --master-lb-enabled \ @@ -111,6 +127,7 @@ for i in {1..240}; do echo "Cluster created" break else + echo "Currtny retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi @@ -119,25 +136,6 @@ done # Get the cluster configuration file eval $(openstack coe cluster config k8s-cluster) -if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then - # Upgrade cluster - openstack coe cluster upgrade k8s-cluster k8s-${UPGRADE_KUBE_TAG} - # Wait for cluster to be "UPDATE_COMPLETE". - for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster -c status -f value) - if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to upgrade" - exit 1 - elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then - echo "Cluster upgraded" - break - else - echo "Cluster status: ${CLUSTER_STATUS}" - sleep 5 - fi - done -fi - # Download sonobuoy curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz tar -xzf sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz @@ -157,3 +155,80 @@ if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed"; echo "Sonobuoy tests failed" exit 1 fi + + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + + openstack coe cluster delete k8s-cluster + # Wait for cluster to be deleted + set +e + for i in {1..60}; do + openstack coe cluster show k8s-cluster 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + sleep 2 + else + echo "Cluster k8s-cluster deleted." + break + fi + done + set -e + # Create cluster + openstack coe cluster create \ + --cluster-template k8s-${KUBE_TAG} \ + --master-count 1 \ + --node-count 1 \ + --merge-labels \ + --label audit_log_enabled=true \ + k8s-cluster-upgrade + + # Wait for cluster creation to be queued + set +e + for i in {1..5}; do + openstack coe cluster show k8s-cluster-upgrade 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + break + else + echo "Error: Cluster k8s-cluster-upgrade could not be found." + sleep 1 + fi + done + set -e + + # Wait for cluster to be "CREATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to create" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then + echo "Cluster created" + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to upgrade" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + exit 0 + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + exit 1 +fi diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index 63ed4ae6..a45f79f8 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -1,6 +1,7 @@ - job: name: magnum-cluster-api-sonobuoy-ubuntu-2204 parent: magnum-cluster-api-sonobuoy + timeout: 10800 abstract: true vars: image_operating_system: ubuntu-2204 @@ -13,6 +14,7 @@ kube_tag: v1.28.11 upgrade_kube_tag: v1.29.6 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-28-11-1719601167.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 - job: name: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.28.11-calico @@ -33,6 +35,7 @@ kube_tag: v1.29.6 upgrade_kube_tag: v1.30.2 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 - job: name: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.29.6-calico @@ -53,6 +56,7 @@ kube_tag: v1.30.2 upgrade_kube_tag: v1.31.1 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 - job: name: magnum-cluster-api-sonobuoy-ubuntu-2204-v1.30.2-calico diff --git a/zuul.d/playbooks/sonobuoy/run.yml b/zuul.d/playbooks/sonobuoy/run.yml index 6a08af5b..6e9da177 100644 --- a/zuul.d/playbooks/sonobuoy/run.yml +++ b/zuul.d/playbooks/sonobuoy/run.yml @@ -25,7 +25,7 @@ get_url: url: "{{ item.url }}" dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" - register: fetch_artifact + register: fetch_artifact_upgrade loop: "{{ zuul.artifacts }}" when: - upgrade_kube_tag is defined @@ -35,13 +35,13 @@ - name: Download image for upgrade get_url: - url: "{{ image_url }}" + url: "{{ upgrade_image_url }}" dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" - register: fetch_artifact + register: fetch_artifact_upgrade retries: 5 delay: 10 when: - - image_url is defined + - upgrade_image_url is defined - upgrade_kube_tag is defined - upgrade_kube_tag != kube_tag @@ -60,6 +60,7 @@ UPGRADE_KUBE_TAG: "{{ upgrade_kube_tag | default(kube_tag) }}" NODE_COUNT: 2 BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" + BUILD_NEW_UPGRADE_IMAGE: "{{ fetch_artifact_upgrade.changed }}" NETWORK_DRIVER: "{{ network_driver }}" - name: Copy Sonobuoy results to output folder From cfa6f954aff26a429fa3d8c2ce32c245692adf56 Mon Sep 17 00:00:00 2001 From: ricolin Date: Wed, 20 Nov 2024 16:53:58 +0800 Subject: [PATCH 04/13] Add Tempest support --- hack/run-tempest-tests.sh | 160 ++++++++++++++++++++++++++++++ hack/stack.sh | 13 +++ zuul.d/jobs-ubuntu-2204.yaml | 106 +++++++++++++++++++- zuul.d/jobs.yaml | 11 ++ zuul.d/playbooks/tempest/post.yml | 9 ++ zuul.d/playbooks/tempest/run.yml | 71 +++++++++++++ zuul.d/project.yaml | 3 +- 7 files changed, 371 insertions(+), 2 deletions(-) create mode 100755 hack/run-tempest-tests.sh create mode 100644 zuul.d/playbooks/tempest/post.yml create mode 100644 zuul.d/playbooks/tempest/run.yml diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh new file mode 100755 index 00000000..46cac1a7 --- /dev/null +++ b/hack/run-tempest-tests.sh @@ -0,0 +1,160 @@ +#!/bin/bash -xe + +# Copyright (c) 2024 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This script will run the full functional tests for a given `KUBE_TAG`. It +# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE` +# and then run `sonobuoy` against it. + +source /opt/stack/openrc admin admin + +OS_DISTRO=${OS_DISTRO:-ubuntu} +IMAGE_OS=${IMAGE_OS:-ubuntu-2204} +NETWORK_DRIVER=${NETWORK_DRIVER:-calico} +DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} +UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} +IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" +UPGRADE_IMAGE_NAME="${IMAGE_OS}-kube-${UPGRADE_KUBE_TAG}" + +# If `BUILD_NEW_IMAGE` is true, then we use the provided artifact, otherwise +# we download the latest promoted image. +if [[ "${BUILD_NEW_IMAGE,,}" != "true" ]]; then + curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${IMAGE_NAME}.qcow2 +else + test -f ${IMAGE_NAME}.qcow2 || exit 1 +fi + +# Upload image to Glance +openstack image create \ + --disk-format=qcow2 \ + --public \ + --container-format=bare \ + --property os_distro=${OS_DISTRO} \ + --file=${IMAGE_NAME}.qcow2 \ + ${IMAGE_NAME} + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + if [[ "${BUILD_NEW_UPGRADE_IMAGE,,}" != "true" ]]; then + curl -LO https://object-storage.public.mtl1.vexxhost.net/swift/v1/a91f106f55e64246babde7402c21b87a/magnum-capi/${UPGRADE_IMAGE_NAME}.qcow2 + else + test -f ${UPGRADE_IMAGE_NAME}.qcow2 || exit 1 + fi + # Upload Upgrade image to Glance + openstack image create \ + --disk-format=qcow2 \ + --public \ + --container-format=bare \ + --property os_distro=${OS_DISTRO} \ + --file=${UPGRADE_IMAGE_NAME}.qcow2 \ + ${UPGRADE_IMAGE_NAME} +fi + +pushd /opt/stack/tempest +/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ + --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' +popd + + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + + # Create cluster template + openstack coe cluster template create \ + --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${KUBE_TAG}; + + # Create cluster template for upgrade + openstack coe cluster template create \ + --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${UPGRADE_KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${UPGRADE_KUBE_TAG}; + + # Create cluster + openstack coe cluster create \ + --cluster-template k8s-${KUBE_TAG} \ + --master-count 1 \ + --node-count 1 \ + --merge-labels \ + --label audit_log_enabled=true \ + k8s-cluster-upgrade + + # Wait for cluster creation to be queued + set +e + for i in {1..5}; do + openstack coe cluster show k8s-cluster-upgrade 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + break + else + echo "Error: Cluster k8s-cluster-upgrade could not be found." + sleep 1 + fi + done + set -e + + # Wait for cluster to be "CREATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to create" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then + echo "Cluster created" + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to upgrade" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + exit 0 + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi + done + exit 1 +fi diff --git a/hack/stack.sh b/hack/stack.sh index 50ea8d78..e7c10a7f 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -18,6 +18,9 @@ sudo mkdir -p /opt/stack sudo chown -R ${USER}. /opt/stack +sudo mkdir -p /etc/tempest/ +sudo chown -R ${USER}. /etc/tempest/ + # Clone repository if not present, otherwise update if [ ! -f /opt/stack/stack.sh ]; then git clone https://git.openstack.org/openstack-dev/devstack /opt/stack @@ -62,6 +65,7 @@ enable_plugin barbican https://opendev.org/openstack/barbican enable_plugin octavia https://opendev.org/openstack/octavia enable_plugin ovn-octavia-provider https://opendev.org/openstack/ovn-octavia-provider enable_service octavia o-api o-cw o-hm o-hk o-da +enable_service tempest # Magnum enable_plugin magnum https://opendev.org/openstack/magnum @@ -113,5 +117,14 @@ EOF pip install -U setuptools pip python-magnumclient $HOME/.local/bin/pip3 install -e . +# install magnum-tempest-plugin with fix +git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin +pushd /opt/stack/magnum-tempest-plugin +git fetch https://review.opendev.org/openstack/magnum-tempest-plugin refs/changes/41/935741/1 && git checkout FETCH_HEAD +source /opt/stack/data/venv/bin/activate +#$HOME/.local/bin/pip3 install -e . +pip install . +popd + # Restart Magnum to pick-up new driver sudo systemctl restart devstack@magnum-{api,cond} diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index a45f79f8..746f6647 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -90,7 +90,7 @@ network_driver: cilium - project-template: - name: magnum-cluster-api-ubuntu-2204 + name: magnum-cluster-api-sonobuoy-ubuntu-2204 check: jobs: - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.28.11-calico @@ -101,3 +101,107 @@ - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.30.2-cilium - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-calico - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204 + parent: magnum-cluster-api-tempest + timeout: 10800 + abstract: true + vars: + image_operating_system: ubuntu-2204 + image_os_distro: ubuntu + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.28.11 + upgrade_kube_tag: v1.29.6 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-28-11-1719601167.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.29.6 + upgrade_kube_tag: v1.30.2 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.30.2 + upgrade_kube_tag: v1.31.1 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 + upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + vars: + network_driver: cilium + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + parent: magnum-cluster-api-tempest-ubuntu-2204 + vars: + kube_tag: v1.31.1 + image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + vars: + network_driver: calico + +- job: + name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium + parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + vars: + network_driver: cilium + +- project-template: + name: magnum-cluster-api-tempest-ubuntu-2204 + check: + jobs: + - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium + - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico + - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index edb5b0bb..65c54b8b 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -30,6 +30,17 @@ files: - magnum_cluster_api/cmd/image_builder.py +- job: + name: magnum-cluster-api-tempest + abstract: true + timeout: 7200 + run: zuul.d/playbooks/tempest/run.yml + post-run: zuul.d/playbooks/tempest/post.yml + nodeset: + nodes: + - name: ubuntu-jammy + label: jammy-16c-64g + - job: name: magnum-cluster-api-sonobuoy abstract: true diff --git a/zuul.d/playbooks/tempest/post.yml b/zuul.d/playbooks/tempest/post.yml new file mode 100644 index 00000000..00b5f62b --- /dev/null +++ b/zuul.d/playbooks/tempest/post.yml @@ -0,0 +1,9 @@ +- hosts: all + tasks: + - name: Return built artifacts to Zuul + zuul_return: + data: + zuul: + artifacts: + - name: "tempest Results" + url: "artifacts/tempest.log" diff --git a/zuul.d/playbooks/tempest/run.yml b/zuul.d/playbooks/tempest/run.yml new file mode 100644 index 00000000..d2d6de77 --- /dev/null +++ b/zuul.d/playbooks/tempest/run.yml @@ -0,0 +1,71 @@ +- hosts: all + tasks: + - name: Fetch artifact + get_url: + url: "{{ item.url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + register: fetch_artifact + loop: "{{ zuul.artifacts }}" + when: + - item.name is defined + - item.name == "{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + + - name: Download image + get_url: + url: "{{ image_url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ kube_tag }}.qcow2" + register: fetch_artifact + retries: 5 + delay: 10 + when: + - image_url is defined + + - name: Fetch artifact for upgrade + get_url: + url: "{{ item.url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact_upgrade + loop: "{{ zuul.artifacts }}" + when: + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + - item.name is defined + - item.name == "{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + + - name: Download image for upgrade + get_url: + url: "{{ upgrade_image_url }}" + dest: "{{ zuul.project.src_dir }}/{{ image_operating_system }}-kube-{{ upgrade_kube_tag }}.qcow2" + register: fetch_artifact_upgrade + retries: 5 + delay: 10 + when: + - upgrade_image_url is defined + - upgrade_kube_tag is defined + - upgrade_kube_tag != kube_tag + + - shell: "./hack/stack.sh" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + IMAGE_OS: "{{ image_operating_system }}" + KUBE_TAG: "{{ kube_tag }}" + NETWORK_DRIVER: "{{ network_driver }}" + + - shell: "./hack/run-tempest-tests.sh" + args: + chdir: "{{ zuul.project.src_dir }}" + environment: + IMAGE_OS: "{{ image_operating_system }}" + OS_DISTRO: "{{ image_os_distro }}" + KUBE_TAG: "{{ kube_tag }}" + UPGRADE_KUBE_TAG: "{{ upgrade_kube_tag | default(kube_tag) }}" + BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" + BUILD_NEW_UPGRADE_IMAGE: "{{ fetch_artifact_upgrade.changed }}" + NETWORK_DRIVER: "{{ network_driver }}" + + - name: Copy Tempest results to output folder + copy: + src: "/opt/stack/tempest/tempest.log" + dest: "{{ zuul_output_dir }}/artifacts/tempest.log" + remote_src: true diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index c05ba3ac..04d0bb82 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -9,4 +9,5 @@ - magnum-cluster-api-flatcar - magnum-cluster-api-rockylinux-8 - magnum-cluster-api-rockylinux-9 - - magnum-cluster-api-ubuntu-2204 + - magnum-cluster-api-tempest-ubuntu-2204 + #- magnum-cluster-api-sonobuoy-ubuntu-2204 From 6ae598a56e80c319d03911e72a7d083c511a5215 Mon Sep 17 00:00:00 2001 From: ricolin Date: Thu, 28 Nov 2024 10:50:03 +0800 Subject: [PATCH 05/13] Add mock_get_server_group to functional --- magnum_cluster_api/tests/functional/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/magnum_cluster_api/tests/functional/conftest.py b/magnum_cluster_api/tests/functional/conftest.py index a59a9b92..a5ce769a 100644 --- a/magnum_cluster_api/tests/functional/conftest.py +++ b/magnum_cluster_api/tests/functional/conftest.py @@ -25,6 +25,7 @@ def cluster( mock_validate_cluster, mock_osc, mock_certificates, + mock_get_server_group, ): try: ubuntu_driver.create_cluster(context, cluster_obj, 60) From f376b96b9d68d9df37aa4c676b96603556e8e825 Mon Sep 17 00:00:00 2001 From: ricolin Date: Mon, 25 Nov 2024 19:55:50 +0800 Subject: [PATCH 06/13] fix tempest --- hack/run-tempest-tests.sh | 33 ++++++++++++++++++++++++++++++- hack/stack.sh | 9 --------- zuul.d/jobs.yaml | 1 - zuul.d/playbooks/tempest/post.yml | 9 --------- zuul.d/playbooks/tempest/run.yml | 6 ------ 5 files changed, 32 insertions(+), 26 deletions(-) delete mode 100644 zuul.d/playbooks/tempest/post.yml diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index 46cac1a7..a8812c13 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -61,9 +61,40 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then ${UPGRADE_IMAGE_NAME} fi +mkdir /tmp/magnum-nodes + pushd /opt/stack/tempest +echo "Tempest configs:" + +cat <> /opt/stack/tempest/etc/tempest.conf + +[magnum] +flavor_id = m1.large +master_flavor_id = m1.large +copy_logs = true +network_driver = ${NETWORK_DRIVER} +image_id = ${IMAGE_OS}-kube-${KUBE_TAG} +coe = kubernetes +labels = '{"kube_tag": "${KUBE_TAG}", "fixed_subnet_cidr": "10.0.0.0/26"}' +docker_storage_driver = overlay2 + +EOF + +if [ ! -d /opt/stack/magnum-tempest-plugin ]; then + git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin +fi + +# install magnum-tempest-plugin +pushd /opt/stack/magnum-tempest-plugin +$HOME/.local/bin/pip3 install -e . +popd + +echo "Run Tempest against configs:" +cat /opt/stack/tempest/etc/tempest.conf + +echo "Run Tempest tests:" /opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ - --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' + --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes' popd diff --git a/hack/stack.sh b/hack/stack.sh index e7c10a7f..4ed4a969 100755 --- a/hack/stack.sh +++ b/hack/stack.sh @@ -117,14 +117,5 @@ EOF pip install -U setuptools pip python-magnumclient $HOME/.local/bin/pip3 install -e . -# install magnum-tempest-plugin with fix -git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin -pushd /opt/stack/magnum-tempest-plugin -git fetch https://review.opendev.org/openstack/magnum-tempest-plugin refs/changes/41/935741/1 && git checkout FETCH_HEAD -source /opt/stack/data/venv/bin/activate -#$HOME/.local/bin/pip3 install -e . -pip install . -popd - # Restart Magnum to pick-up new driver sudo systemctl restart devstack@magnum-{api,cond} diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 65c54b8b..2be8a343 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -35,7 +35,6 @@ abstract: true timeout: 7200 run: zuul.d/playbooks/tempest/run.yml - post-run: zuul.d/playbooks/tempest/post.yml nodeset: nodes: - name: ubuntu-jammy diff --git a/zuul.d/playbooks/tempest/post.yml b/zuul.d/playbooks/tempest/post.yml deleted file mode 100644 index 00b5f62b..00000000 --- a/zuul.d/playbooks/tempest/post.yml +++ /dev/null @@ -1,9 +0,0 @@ -- hosts: all - tasks: - - name: Return built artifacts to Zuul - zuul_return: - data: - zuul: - artifacts: - - name: "tempest Results" - url: "artifacts/tempest.log" diff --git a/zuul.d/playbooks/tempest/run.yml b/zuul.d/playbooks/tempest/run.yml index d2d6de77..61d407cf 100644 --- a/zuul.d/playbooks/tempest/run.yml +++ b/zuul.d/playbooks/tempest/run.yml @@ -63,9 +63,3 @@ BUILD_NEW_IMAGE: "{{ fetch_artifact.changed }}" BUILD_NEW_UPGRADE_IMAGE: "{{ fetch_artifact_upgrade.changed }}" NETWORK_DRIVER: "{{ network_driver }}" - - - name: Copy Tempest results to output folder - copy: - src: "/opt/stack/tempest/tempest.log" - dest: "{{ zuul_output_dir }}/artifacts/tempest.log" - remote_src: true From 136c499b36e9d430bc9a34081dafa9aa6d1ea3d8 Mon Sep 17 00:00:00 2001 From: ricolin Date: Fri, 29 Nov 2024 00:37:41 +0800 Subject: [PATCH 07/13] fix tests --- hack/run-tempest-tests.sh | 3 ++- magnum_cluster_api/tests/functional/test_driver.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index a8812c13..09962167 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -94,7 +94,8 @@ cat /opt/stack/tempest/etc/tempest.conf echo "Run Tempest tests:" /opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ - --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes' + --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' + #--exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes' popd diff --git a/magnum_cluster_api/tests/functional/test_driver.py b/magnum_cluster_api/tests/functional/test_driver.py index 6d9891f0..97c1b294 100644 --- a/magnum_cluster_api/tests/functional/test_driver.py +++ b/magnum_cluster_api/tests/functional/test_driver.py @@ -91,7 +91,7 @@ def _create_node_group(self, context, driver, node_group_name, cluster_template) return new_node_group def test_upgrade_cluster_to_same_version( - self, kube_tag, context, ubuntu_driver, cluster_template + self, kube_tag, context, ubuntu_driver, cluster_template, mock_get_server_group ): cluster_template.labels["kube_tag"] = kube_tag @@ -116,6 +116,7 @@ def test_upgrade_cluster_with_multiple_node_groups( context, ubuntu_driver, cluster_template, + mock_get_server_group, ): new_node_group = self._create_node_group( context, ubuntu_driver, "high-cpu", cluster_template From ccd359e4a69ed4ea6588fef2ad4a7e04f9d62078 Mon Sep 17 00:00:00 2001 From: ricolin Date: Fri, 10 Jan 2025 14:21:45 +0800 Subject: [PATCH 08/13] targeting docker_storage_driver config limit https://github.com/openstack/magnum-tempest-plugin/commit/22272488bc746d1c8eb8a1f8a979ad5af8dc4457 --- hack/run-tempest-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index 09962167..a5564767 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -76,7 +76,7 @@ network_driver = ${NETWORK_DRIVER} image_id = ${IMAGE_OS}-kube-${KUBE_TAG} coe = kubernetes labels = '{"kube_tag": "${KUBE_TAG}", "fixed_subnet_cidr": "10.0.0.0/26"}' -docker_storage_driver = overlay2 +docker_storage_driver = overlay EOF From d34699b035d141ad49deae4c11e7da36042cdfcf Mon Sep 17 00:00:00 2001 From: ricolin Date: Mon, 13 Jan 2025 16:37:12 +0800 Subject: [PATCH 09/13] chores: clean side note --- hack/run-tempest-tests.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index a5564767..b6b02a2f 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -95,7 +95,6 @@ cat /opt/stack/tempest/etc/tempest.conf echo "Run Tempest tests:" /opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' - #--exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest.test_create_cluster_with_zero_nodes' popd From 2fcd8dece646a53f7baaa8f94a3ac609e9961f31 Mon Sep 17 00:00:00 2001 From: ricolin Date: Mon, 13 Jan 2025 17:01:08 +0800 Subject: [PATCH 10/13] test: Enable k8s access test + tempest --- hack/run-tempest-tests.sh | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index b6b02a2f..af8b2668 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -14,10 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -# This script will run the full functional tests for a given `KUBE_TAG`. It -# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE` -# and then run `sonobuoy` against it. - source /opt/stack/openrc admin admin OS_DISTRO=${OS_DISTRO:-ubuntu} @@ -169,6 +165,26 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then fi done + # Wait for kubernetes accessible and Ready. + for i in {1..10}; do + # Get the cluster configuration file + eval $(openstack coe cluster config k8s-cluster-upgrade) + Node=$(kubectl get node -o wide) + if [[ ${Node} == *"NotReady"* ]]; then + echo "Kubernetes accessible with Not Ready node." + echo "Node status: ${Node}" + sleep 5 + elif [[ ${Node} == *"Ready"* ]]; then + echo "Kubernetes accessible with Ready node." + echo "Node status: ${Node}" + break + else + echo "Currtny retry count: $i" + echo "Node status: ${Node}" + sleep 5 + fi + done + # Upgrade cluster openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} # Wait for cluster to be "UPDATE_COMPLETE". From 3d2591422e2c13b174ada5d34405d1de7a099e2b Mon Sep 17 00:00:00 2001 From: ricolin Date: Tue, 14 Jan 2025 02:11:20 +0800 Subject: [PATCH 11/13] allow newest to run accessible check --- hack/run-tempest-tests.sh | 153 +++++++++++++++++++++----------------- 1 file changed, 85 insertions(+), 68 deletions(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index af8b2668..a93254f4 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -94,22 +94,80 @@ echo "Run Tempest tests:" popd -if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then +# Create cluster template +openstack coe cluster template create \ + --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${KUBE_TAG}; + +# Create cluster +openstack coe cluster create \ + --cluster-template k8s-${KUBE_TAG} \ + --master-count 1 \ + --node-count 1 \ + --merge-labels \ + --label audit_log_enabled=true \ + k8s-cluster-test + +# Wait for cluster creation to be queued +set +e +for i in {1..5}; do + openstack coe cluster show k8s-cluster-test 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + break + else + echo "Error: Cluster k8s-cluster-test could not be found." + sleep 1 + fi +done +set -e + +# Wait for cluster to be "CREATE_COMPLETE". +for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to create" + exit 1 + elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then + echo "Cluster created" + break + else + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" + sleep 5 + fi +done + +# Wait for kubernetes accessible and Ready. +for i in {1..10}; do + # Get the cluster configuration file + eval $(openstack coe cluster config k8s-cluster-test) + Node=$(kubectl get node -o wide) + ready_count=$(echo $Node | grep " Ready " | wc -l) + tag_count=$(echo $Node | grep $KUBE_TAG | wc -l) + if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then + echo "Kubernetes accessible with Ready node." + echo "Node status: ${Node}" + break + else + echo "Currtny retry count: $i" + echo "Node status: ${Node}" + sleep 5 + fi +done - # Create cluster template - openstack coe cluster template create \ - --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ - --external-network public \ - --dns-nameserver ${DNS_NAMESERVER} \ - --master-lb-enabled \ - --master-flavor m1.large \ - --flavor m1.large \ - --network-driver ${NETWORK_DRIVER} \ - --docker-storage-driver overlay2 \ - --coe kubernetes \ - --label kube_tag=${KUBE_TAG} \ - --label fixed_subnet_cidr=192.168.24.0/24 \ - k8s-${KUBE_TAG}; + +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then # Create cluster template for upgrade openstack coe cluster template create \ @@ -126,37 +184,17 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then --label fixed_subnet_cidr=192.168.24.0/24 \ k8s-${UPGRADE_KUBE_TAG}; - # Create cluster - openstack coe cluster create \ - --cluster-template k8s-${KUBE_TAG} \ - --master-count 1 \ - --node-count 1 \ - --merge-labels \ - --label audit_log_enabled=true \ - k8s-cluster-upgrade - - # Wait for cluster creation to be queued - set +e - for i in {1..5}; do - openstack coe cluster show k8s-cluster-upgrade 2>&1 - exit_status=$? - if [ $exit_status -eq 0 ]; then - break - else - echo "Error: Cluster k8s-cluster-upgrade could not be found." - sleep 1 - fi - done - set -e - - # Wait for cluster to be "CREATE_COMPLETE". + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-test k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to create" + echo "Cluster failed to upgrade" exit 1 - elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then - echo "Cluster created" + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + exit 0 break else echo "Currtny retry count: $i" @@ -168,13 +206,11 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then # Wait for kubernetes accessible and Ready. for i in {1..10}; do # Get the cluster configuration file - eval $(openstack coe cluster config k8s-cluster-upgrade) + eval $(openstack coe cluster config k8s-cluster-test) Node=$(kubectl get node -o wide) - if [[ ${Node} == *"NotReady"* ]]; then - echo "Kubernetes accessible with Not Ready node." - echo "Node status: ${Node}" - sleep 5 - elif [[ ${Node} == *"Ready"* ]]; then + ready_count=$(echo $Node | grep " Ready " | wc -l) + tag_count=$(echo $Node | grep $UPGRADE_KUBE_TAG | wc -l) + if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then echo "Kubernetes accessible with Ready node." echo "Node status: ${Node}" break @@ -184,24 +220,5 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then sleep 5 fi done - - # Upgrade cluster - openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} - # Wait for cluster to be "UPDATE_COMPLETE". - for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) - if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to upgrade" - exit 1 - elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then - echo "Cluster upgraded" - exit 0 - break - else - echo "Currtny retry count: $i" - echo "Cluster status: ${CLUSTER_STATUS}" - sleep 5 - fi - done exit 1 fi From 482a856de9dacac991513c002753b7a86232d3b2 Mon Sep 17 00:00:00 2001 From: ricolin Date: Tue, 14 Jan 2025 12:26:42 +0800 Subject: [PATCH 12/13] correct count on nodes --- hack/run-tempest-tests.sh | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/hack/run-tempest-tests.sh b/hack/run-tempest-tests.sh index a93254f4..d70dd70a 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-tempest-tests.sh @@ -142,26 +142,30 @@ for i in {1..240}; do echo "Cluster created" break else - echo "Currtny retry count: $i" + echo "Current retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done # Wait for kubernetes accessible and Ready. -for i in {1..10}; do +for i in {1..30}; do # Get the cluster configuration file eval $(openstack coe cluster config k8s-cluster-test) Node=$(kubectl get node -o wide) - ready_count=$(echo $Node | grep " Ready " | wc -l) - tag_count=$(echo $Node | grep $KUBE_TAG | wc -l) + ready_count=$(echo $Node | grep -o " Ready " | wc -l) + tag_count=$(echo $Node | grep -o $KUBE_TAG | wc -l) if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then echo "Kubernetes accessible with Ready node." echo "Node status: ${Node}" break - else - echo "Currtny retry count: $i" + elif [[ $i -eq 30 ]]; then + echo "Failed: Unable to reach ready status for nodes..." echo "Node status: ${Node}" + exit 1 + else + echo "Current retry count: $i" + # echo "Node status: ${Node}" sleep 5 fi done @@ -194,29 +198,32 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then exit 1 elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then echo "Cluster upgraded" - exit 0 break else - echo "Currtny retry count: $i" + echo "Current retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done # Wait for kubernetes accessible and Ready. - for i in {1..10}; do + for i in {1..30}; do # Get the cluster configuration file eval $(openstack coe cluster config k8s-cluster-test) Node=$(kubectl get node -o wide) - ready_count=$(echo $Node | grep " Ready " | wc -l) - tag_count=$(echo $Node | grep $UPGRADE_KUBE_TAG | wc -l) + ready_count=$(echo $Node | grep -o " Ready " | wc -l) + tag_count=$(echo $Node | grep -o $UPGRADE_KUBE_TAG | wc -l) if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then echo "Kubernetes accessible with Ready node." echo "Node status: ${Node}" - break - else - echo "Currtny retry count: $i" + exit 0 + elif [[ $i -eq 30 ]]; then + echo "Failed: Unable to reach ready status for nodes..." echo "Node status: ${Node}" + exit 1 + else + echo "Current retry count: $i" + # echo "Node status: ${Node}" sleep 5 fi done From e59c5397d589e67c80642bb71980d7c9c7788a12 Mon Sep 17 00:00:00 2001 From: ricolin Date: Tue, 14 Jan 2025 12:45:51 +0800 Subject: [PATCH 13/13] rename zuul jobs --- hack/run-integration-tests.sh | 215 +++++++++--------- ...tempest-tests.sh => run-sonobuoy-tests.sh} | 215 +++++++++--------- zuul.d/jobs-ubuntu-2204.yaml | 70 +++--- zuul.d/jobs.yaml | 4 +- .../{tempest => integration}/run.yml | 2 +- zuul.d/playbooks/sonobuoy/run.yml | 2 +- zuul.d/project.yaml | 3 +- 7 files changed, 255 insertions(+), 256 deletions(-) rename hack/{run-tempest-tests.sh => run-sonobuoy-tests.sh} (58%) rename zuul.d/playbooks/{tempest => integration}/run.yml (97%) diff --git a/hack/run-integration-tests.sh b/hack/run-integration-tests.sh index 096cd5d0..d70dd70a 100755 --- a/hack/run-integration-tests.sh +++ b/hack/run-integration-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -xe -# Copyright (c) 2023 VEXXHOST, Inc. +# Copyright (c) 2024 VEXXHOST, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,18 +14,11 @@ # License for the specific language governing permissions and limitations # under the License. -# This script will run the full functional tests for a given `KUBE_TAG`. It -# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE` -# and then run `sonobuoy` against it. - -source /opt/stack/openrc +source /opt/stack/openrc admin admin OS_DISTRO=${OS_DISTRO:-ubuntu} IMAGE_OS=${IMAGE_OS:-ubuntu-2204} -NODE_COUNT=${NODE_COUNT:-2} NETWORK_DRIVER=${NETWORK_DRIVER:-calico} -SONOBUOY_VERSION=${SONOBUOY_VERSION:-0.56.16} -SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64} DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" @@ -42,6 +35,7 @@ fi # Upload image to Glance openstack image create \ --disk-format=qcow2 \ + --public \ --container-format=bare \ --property os_distro=${OS_DISTRO} \ --file=${IMAGE_NAME}.qcow2 \ @@ -56,12 +50,50 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then # Upload Upgrade image to Glance openstack image create \ --disk-format=qcow2 \ + --public \ --container-format=bare \ --property os_distro=${OS_DISTRO} \ --file=${UPGRADE_IMAGE_NAME}.qcow2 \ ${UPGRADE_IMAGE_NAME} fi +mkdir /tmp/magnum-nodes + +pushd /opt/stack/tempest +echo "Tempest configs:" + +cat <> /opt/stack/tempest/etc/tempest.conf + +[magnum] +flavor_id = m1.large +master_flavor_id = m1.large +copy_logs = true +network_driver = ${NETWORK_DRIVER} +image_id = ${IMAGE_OS}-kube-${KUBE_TAG} +coe = kubernetes +labels = '{"kube_tag": "${KUBE_TAG}", "fixed_subnet_cidr": "10.0.0.0/26"}' +docker_storage_driver = overlay + +EOF + +if [ ! -d /opt/stack/magnum-tempest-plugin ]; then + git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin +fi + +# install magnum-tempest-plugin +pushd /opt/stack/magnum-tempest-plugin +$HOME/.local/bin/pip3 install -e . +popd + +echo "Run Tempest against configs:" +cat /opt/stack/tempest/etc/tempest.conf + +echo "Run Tempest tests:" +/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ + --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' +popd + + # Create cluster template openstack coe cluster template create \ --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ @@ -77,41 +109,24 @@ openstack coe cluster template create \ --label fixed_subnet_cidr=192.168.24.0/24 \ k8s-${KUBE_TAG}; -if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then - # Create cluster template for upgrade - openstack coe cluster template create \ - --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ - --external-network public \ - --dns-nameserver ${DNS_NAMESERVER} \ - --master-lb-enabled \ - --master-flavor m1.large \ - --flavor m1.large \ - --network-driver ${NETWORK_DRIVER} \ - --docker-storage-driver overlay2 \ - --coe kubernetes \ - --label kube_tag=${UPGRADE_KUBE_TAG} \ - --label fixed_subnet_cidr=192.168.24.0/24 \ - k8s-${UPGRADE_KUBE_TAG}; -fi - # Create cluster openstack coe cluster create \ --cluster-template k8s-${KUBE_TAG} \ --master-count 1 \ - --node-count ${NODE_COUNT} \ + --node-count 1 \ --merge-labels \ --label audit_log_enabled=true \ - k8s-cluster + k8s-cluster-test # Wait for cluster creation to be queued set +e for i in {1..5}; do - openstack coe cluster show k8s-cluster 2>&1 + openstack coe cluster show k8s-cluster-test 2>&1 exit_status=$? if [ $exit_status -eq 0 ]; then break else - echo "Error: Cluster k8s-cluster could not be found." + echo "Error: Cluster k8s-cluster-test could not be found." sleep 1 fi done @@ -119,7 +134,7 @@ set -e # Wait for cluster to be "CREATE_COMPLETE". for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster -c status -f value) + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then echo "Cluster failed to create" exit 1 @@ -127,106 +142,88 @@ for i in {1..240}; do echo "Cluster created" break else - echo "Currtny retry count: $i" + echo "Current retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done -# Get the cluster configuration file -eval $(openstack coe cluster config k8s-cluster) - -# Download sonobuoy -curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz -tar -xzf sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz - -# Run sonobuoy -./sonobuoy run --wait --mode certified-conformance --plugin-env=e2e.E2E_PARALLEL=true - -# Retrieve results -RESULTS_FILE=$(./sonobuoy retrieve --filename sonobuoy-results.tar.gz) - -# Print results -./sonobuoy results ${RESULTS_FILE} - - -# Fail if the Sonobuoy tests failed -if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed"; then - echo "Sonobuoy tests failed" - exit 1 -fi +# Wait for kubernetes accessible and Ready. +for i in {1..30}; do + # Get the cluster configuration file + eval $(openstack coe cluster config k8s-cluster-test) + Node=$(kubectl get node -o wide) + ready_count=$(echo $Node | grep -o " Ready " | wc -l) + tag_count=$(echo $Node | grep -o $KUBE_TAG | wc -l) + if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then + echo "Kubernetes accessible with Ready node." + echo "Node status: ${Node}" + break + elif [[ $i -eq 30 ]]; then + echo "Failed: Unable to reach ready status for nodes..." + echo "Node status: ${Node}" + exit 1 + else + echo "Current retry count: $i" + # echo "Node status: ${Node}" + sleep 5 + fi +done if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then - openstack coe cluster delete k8s-cluster - # Wait for cluster to be deleted - set +e - for i in {1..60}; do - openstack coe cluster show k8s-cluster 2>&1 - exit_status=$? - if [ $exit_status -eq 0 ]; then - sleep 2 - else - echo "Cluster k8s-cluster deleted." - break - fi - done - set -e - # Create cluster - openstack coe cluster create \ - --cluster-template k8s-${KUBE_TAG} \ - --master-count 1 \ - --node-count 1 \ - --merge-labels \ - --label audit_log_enabled=true \ - k8s-cluster-upgrade - - # Wait for cluster creation to be queued - set +e - for i in {1..5}; do - openstack coe cluster show k8s-cluster-upgrade 2>&1 - exit_status=$? - if [ $exit_status -eq 0 ]; then - break - else - echo "Error: Cluster k8s-cluster-upgrade could not be found." - sleep 1 - fi - done - set -e + # Create cluster template for upgrade + openstack coe cluster template create \ + --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${UPGRADE_KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${UPGRADE_KUBE_TAG}; - # Wait for cluster to be "CREATE_COMPLETE". + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-test k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to create" + echo "Cluster failed to upgrade" exit 1 - elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then - echo "Cluster created" + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" break else - echo "Currtny retry count: $i" + echo "Current retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done - # Upgrade cluster - openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} - # Wait for cluster to be "UPDATE_COMPLETE". - for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) - if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to upgrade" - exit 1 - elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then - echo "Cluster upgraded" + # Wait for kubernetes accessible and Ready. + for i in {1..30}; do + # Get the cluster configuration file + eval $(openstack coe cluster config k8s-cluster-test) + Node=$(kubectl get node -o wide) + ready_count=$(echo $Node | grep -o " Ready " | wc -l) + tag_count=$(echo $Node | grep -o $UPGRADE_KUBE_TAG | wc -l) + if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then + echo "Kubernetes accessible with Ready node." + echo "Node status: ${Node}" exit 0 - break + elif [[ $i -eq 30 ]]; then + echo "Failed: Unable to reach ready status for nodes..." + echo "Node status: ${Node}" + exit 1 else - echo "Currtny retry count: $i" - echo "Cluster status: ${CLUSTER_STATUS}" + echo "Current retry count: $i" + # echo "Node status: ${Node}" sleep 5 fi done diff --git a/hack/run-tempest-tests.sh b/hack/run-sonobuoy-tests.sh similarity index 58% rename from hack/run-tempest-tests.sh rename to hack/run-sonobuoy-tests.sh index d70dd70a..096cd5d0 100755 --- a/hack/run-tempest-tests.sh +++ b/hack/run-sonobuoy-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -xe -# Copyright (c) 2024 VEXXHOST, Inc. +# Copyright (c) 2023 VEXXHOST, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,11 +14,18 @@ # License for the specific language governing permissions and limitations # under the License. -source /opt/stack/openrc admin admin +# This script will run the full functional tests for a given `KUBE_TAG`. It +# will download the image, create a cluster, wait for it to hit `CREATE_COMPLETE` +# and then run `sonobuoy` against it. + +source /opt/stack/openrc OS_DISTRO=${OS_DISTRO:-ubuntu} IMAGE_OS=${IMAGE_OS:-ubuntu-2204} +NODE_COUNT=${NODE_COUNT:-2} NETWORK_DRIVER=${NETWORK_DRIVER:-calico} +SONOBUOY_VERSION=${SONOBUOY_VERSION:-0.56.16} +SONOBUOY_ARCH=${SONOBUOY_ARCH:-amd64} DNS_NAMESERVER=${DNS_NAMESERVER:-1.1.1.1} UPGRADE_KUBE_TAG=${UPGRADE_KUBE_TAG:-KUBE_TAG} IMAGE_NAME="${IMAGE_OS}-kube-${KUBE_TAG}" @@ -35,7 +42,6 @@ fi # Upload image to Glance openstack image create \ --disk-format=qcow2 \ - --public \ --container-format=bare \ --property os_distro=${OS_DISTRO} \ --file=${IMAGE_NAME}.qcow2 \ @@ -50,50 +56,12 @@ if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then # Upload Upgrade image to Glance openstack image create \ --disk-format=qcow2 \ - --public \ --container-format=bare \ --property os_distro=${OS_DISTRO} \ --file=${UPGRADE_IMAGE_NAME}.qcow2 \ ${UPGRADE_IMAGE_NAME} fi -mkdir /tmp/magnum-nodes - -pushd /opt/stack/tempest -echo "Tempest configs:" - -cat <> /opt/stack/tempest/etc/tempest.conf - -[magnum] -flavor_id = m1.large -master_flavor_id = m1.large -copy_logs = true -network_driver = ${NETWORK_DRIVER} -image_id = ${IMAGE_OS}-kube-${KUBE_TAG} -coe = kubernetes -labels = '{"kube_tag": "${KUBE_TAG}", "fixed_subnet_cidr": "10.0.0.0/26"}' -docker_storage_driver = overlay - -EOF - -if [ ! -d /opt/stack/magnum-tempest-plugin ]; then - git clone https://github.com/openstack/magnum-tempest-plugin /opt/stack/magnum-tempest-plugin -fi - -# install magnum-tempest-plugin -pushd /opt/stack/magnum-tempest-plugin -$HOME/.local/bin/pip3 install -e . -popd - -echo "Run Tempest against configs:" -cat /opt/stack/tempest/etc/tempest.conf - -echo "Run Tempest tests:" -/opt/stack/data/venv/bin/tempest run -r '(^magnum_tempest_plugin)' \ - --exclude-regex '^magnum_tempest_plugin.tests.api.v1.test_cluster.ClusterTest\.(test_create_cluster_with_zero_nodes|test_create_list_sign_delete_clusters)' -popd - - # Create cluster template openstack coe cluster template create \ --image $(openstack image show ${IMAGE_NAME} -c id -f value) \ @@ -109,24 +77,41 @@ openstack coe cluster template create \ --label fixed_subnet_cidr=192.168.24.0/24 \ k8s-${KUBE_TAG}; +if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then + # Create cluster template for upgrade + openstack coe cluster template create \ + --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ + --external-network public \ + --dns-nameserver ${DNS_NAMESERVER} \ + --master-lb-enabled \ + --master-flavor m1.large \ + --flavor m1.large \ + --network-driver ${NETWORK_DRIVER} \ + --docker-storage-driver overlay2 \ + --coe kubernetes \ + --label kube_tag=${UPGRADE_KUBE_TAG} \ + --label fixed_subnet_cidr=192.168.24.0/24 \ + k8s-${UPGRADE_KUBE_TAG}; +fi + # Create cluster openstack coe cluster create \ --cluster-template k8s-${KUBE_TAG} \ --master-count 1 \ - --node-count 1 \ + --node-count ${NODE_COUNT} \ --merge-labels \ --label audit_log_enabled=true \ - k8s-cluster-test + k8s-cluster # Wait for cluster creation to be queued set +e for i in {1..5}; do - openstack coe cluster show k8s-cluster-test 2>&1 + openstack coe cluster show k8s-cluster 2>&1 exit_status=$? if [ $exit_status -eq 0 ]; then break else - echo "Error: Cluster k8s-cluster-test could not be found." + echo "Error: Cluster k8s-cluster could not be found." sleep 1 fi done @@ -134,7 +119,7 @@ set -e # Wait for cluster to be "CREATE_COMPLETE". for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster -c status -f value) if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then echo "Cluster failed to create" exit 1 @@ -142,88 +127,106 @@ for i in {1..240}; do echo "Cluster created" break else - echo "Current retry count: $i" + echo "Currtny retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done -# Wait for kubernetes accessible and Ready. -for i in {1..30}; do - # Get the cluster configuration file - eval $(openstack coe cluster config k8s-cluster-test) - Node=$(kubectl get node -o wide) - ready_count=$(echo $Node | grep -o " Ready " | wc -l) - tag_count=$(echo $Node | grep -o $KUBE_TAG | wc -l) - if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then - echo "Kubernetes accessible with Ready node." - echo "Node status: ${Node}" - break - elif [[ $i -eq 30 ]]; then - echo "Failed: Unable to reach ready status for nodes..." - echo "Node status: ${Node}" - exit 1 - else - echo "Current retry count: $i" - # echo "Node status: ${Node}" - sleep 5 - fi -done +# Get the cluster configuration file +eval $(openstack coe cluster config k8s-cluster) + +# Download sonobuoy +curl -LO https://github.com/vmware-tanzu/sonobuoy/releases/download/v${SONOBUOY_VERSION}/sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz +tar -xzf sonobuoy_${SONOBUOY_VERSION}_linux_${SONOBUOY_ARCH}.tar.gz + +# Run sonobuoy +./sonobuoy run --wait --mode certified-conformance --plugin-env=e2e.E2E_PARALLEL=true + +# Retrieve results +RESULTS_FILE=$(./sonobuoy retrieve --filename sonobuoy-results.tar.gz) + +# Print results +./sonobuoy results ${RESULTS_FILE} + + +# Fail if the Sonobuoy tests failed +if ! ./sonobuoy results --plugin e2e ${RESULTS_FILE} | grep -q "Status: passed"; then + echo "Sonobuoy tests failed" + exit 1 +fi if [[ ${UPGRADE_KUBE_TAG} != ${KUBE_TAG} ]]; then - # Create cluster template for upgrade - openstack coe cluster template create \ - --image $(openstack image show ${UPGRADE_IMAGE_NAME} -c id -f value) \ - --external-network public \ - --dns-nameserver ${DNS_NAMESERVER} \ - --master-lb-enabled \ - --master-flavor m1.large \ - --flavor m1.large \ - --network-driver ${NETWORK_DRIVER} \ - --docker-storage-driver overlay2 \ - --coe kubernetes \ - --label kube_tag=${UPGRADE_KUBE_TAG} \ - --label fixed_subnet_cidr=192.168.24.0/24 \ - k8s-${UPGRADE_KUBE_TAG}; + openstack coe cluster delete k8s-cluster + # Wait for cluster to be deleted + set +e + for i in {1..60}; do + openstack coe cluster show k8s-cluster 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + sleep 2 + else + echo "Cluster k8s-cluster deleted." + break + fi + done + set -e + # Create cluster + openstack coe cluster create \ + --cluster-template k8s-${KUBE_TAG} \ + --master-count 1 \ + --node-count 1 \ + --merge-labels \ + --label audit_log_enabled=true \ + k8s-cluster-upgrade + + # Wait for cluster creation to be queued + set +e + for i in {1..5}; do + openstack coe cluster show k8s-cluster-upgrade 2>&1 + exit_status=$? + if [ $exit_status -eq 0 ]; then + break + else + echo "Error: Cluster k8s-cluster-upgrade could not be found." + sleep 1 + fi + done + set -e - # Upgrade cluster - openstack coe cluster upgrade k8s-cluster-test k8s-${UPGRADE_KUBE_TAG} - # Wait for cluster to be "UPDATE_COMPLETE". + # Wait for cluster to be "CREATE_COMPLETE". for i in {1..240}; do - CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-test -c status -f value) + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then - echo "Cluster failed to upgrade" + echo "Cluster failed to create" exit 1 - elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then - echo "Cluster upgraded" + elif [[ ${CLUSTER_STATUS} == *"CREATE_COMPLETE"* ]]; then + echo "Cluster created" break else - echo "Current retry count: $i" + echo "Currtny retry count: $i" echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done - # Wait for kubernetes accessible and Ready. - for i in {1..30}; do - # Get the cluster configuration file - eval $(openstack coe cluster config k8s-cluster-test) - Node=$(kubectl get node -o wide) - ready_count=$(echo $Node | grep -o " Ready " | wc -l) - tag_count=$(echo $Node | grep -o $UPGRADE_KUBE_TAG | wc -l) - if [[ $ready_count -eq 2 && $tag_count -eq 2 ]]; then - echo "Kubernetes accessible with Ready node." - echo "Node status: ${Node}" - exit 0 - elif [[ $i -eq 30 ]]; then - echo "Failed: Unable to reach ready status for nodes..." - echo "Node status: ${Node}" + # Upgrade cluster + openstack coe cluster upgrade k8s-cluster-upgrade k8s-${UPGRADE_KUBE_TAG} + # Wait for cluster to be "UPDATE_COMPLETE". + for i in {1..240}; do + CLUSTER_STATUS=$(openstack coe cluster show k8s-cluster-upgrade -c status -f value) + if [[ ${CLUSTER_STATUS} == *"FAILED"* ]]; then + echo "Cluster failed to upgrade" exit 1 + elif [[ ${CLUSTER_STATUS} == *"UPDATE_COMPLETE"* ]]; then + echo "Cluster upgraded" + exit 0 + break else - echo "Current retry count: $i" - # echo "Node status: ${Node}" + echo "Currtny retry count: $i" + echo "Cluster status: ${CLUSTER_STATUS}" sleep 5 fi done diff --git a/zuul.d/jobs-ubuntu-2204.yaml b/zuul.d/jobs-ubuntu-2204.yaml index 746f6647..c65bdd0f 100644 --- a/zuul.d/jobs-ubuntu-2204.yaml +++ b/zuul.d/jobs-ubuntu-2204.yaml @@ -103,8 +103,8 @@ - magnum-cluster-api-sonobuoy-ubuntu-2204-v1.31.1-cilium - job: - name: magnum-cluster-api-tempest-ubuntu-2204 - parent: magnum-cluster-api-tempest + name: magnum-cluster-api-integration-ubuntu-2204 + parent: magnum-cluster-api-integration timeout: 10800 abstract: true vars: @@ -112,8 +112,8 @@ image_os_distro: ubuntu - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 - parent: magnum-cluster-api-tempest-ubuntu-2204 + name: magnum-cluster-api-integration-ubuntu-2204-v1.28.11 + parent: magnum-cluster-api-integration-ubuntu-2204 vars: kube_tag: v1.28.11 upgrade_kube_tag: v1.29.6 @@ -121,20 +121,20 @@ upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-29-6-1720107687.qcow2 - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + name: magnum-cluster-api-integration-ubuntu-2204-v1.28.11-calico + parent: magnum-cluster-api-integration-ubuntu-2204-v1.28.11 vars: network_driver: calico - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.28.11 + name: magnum-cluster-api-integration-ubuntu-2204-v1.28.11-cilium + parent: magnum-cluster-api-integration-ubuntu-2204-v1.28.11 vars: network_driver: cilium - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 - parent: magnum-cluster-api-tempest-ubuntu-2204 + name: magnum-cluster-api-integration-ubuntu-2204-v1.29.6 + parent: magnum-cluster-api-integration-ubuntu-2204 vars: kube_tag: v1.29.6 upgrade_kube_tag: v1.30.2 @@ -142,20 +142,20 @@ upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-30-2-1720107688.qcow2 - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + name: magnum-cluster-api-integration-ubuntu-2204-v1.29.6-calico + parent: magnum-cluster-api-integration-ubuntu-2204-v1.29.6 vars: network_driver: calico - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.29.6 + name: magnum-cluster-api-integration-ubuntu-2204-v1.29.6-cilium + parent: magnum-cluster-api-integration-ubuntu-2204-v1.29.6 vars: network_driver: cilium - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 - parent: magnum-cluster-api-tempest-ubuntu-2204 + name: magnum-cluster-api-integration-ubuntu-2204-v1.30.2 + parent: magnum-cluster-api-integration-ubuntu-2204 vars: kube_tag: v1.30.2 upgrade_kube_tag: v1.31.1 @@ -163,45 +163,45 @@ upgrade_image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + name: magnum-cluster-api-integration-ubuntu-2204-v1.30.2-calico + parent: magnum-cluster-api-integration-ubuntu-2204-v1.30.2 vars: network_driver: calico - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.30.2 + name: magnum-cluster-api-integration-ubuntu-2204-v1.30.2-cilium + parent: magnum-cluster-api-integration-ubuntu-2204-v1.30.2 vars: network_driver: cilium - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 - parent: magnum-cluster-api-tempest-ubuntu-2204 + name: magnum-cluster-api-integration-ubuntu-2204-v1.31.1 + parent: magnum-cluster-api-integration-ubuntu-2204 vars: kube_tag: v1.31.1 image_url: https://static.atmosphere.dev/artifacts/magnum-cluster-api/ubuntu-jammy-kubernetes-1-31-1-1728920853.qcow2 - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + name: magnum-cluster-api-integration-ubuntu-2204-v1.31.1-calico + parent: magnum-cluster-api-integration-ubuntu-2204-v1.31.1 vars: network_driver: calico - job: - name: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium - parent: magnum-cluster-api-tempest-ubuntu-2204-v1.31.1 + name: magnum-cluster-api-integration-ubuntu-2204-v1.31.1-cilium + parent: magnum-cluster-api-integration-ubuntu-2204-v1.31.1 vars: network_driver: cilium - project-template: - name: magnum-cluster-api-tempest-ubuntu-2204 + name: magnum-cluster-api-integration-ubuntu-2204 check: jobs: - - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-calico - - magnum-cluster-api-tempest-ubuntu-2204-v1.28.11-cilium - - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-calico - - magnum-cluster-api-tempest-ubuntu-2204-v1.29.6-cilium - - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-calico - - magnum-cluster-api-tempest-ubuntu-2204-v1.30.2-cilium - - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-calico - - magnum-cluster-api-tempest-ubuntu-2204-v1.31.1-cilium + - magnum-cluster-api-integration-ubuntu-2204-v1.28.11-calico + - magnum-cluster-api-integration-ubuntu-2204-v1.28.11-cilium + - magnum-cluster-api-integration-ubuntu-2204-v1.29.6-calico + - magnum-cluster-api-integration-ubuntu-2204-v1.29.6-cilium + - magnum-cluster-api-integration-ubuntu-2204-v1.30.2-calico + - magnum-cluster-api-integration-ubuntu-2204-v1.30.2-cilium + - magnum-cluster-api-integration-ubuntu-2204-v1.31.1-calico + - magnum-cluster-api-integration-ubuntu-2204-v1.31.1-cilium diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 2be8a343..2e356a63 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -31,10 +31,10 @@ - magnum_cluster_api/cmd/image_builder.py - job: - name: magnum-cluster-api-tempest + name: magnum-cluster-api-integration abstract: true timeout: 7200 - run: zuul.d/playbooks/tempest/run.yml + run: zuul.d/playbooks/integration/run.yml nodeset: nodes: - name: ubuntu-jammy diff --git a/zuul.d/playbooks/tempest/run.yml b/zuul.d/playbooks/integration/run.yml similarity index 97% rename from zuul.d/playbooks/tempest/run.yml rename to zuul.d/playbooks/integration/run.yml index 61d407cf..1f5431e4 100644 --- a/zuul.d/playbooks/tempest/run.yml +++ b/zuul.d/playbooks/integration/run.yml @@ -52,7 +52,7 @@ KUBE_TAG: "{{ kube_tag }}" NETWORK_DRIVER: "{{ network_driver }}" - - shell: "./hack/run-tempest-tests.sh" + - shell: "./hack/run-integration-tests.sh" args: chdir: "{{ zuul.project.src_dir }}" environment: diff --git a/zuul.d/playbooks/sonobuoy/run.yml b/zuul.d/playbooks/sonobuoy/run.yml index 6e9da177..be5a0248 100644 --- a/zuul.d/playbooks/sonobuoy/run.yml +++ b/zuul.d/playbooks/sonobuoy/run.yml @@ -50,7 +50,7 @@ args: chdir: "{{ zuul.project.src_dir }}" - - shell: "./hack/run-integration-tests.sh" + - shell: "./hack/run-sonobuoy-tests.sh" args: chdir: "{{ zuul.project.src_dir }}" environment: diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 04d0bb82..308ebc27 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -9,5 +9,4 @@ - magnum-cluster-api-flatcar - magnum-cluster-api-rockylinux-8 - magnum-cluster-api-rockylinux-9 - - magnum-cluster-api-tempest-ubuntu-2204 - #- magnum-cluster-api-sonobuoy-ubuntu-2204 + - magnum-cluster-api-integration-ubuntu-2204