Merge pull request #11530 from VannTen/ci/cleanup_with_k8s_gc

[CI] Use Kubernetes GC to clean kubevirt VMs (packet-* jobs)

.gitlab-ci.yml

@@ -11,24 +11,19 @@ variables:
   GITLAB_REPOSITORY: 'kargo-ci/kubernetes-sigs-kubespray'
   ANSIBLE_FORCE_COLOR: "true"
   MAGIC: "ci check this"
-  TEST_ID: "$CI_PIPELINE_ID-$CI_JOB_ID"
-  CI_TEST_VARS: "./tests/files/${CI_JOB_NAME}.yml"
-  CI_TEST_REGISTRY_MIRROR: "./tests/common/_docker_hub_registry_mirror.yml"
-  CI_TEST_SETTING: "./tests/common/_kubespray_test_settings.yml"
   GS_ACCESS_KEY_ID: $GS_KEY
   GS_SECRET_ACCESS_KEY: $GS_SECRET
   CONTAINER_ENGINE: docker
   SSH_USER: root
   GCE_PREEMPTIBLE: "false"
   ANSIBLE_KEEP_REMOTE_FILES: "1"
   ANSIBLE_CONFIG: ./tests/ansible.cfg
-  ANSIBLE_INVENTORY: ./inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini
   IDEMPOT_CHECK: "false"
   RESET_CHECK: "false"
   REMOVE_NODE_CHECK: "false"
   UPGRADE_TEST: "false"
   MITOGEN_ENABLE: "false"
-  ANSIBLE_LOG_LEVEL: "-vv"
+  ANSIBLE_VERBOSITY: 2
   RECOVER_CONTROL_PLANE_TEST: "false"
   RECOVER_CONTROL_PLANE_TEST_GROUPS: "etcd[2:]:kube_control_plane[1:]"
   TERRAFORM_VERSION: 1.3.7

.gitlab-ci/packet.yml

@@ -65,14 +65,6 @@
   allow_failure: true
   extends: .packet
 
-packet_cleanup_old:
-  stage: deploy-part1
-  extends: .packet_periodic
-  script:
-    - cd tests
-    - make cleanup-packet
-  after_script: []
-
 # The ubuntu20-calico-all-in-one jobs are meant as early stages to prevent running the full CI if something is horribly broken
 packet_ubuntu20-calico-all-in-one:
   stage: deploy-part1

tests/Makefile

@@ -1,5 +1,3 @@
-INVENTORY=$(PWD)/../inventory/sample/${CI_JOB_NAME}-${BUILD_NUMBER}.ini
-
 init-packet:
 	mkdir -p $(HOME)/.ssh
 	echo $(PACKET_VM_SSH_PRIVATE_KEY) | base64 -d > $(HOME)/.ssh/id_rsa
@@ -13,30 +11,17 @@ delete-tf:
 
 create-packet: init-packet
 	ansible-playbook cloud_playbooks/create-packet.yml -c local \
-	$(ANSIBLE_LOG_LEVEL) \
-	-e @"files/${CI_JOB_NAME}.yml" \
-	-e test_id=$(TEST_ID) \
-	-e branch="$(CI_COMMIT_BRANCH)" \
-	-e pipeline_id="$(CI_PIPELINE_ID)" \
-	-e inventory_path=$(INVENTORY)
-
-delete-packet:
-	ansible-playbook cloud_playbooks/delete-packet.yml -c local \
-	$(ANSIBLE_LOG_LEVEL) \
 	-e @"files/${CI_JOB_NAME}.yml" \
-	-e test_id=$(TEST_ID) \
+	-e test_name="$(subst .,-,$(CI_PIPELINE_ID)-$(CI_JOB_ID))" \
 	-e branch="$(CI_COMMIT_BRANCH)" \
 	-e pipeline_id="$(CI_PIPELINE_ID)" \
-	-e inventory_path=$(INVENTORY)
+	-e inventory_path=$(INVENTORY_DIR)
 
-cleanup-packet:
-	ansible-playbook cloud_playbooks/cleanup-packet.yml -c local \
-	$(ANSIBLE_LOG_LEVEL)
+delete-packet: ;
 
 create-vagrant:
 	vagrant up
-	cp  $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $(INVENTORY)
-
+	cp $(CI_PROJECT_DIR)/.vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory $(INVENTORY_DIR)
 
 delete-vagrant:
 	vagrant destroy -f

tests/cloud_playbooks/create-packet.yml

@@ -4,8 +4,5 @@
   hosts: localhost
   gather_facts: false
   become: true
-  vars:
-    ci_job_name: "{{ lookup('env', 'CI_JOB_NAME') }}"
-    test_name: "{{ test_id | regex_replace('\\.', '-') }}"
   roles:
     - { role: packet-ci, vm_cleanup: false }

tests/cloud_playbooks/roles/packet-ci/tasks/main.yml

@@ -1,17 +1,52 @@
 ---
-
-- name: "Include custom vars for ci job: {{ ci_job_name }}"
+- name: Include custom vars for ci job
   include_vars: "../files/{{ ci_job_name }}.yml"
 
-- name: Cleamup old VMs
-  import_tasks: cleanup-old-vms.yml
+- name: Start vms for CI job
+  vars:
+    # Workaround for compatibility when testing upgrades with old == before e9d406ed088d4291ef1d9018c170a4deed2bf928
+    # TODO: drop after 2.27.0
+    legacy_groups: "{{ (['kube_control_plane', 'kube_node', 'calico_rr'] | intersect(item) | length > 0) | ternary(['k8s_cluster'], []) }}"
+    tvars:
+      kubespray_groups: "{{ item + legacy_groups }}"
+  kubernetes.core.k8s:
+    definition: "{{ lookup('template', 'vm.yml.j2', template_vars=tvars) }}"
+  loop: "{{ scenarios[mode | d('default')] }}"
+
+- name: Wait for vms to have IP addresses
+  kubernetes.core.k8s_info:
+    api_version: kubevirt.io/v1
+    kind: VirtualMachineInstance
+    label_selectors:
+      - "ci_job_id={{ ci_job_id }}"
+    namespace: "{{ pod_namespace }}"
+  register: vmis
+  until: vmis.resources
+          | map(attribute='status.interfaces.0')
+          | rejectattr('ipAddress', 'defined') == []
+  retries: 30
+  delay: 10
 
-- name: Create VMs
-  import_tasks: create-vms.yml
-  when:
-    - not vm_cleanup
+- name: Massage VirtualMachineInstance data into an Ansible inventory structure
+  vars:
+    ips: "{{ vmis.resources | map(attribute='status.interfaces.0.ipAddress') }}"
+    names: "{{ vmis.resources | map(attribute='metadata.name') }}"
+    _groups: "{{ vmis.resources | map(attribute='metadata.annotations.ansible_groups') | map('split', ',') }}"
+    hosts: "{{ ips | zip(_groups, names)
+                | map('zip', ['ansible_host', 'ansible_groups', 'k8s_vmi_name'])
+                | map('map', 'reverse') | map('community.general.dict') }}"
+  loop: "{{ hosts | map(attribute='ansible_groups') | flatten | unique }}"
+  set_fact:
+    ci_inventory: "{{ ci_inventory|d({}) | combine({
+                    item: {
+                      'hosts': hosts | selectattr('ansible_groups', 'contains', item)
+                                     | rekey_on_member('k8s_vmi_name')
+                      }
+                    })
+                  }}"
 
-- name: Delete VMs
-  import_tasks: delete-vms.yml
-  when:
-    - vm_cleanup | default(false)
+- name: Create inventory for CI tests
+  copy:
+    content: "{{ ci_inventory | to_yaml }}"
+    dest: "{{ inventory_path }}/ci_inventory.yml"
+    mode: "0644"