diff --git a/chaoslib/litmus/container_kill/containerd_chaos/containerd.j2 b/chaoslib/litmus/container_kill/containerd_chaos/containerd.j2 index 38522a74fe1..2dd099001f6 100644 --- a/chaoslib/litmus/container_kill/containerd_chaos/containerd.j2 +++ b/chaoslib/litmus/container_kill/containerd_chaos/containerd.j2 @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: containerd-chaos + labels: + chaosUID: {{ chaosUID }} spec: selector: matchLabels: @@ -30,4 +32,4 @@ spec: name: cri-socket - hostPath: path: /etc/crictl.yml - name: cri-config \ No newline at end of file + name: cri-config diff --git a/chaoslib/litmus/cpu_hog/app_cpu_stress.j2 b/chaoslib/litmus/cpu_hog/app_cpu_stress.j2 index af5d1a2f5aa..4132bede196 100644 --- a/chaoslib/litmus/cpu_hog/app_cpu_stress.j2 +++ b/chaoslib/litmus/cpu_hog/app_cpu_stress.j2 @@ -2,6 +2,8 @@ apiVersion: batch/v1 kind: Job metadata: name: app-cpu-stress-{{ run_id }} + labels: + chaosUID: {{ chaosUID }} spec: template: metadata: @@ -43,4 +45,4 @@ spec: volumes: - hostPath: path: /var/run/docker.sock - name: dockersocket \ No newline at end of file + name: dockersocket diff --git a/chaoslib/litmus/disk_fill/disk_fill_ds.j2 b/chaoslib/litmus/disk_fill/disk_fill_ds.j2 index 3e416178cb4..1f81706ddec 100644 --- a/chaoslib/litmus/disk_fill/disk_fill_ds.j2 +++ b/chaoslib/litmus/disk_fill/disk_fill_ds.j2 @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: disk-fill + labels: + chaosUID: {{ chaosUID }} spec: selector: matchLabels: @@ -30,4 +32,4 @@ spec: volumes: - name: udev hostPath: - path: /var/lib/docker/containers \ No newline at end of file + path: /var/lib/docker/containers diff --git a/chaoslib/litmus/kill_random_pod.yml b/chaoslib/litmus/kill_random_pod.yml index 651e5680a84..f0a1ecbc949 100644 --- a/chaoslib/litmus/kill_random_pod.yml +++ b/chaoslib/litmus/kill_random_pod.yml @@ -1,52 +1,53 @@ - block: - - name: Get a list of all pods from given namespace - k8s_facts: - kind: Pod - namespace: "{{ app_ns }}" - label_selectors: - - "{{ app_label }}" - register: pod_list - - - name: Initialize deletion list - set_fact: - target_pod_list: [] - - - block: - - - name: Select a random pod to kill - set_fact: - app_pod_random: "{{ pod_list.resources | random | json_query('metadata.name') }}" - - - name: Construct the deletion list with single random pod - set_fact: - target_pod_list: "{{ target_pod_list + [app_pod_random] }}" + - name: Get a list of all pods from given namespace + k8s_facts: + kind: Pod + namespace: "{{ app_ns }}" + label_selectors: + - "{{ app_label }}" + register: pod_list + + - name: Initialize deletion list + set_fact: + target_pod_list: [] + + - block: + + - name: Select a random pod to kill + set_fact: + app_pod_random: "{{ pod_list.resources | random | json_query('metadata.name') }}" + + - name: Construct the deletion list with single random pod + set_fact: + target_pod_list: "{{ target_pod_list + [app_pod_random] }}" - - set_fact: - app_pod: "{{ target_pod_list | join(' ') }}" + - set_fact: + app_pod: "{{ target_pod_list | join(' ') }}" - when: "kill_count is not defined or kill_count == ''" + when: "kill_count is not defined or kill_count == ''" - - block: + - block: - - name: Add elements into target pod group list - set_fact: - target_pod_list: "{{ target_pod_list + [resource | json_query('metadata.name')] }}" - loop: "{{ pod_list.resources }}" - loop_control: - loop_var: resource + - name: Add elements into target pod group list + set_fact: + target_pod_list: "{{ target_pod_list + [resource | json_query('metadata.name')] }}" + loop: "{{ pod_list.resources }}" + loop_control: + loop_var: resource - - set_fact: - app_pod: "{{ target_pod_list[0:(kill_count|int)] | join(' ') }}" + - set_fact: + app_pod: "{{ target_pod_list[0:(kill_count|int)] | join(' ') }}" - when: "kill_count is defined and kill_count != ''" + when: "kill_count is defined and kill_count != ''" when: "app_pod_name is not defined or app_pod_name == ''" - block: - - name: Record app pod - set_fact: - app_pod: "{{ app_pod_name }}" + + - name: Record app pod + set_fact: + app_pod: "{{ app_pod_name }}" when: "app_pod_name is defined and app_pod_name != ''" diff --git a/chaoslib/litmus/platform/gke/node-cpu-hog-daemonset.j2 b/chaoslib/litmus/platform/gke/node-cpu-hog-daemonset.j2 index d86820a4184..a0fa02437de 100644 --- a/chaoslib/litmus/platform/gke/node-cpu-hog-daemonset.j2 +++ b/chaoslib/litmus/platform/gke/node-cpu-hog-daemonset.j2 @@ -2,6 +2,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: node-cpu-hog + labels: + chaosUID: {{ chaosUID }} spec: selector: matchLabels: @@ -19,4 +21,4 @@ spec: imagePullPolicy: Always name: cpu-hog command: ["/bin/bash"] - args: ["-c", "sleep 10000"] \ No newline at end of file + args: ["-c", "sleep 10000"] diff --git a/chaoslib/powerfulseal/powerfulseal.j2 b/chaoslib/powerfulseal/powerfulseal.j2 index b68bc9e33bc..3b0ed83d69b 100644 --- a/chaoslib/powerfulseal/powerfulseal.j2 +++ b/chaoslib/powerfulseal/powerfulseal.j2 @@ -27,6 +27,8 @@ apiVersion: apps/v1 kind: Deployment metadata: name: powerfulseal + labels: + chaosUID: {{ chaosUID }} spec: selector: matchLabels: @@ -57,4 +59,4 @@ spec: volumes: - name: policyfile configMap: - name: policy \ No newline at end of file + name: policy diff --git a/chaoslib/pumba/network_chaos/pumba_netem_job.j2 b/chaoslib/pumba/network_chaos/pumba_netem_job.j2 index d8068f46142..c780a8d595b 100644 --- a/chaoslib/pumba/network_chaos/pumba_netem_job.j2 +++ b/chaoslib/pumba/network_chaos/pumba_netem_job.j2 @@ -2,6 +2,8 @@ apiVersion: batch/v1 kind: Job metadata: name: pumba-netem-{{ run_id }} + labels: + chaosUID: {{ chaosUID }} spec: template: metadata: @@ -52,4 +54,4 @@ spec: volumes: - hostPath: path: /var/run/docker.sock - name: dockersocket \ No newline at end of file + name: dockersocket diff --git a/chaoslib/pumba/pumba.j2 b/chaoslib/pumba/pumba.j2 index e1105b11db3..1c4b0dcae91 100644 --- a/chaoslib/pumba/pumba.j2 +++ b/chaoslib/pumba/pumba.j2 @@ -3,6 +3,8 @@ apiVersion: apps/v1 kind: DaemonSet metadata: name: pumba + labels: + chaosUID: {{ chaosUID }} spec: selector: matchLabels: @@ -36,4 +38,4 @@ spec: volumes: - hostPath: path: /var/run/docker.sock - name: dockersocket \ No newline at end of file + name: dockersocket diff --git a/experiments/openebs/openebs-control-plane-pod-delete/chaosutil.j2 b/experiments/openebs/openebs-control-plane-validation/chaosutil.j2 similarity index 100% rename from experiments/openebs/openebs-control-plane-pod-delete/chaosutil.j2 rename to experiments/openebs/openebs-control-plane-validation/chaosutil.j2 diff --git a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_logic.yml b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_logic.yml similarity index 90% rename from experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_logic.yml rename to experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_logic.yml index d2d0c493c2c..3c3c3a9d766 100644 --- a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_logic.yml +++ b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_logic.yml @@ -2,7 +2,7 @@ connection: local vars: - c_experiment: "openebs-control-plane-pod-delete" + c_experiment: "openebs-control-plane-validation" c_duration: "{{ lookup('env','TOTAL_CHAOS_DURATION') }}" c_interval: "{{ lookup('env','CHAOS_INTERVAL') }}" ramp_time: "{{ lookup('env','RAMP_TIME') }}" @@ -14,7 +14,7 @@ tasks: - block: - - include: openebs_control_plane_pod_delete_ansible_prerequisites.yml + - include: openebs_control_plane_validation_ansible_prerequisites.yml - include_vars: file: chaosutil.yml @@ -41,7 +41,7 @@ ## INJECT CHAOS ON CONTROL PLANE COMPONENTS - name: Including main components - include_tasks: openebs_control_plane_pod_delete_internal_tasks.yml + include_tasks: openebs_control_plane_validation_internal_tasks.yml loop: "{{ openebs.components }}" - set_fact: diff --git a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_prerequisites.yml b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_prerequisites.yml similarity index 100% rename from experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_prerequisites.yml rename to experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_prerequisites.yml diff --git a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_internal_tasks.yml b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_internal_tasks.yml similarity index 100% rename from experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_internal_tasks.yml rename to experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_internal_tasks.yml diff --git a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_k8s_job.yml b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_k8s_job.yml similarity index 88% rename from experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_k8s_job.yml rename to experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_k8s_job.yml index ceda4c1a3c8..a5b0124bc5c 100644 --- a/experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_k8s_job.yml +++ b/experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_k8s_job.yml @@ -2,12 +2,12 @@ apiVersion: batch/v1 kind: Job metadata: - generateName: openebs-control-plane-pod-delete- + generateName: openebs-control-plane-validation- spec: template: metadata: labels: - experiment: openebs-control-plane-pod-delete + experiment: openebs-control-plane-validation spec: # Placeholder that is updated by the executor for automated runs # Provide appropriate SA (with desired permissions) if executed manually @@ -46,4 +46,4 @@ spec: fieldPath: spec.serviceAccountName command: ["/bin/bash"] - args: ["-c", "ansible-playbook ./experiments/openebs/openebs-control-plane-pod-delete/openebs_control_plane_pod_delete_ansible_logic.yml -i /etc/ansible/hosts -vv; exit 0"] \ No newline at end of file + args: ["-c", "ansible-playbook ./experiments/openebs/openebs-control-plane-validation/openebs_control_plane_validation_ansible_logic.yml -i /etc/ansible/hosts -vv; exit 0"] \ No newline at end of file diff --git a/experiments/openebs/openebs-pool-pod-failure/chaosutil.j2 b/experiments/openebs/openebs-pool-pod-failure/chaosutil.j2 new file mode 100644 index 00000000000..abc55ed57c9 --- /dev/null +++ b/experiments/openebs/openebs-pool-pod-failure/chaosutil.j2 @@ -0,0 +1,5 @@ +{% if a_pvc is defined and a_pvc != '' %} +c_util: /experiments/openebs/openebs-pool-pod-failure/cstor_pool_delete.yml +{% else %} +c_util: /chaoslib/litmus/kill_random_pod.yml +{% endif %} \ No newline at end of file diff --git a/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_ansible_logic.yml b/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_ansible_logic.yml index 4284fb6d157..bfbdf7a6758 100644 --- a/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_ansible_logic.yml +++ b/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_ansible_logic.yml @@ -24,27 +24,34 @@ tasks: - block: - ## PRE-CHAOS APPLICATION LIVENESS CHECK + - block: - - include_tasks: /utils/common/application_liveness_check.yml - when: liveness_label != '' + ## PRE-CHAOS APPLICATION LIVENESS CHECK + - include_tasks: /utils/common/application_liveness_check.yml + when: "liveness_label != ''" - - name: Identify the data consistency util to be invoked - template: - src: data_persistence.j2 - dest: data_persistence.yml + - name: Identify the data consistency util to be invoked + template: + src: data_persistence.j2 + dest: data_persistence.yml - - include_vars: - file: data_persistence.yml + - include_vars: + file: data_persistence.yml + + - name: Record the data consistency util path + set_fact: + data_consistency_util_path: "{{ consistencyutil }}" + when: data_persistence != '' - - name: Record the chaos util path - set_fact: - chaos_util_path: "/experiments/openebs/openebs-pool-pod-failure/cstor_pool_delete.yml" + when: "a_pvc is defined and a_pvc != ''" - - name: Record the data consistency util path - set_fact: - data_consistency_util_path: "{{ consistencyutil }}" - when: data_persistence != '' + - name: Identify the chaos util to be invoked + template: + src: chaosutil.j2 + dest: chaosutil.yml + + - include_vars: + file: chaosutil.yml ## GENERATE EXP RESULT NAME - block: @@ -74,14 +81,7 @@ - "Label : {{ a_label }}" - "PVC : {{ a_pvc }}" - ## PRE-CHAOS APPLICATION LIVENESS CHECK - - name: Get application pod name - shell: > - kubectl get pods -n {{ a_ns }} -l {{ a_label }} --no-headers - -o=custom-columns=NAME:".metadata.name" - args: - executable: /bin/bash - register: app_pod_name + ## PRE-CHAOS APPLICATION STATUS CHECK - name: Verify that the AUT (Application Under Test) is running include_tasks: "/utils/common/status_app_pod.yml" @@ -91,21 +91,36 @@ delay: 5 retries: 60 - - name: Create some test data - include: "{{ data_consistency_util_path }}" - vars: - status: 'LOAD' - ns: "{{ a_ns }}" - pod_name: "{{ app_pod_name.stdout }}" - when: data_persistence != '' + - block: - ## STORAGE FAULT INJECTION + - name: Get application pod name + shell: > + kubectl get pods -n {{ a_ns }} -l {{ a_label }} --no-headers + -o=custom-columns=NAME:".metadata.name" + args: + executable: /bin/bash + register: app_pod + + - name: Create some test data + include: "{{ data_consistency_util_path }}" + vars: + status: 'LOAD' + ns: "{{ a_ns }}" + pod_name: "{{ app_pod.stdout }}" + when: data_persistence != '' + + when: "a_pvc is defined and a_pvc != ''" - - include: "{{ chaos_util_path }}" - error_messages: "{{ pool_debug_msg }}" - post_chaos_soak_time : "{{ c_duration }}" + ## STORAGE FAULT INJECTION + + - include_tasks: "{{ c_util }}" + vars: + error_messages: "{{ pool_debug_msg }}" + post_chaos_soak_time : "{{ c_duration }}" + app_ns: "{{ a_ns }}" + app_label: "{{ a_label }}" - ## POST-CHAOS APPLICATION LIVENESS CHECK + ## POST-CHAOS APPLICATION STATUS CHECK - name: Verify AUT liveness post fault-injection include_tasks: "/utils/common/status_app_pod.yml" @@ -114,36 +129,40 @@ app_label: "{{ a_label }}" delay: 5 retries: 60 - - - name: Verify application data persistence - include: "{{ data_consistency_util_path }}" - vars: - status: 'VERIFY' - ns: "{{ a_ns }}" - label: "{{ a_label }}" - pod_name: "{{ app_pod_name.stdout }}" - when: data_persistence != '' - - - name: Get application pod name - shell: > - kubectl get pods -n {{ a_ns }} -l {{ a_label }} --no-headers - -o=custom-columns=NAME:".metadata.name" - args: - executable: /bin/bash - register: new_app_pod - - - name: Verify successful database delete - include: "{{ data_consistency_util_path }}" - vars: - status: 'DELETE' - ns: "{{ a_ns }}" - label: "{{ a_label }}" - pod_name: "{{ new_app_pod.stdout }}" - when: data_persistence != '' - - # Check application liveness post chaos - - include_tasks: /utils/common/application_liveness_check.yml - when: liveness_label != '' + + - block: + + - name: Get application pod name + shell: > + kubectl get pods -n {{ a_ns }} -l {{ a_label }} --no-headers + -o=custom-columns=NAME:".metadata.name" + args: + executable: /bin/bash + register: new_app_pod + + - name: Verify application data persistence + include: "{{ data_consistency_util_path }}" + vars: + status: 'VERIFY' + ns: "{{ a_ns }}" + label: "{{ a_label }}" + pod_name: "{{ new_app_pod.stdout }}" + when: data_persistence != '' + + - name: Verify successful database delete + include: "{{ data_consistency_util_path }}" + vars: + status: 'DELETE' + ns: "{{ a_ns }}" + label: "{{ a_label }}" + pod_name: "{{ new_app_pod.stdout }}" + when: data_persistence != '' + + # Check application liveness post chaos + - include_tasks: /utils/common/application_liveness_check.yml + when: liveness_label != '' + + when: "a_pvc is defined and a_pvc != ''" - set_fact: flag: "Pass" diff --git a/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_k8s_job.yml b/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_k8s_job.yml index 3a92a455202..d765d54b318 100644 --- a/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_k8s_job.yml +++ b/experiments/openebs/openebs-pool-pod-failure/openebs_pool_pod_failure_k8s_job.yml @@ -40,7 +40,8 @@ spec: - name: APP_LABEL value: '' - #provide application pvc + #provide application pvc + # NOTE: Leave it blank for pool-validation - name: APP_PVC value: ''