[Cherry-pick for 1.3.0] (#1453)

* refactor(experiments): Adding chaos_ns and pod_name(powerfulseal) in experiment job (#1420) * refactor(node-drain): Adding the task banner in the node-drain experiment (#1434) * refactor(container-kill): Adding the task banner in the container-kill experiment (#1422) * refactor(pod-memory-hog): Adding the task banner in the pod-memory-hog experiment (#1433) * refactor(node-memory-hog): Adding the task banner in the node-memory-hog experiment (#1432) * refactor(pod-cpu-hog): Adding the task banner in the pod-cpu-hog experiment (#1431) * refactor(node-cpu-hog): Adding the task banner in the node-cpu-hog experiment (#1430) * refactor(network-exp): Adding the task banner in the network experiment (#1429) * refactor(disk-loss): Adding the task banner in the disk-loss experiment (#1428) * refactor(disk-fill): Adding the task banner in the disk-fill experiment (#1424) * adding chaos_util env and refactor external liveness check of nfs (#1435) * adding chaos_util env * refactor(contributor-guide): Adding the task banner in the contributor guide (#1437) * feat(cassandra): Adding cassandra pod delete experiment (#1363) * refactor(pod-delete): Adding kill_count in all experiments (#1440) * feat(pod-memory-hog): Add doccumentation link of pod memory hog in README (#1427) * (Docs): Adding docs for the experiments (#1447) * adding doc links for the experiments * Minor refactor (#1446) * minor refactor of result update util (remove unused var) * refactor(openebs-exp): Adding the task banner in the openebs pod-delete experiments (#1443) * refactor(openebs-exp): Adding the task banner in the openebs pod-delete experiment * refactor(openebs-exp): Adding the task banner in the disk-loss/control-plan experiments (#1444) * refactor(openebs-exp): Adding the task banner in the disk-loss/control-plan experiment * refactor(openebs-exp): Adding the task banner in the openebs network experiment (#1442) * Adding nfs kill doc link (#1448) * adding nfs doc link * refactor(openebs-exp): Adding the task banner in the openebs container-kill experiment (#1441) * refactor(cassandra-exp): Adding the task banner in the cassandra-pod-delete experiment (#1439) * refactor(kafka-exp): Adding the task banner in the kafka-experiments experiment (#1438) * fix(openebs-exp): Fix the openebs experiments (#1451) * fix(cresult): update the cresult jinja template (#1452)
litmuschaos · Apr 15, 2020 · 4248bdb · 4248bdb
1 parent ec3bbb5
commit 4248bdb
Show file tree

Hide file tree

Showing 158 changed files with 2,490 additions and 1,006 deletions.
diff --git a/chaoslib/litmus/disk_fill/convert_fill_percentage.yml b/chaoslib/litmus/disk_fill/convert_fill_percentage.yml
@@ -1,15 +1,15 @@
-- name: Multiply the request value with unit in KB
+- name: "[Prepare]: Multiply the request value with unit in KB"
   set_fact:
     req_KB: "{{ req_value.stdout|int * req_storage_KB|int }}"
 
-- name: Multiply the limit value with unit in KB
+- name: "[Prepare]: Multiply the limit value with unit in KB"
   set_fact:
     limit_KB: "{{ limit_value.stdout|int * limit_storage_KB|int }}"
 
-- name: Convert the fill percent
+- name: "[Prepare]: Convert the value in the form of percentage"
   set_fact: 
     fill: "{{ fill_percentage |float / 100 }}"
 
-- name: Fill percent into storage
+- name: "[Prepare]: Get the fill value in the unit of KB"
   set_fact:
     to_fill: "{{ fill|float * limit_KB|float }}"
diff --git a/chaoslib/litmus/disk_fill/disk_fill_by_litmus.yml b/chaoslib/litmus/disk_fill/disk_fill_by_litmus.yml
@@ -1,67 +1,67 @@
 - block:
 
-    - name: Get a Random Pod name
+    - name: "[Prepare]: Get a Random Application Pod name"
       shell: >
         kubectl get pods -l {{ a_label }} -n {{ a_ns }} --no-headers -o custom-columns=:metadata.name | shuf -n 1 
       args:
         executable: /bin/bash
       register: pod_name
 
-    - name: Get ephemeral-storage Limit Value
+    - name: "[Prepare]: Get the value of ephemeral-storage Limit"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.spec.containers[?(@.name=="{{ c_container }}")].resources.limits.ephemeral-storage}' | grep -Eo '+|[0-9]+'
       args:
         executable: /bin/bash
       register: limit_value
 
-    - name: Get ephemeral-storage Limit storage unit
+    - name: "[Prepare]: Get the unit of ephemeral-storage limit"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.spec.containers[?(@.name=="{{ c_container }}")].resources.limits.ephemeral-storage}' | grep -Eo '[[:alpha:]]+'
       args:
         executable: /bin/bash
       register: limit_value_storage
 
-    - name: Get ephemeral-storage Request Value
+    - name: "[Prepare]: Get the requested ephemeral-storage value"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.spec.containers[?(@.name=="{{ c_container }}")].resources.requests.ephemeral-storage}' | grep -Eo '+|[0-9]+'
       args:
         executable: /bin/bash
       register: req_value
 
-    - name: Get ephemeral-storage Request storage unit
+    - name: "[Prepare]: Get the requested ephemeral-storage unit"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.spec.containers[?(@.name=="{{ c_container }}")].resources.requests.ephemeral-storage}' | grep -Eo '[[:alpha:]]+'
       args:
         executable: /bin/bash
       register: req_value_storage
 
-    - name: Fetch the value of Request ephemeral-storage Unit in KB
+    - name: "[Prepare]: Fetch the value of Request ephemeral-storage Unit in KB"
       set_fact:
         req_storage_KB: "{{ lookup('vars', req_value_storage.stdout) }}"
 
-    - name: Fetch the value of Limit ephemeral-storage Unit in KB
+    - name: "[Prepare]: Fetch the value of Limit ephemeral-storage Unit in KB"
       set_fact:
         limit_storage_KB: "{{ lookup('vars', limit_value_storage.stdout) }}"
 
     - include_tasks: /chaoslib/litmus/disk_fill/convert_fill_percentage.yml
 
     - block:
-        - name: Generate a run id if not passed from the engine/experiment
+        - name: "[Prepare:] Generate a run id if not passed from the engine/experiment"
           shell: echo $(mktemp) | cut -d '.' -f 2 | cut -c -6
           register: rand_string   
 
         - set_fact:
             run_id: "{{ rand_string.stdout | lower }}"
       when: "run_id is not defined or run_id == ''"
 
-    - name: Get the Container ID
+    - name: "[Preapre]: Get the Container ID of Target Container"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.status.containerStatuses[?(@.name=="{{ c_container }}")].containerID}' | awk -F'//' '{print $2}'
       args:
         executable: /bin/bash
       register: containerID
 
-    - name: Identify the application node
+    - name: "[Prepare]: Identify the application node name"
       shell: >
         kubectl get pods {{ pod_name.stdout }} -n {{ a_ns }} --no-headers -o custom-columns=:spec.nodeName
       args:
@@ -71,27 +71,38 @@
     - set_fact: 
         app_node: "{{ app_node.stdout }}"
 
-    - name: Patch the engine UID and node name 
+    - name: "[Prepare]: Patch the engine UID and node name"
       template:
         src:  /chaoslib/litmus/disk_fill/disk_fill_pod.j2
         dest: /chaoslib/litmus/disk_fill/disk_fill_pod.yml
 
     ## RECORD EVENT FOR CHAOS INJECTION
-    - include_tasks: /utils/common/generate-kubernetes-chaos-events.yml
+    - name: "[Event]: Generating an Event for ChaosInjection"
+      include_tasks: /utils/common/generate-kubernetes-chaos-events.yml
       vars:
         stage: "ChaosInject"
         exp_pod_name: "{{ chaos_pod_name }}"
         engine_ns: "{{ c_ns }}"
         message: "Injecting {{ c_experiment }} chaos on {{ pod_name.stdout }}"
       when: "c_engine is defined and c_engine != ''"
 
+    - block:
+        - debug:
+            msg: "***** Waiting for the ramp interval of {{ ramp_time }}s *****"
+
+        - name: "[Ramp]: Waiting for the specified ramp time before injecting chaos"
+          wait_for: timeout="{{ ramp_time }}"
+
+      when: "ramp_time is defined and ramp_time != ''" 
+
     # Setting pod_running_status to nil
     - set_fact: 
         pod_running_status: ""
 
      # Disk Fill pods creation is attempted for a total of 3 times, if it is not immediately schedulable due to transient node conditions
      # If the disk-fill pod is not schedulable across these 3 tries, the experiment is failed with message indicating improper cluster state.
-    - include_tasks: /utils/common/create_chaos_pod.yml
+    - name: "[Prepare]: Including the util to create the chaos pod"
+      include_tasks: /utils/common/create_chaos_pod.yml
       vars:
         pod_ns: "{{ a_ns }}"
         c_path: "/chaoslib/litmus/disk_fill/disk_fill_pod.yml"
@@ -103,82 +114,83 @@
         msg: "disk_fill_by_litmus lib failed, Unable to create as disk-fill pod couldn't be scheduled on the {{ app_node }} node"
       when: "pod_running_status is not defined or pod_running_status != 'Running'"
 
-    - name: Wait for the specified ramp time before injecting chaos
-      wait_for: timeout="{{ ramp_time }}"
-      when: "ramp_time is defined and ramp_time != ''"
-
-    - name: Record the disk-fill pod on app node
+    - name: "[Prepare]: Record the disk-fill pod on app node"
       shell: >
         kubectl get pods -l name=disk-fill-{{ run_id }} -n {{ a_ns }} -o jsonpath='{.items[?(@.spec.nodeName=="{{ app_node }}")].metadata.name}'
       args:
         executable: /bin/bash
       register: disk_fill_pod
 
-    - name: Exec into the pod to get ephemeral-storage size
+    - name: "[Prepare]: Exec into the pod to get ephemeral-storage size"
       shell: >
         kubectl exec -it {{ disk_fill_pod.stdout }} -n {{ c_ns }} -- sh -c "cd /diskfill && cd {{ containerID.stdout }} && du -k |tail -n1 | awk '{print $1}'"
       args:
         executable: /bin/bash
       register: pod_current_size_full
 
-    - name: Echo the filled storage in KB to get the integer part for multiplication
+    - name: "[Prepare]: Deriving the magnitude of current filled storage in KB"
       shell: >
         echo {{ pod_current_size_full.stdout }} | grep -Eo '+|[0-9]+'
       args:
         executable: /bin/bash
       register: pod_current_size
 
-    - name: Total Used Ephemeral Storage in KB
+    - name: "[Prepare]: Record the total Used Ephemeral Storage in KB"
       set_fact:
         used_storage: "{{ pod_current_size.stdout }}"
 
-    - name: Subtract to get the file allocation size
+    - name: "[Prepare]: Deriving the file allocation size"
       set_fact:
         fill_storage: "{{ to_fill|int - used_storage|int }}"
 
     - include_tasks: /chaoslib/litmus/disk_fill/file_creation.yml
 
-    - name: Waiting for Chaos Duration
+    - name: "[Wait]: Waiting for the Chaos Duration"
       wait_for:
         timeout: "{{ c_duration }}"
 
-    - name: Get the Pod Status 
+    - name: "[Status]: Get the Pod Status"
       shell: >
         kubectl get pod {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.status.phase}'
       args:
         executable: /bin/bash
       register: pod_status
 
-    - name: Check if the pod is Evicted
+    - name: "[CleanUP]: Check if the pod is Evicted"
       shell: >
         kubectl get pod {{ pod_name.stdout }} -n {{ a_ns }} -o jsonpath='{.status.reason}'
       args:
         executable: /bin/bash
       register: eviction_status
 
-    - name: Delete Pod if evicted
+    - name: "[CleanUP]: Delete Pod if evicted"
       shell: >
         kubectl delete pod {{ pod_name.stdout }} -n {{ a_ns }}
       when: eviction_status.stdout == "Evicted"
 
-    - name: If Pod is not evicted / running
+    - name: "[CleanUP]: If Pod is not evicted / running"
       shell: >
         kubectl exec -it {{ disk_fill_pod.stdout }} -n {{ c_ns }} -- sh -c "rm -rf /diskfill/{{ containerID.stdout }}/diskfill"
       when: pod_status.stdout == "Running"
 
-    - name: Wait for the specified ramp time after injecting chaos
-      wait_for: timeout="{{ ramp_time }}"
+    - block:
+        - debug:
+            msg: "***** Waiting for the ramp interval of {{ ramp_time }}s *****"
+
+        - name: "[Ramp]: Waiting for the specified ramp time after injecting chaos"
+          wait_for: timeout="{{ ramp_time }}"
+
       when: "ramp_time is defined and ramp_time != ''" 
 
     - block:
 
         ## DELETING THE DISK FILL POD
-        - name: Deleting the Disk Fill Pod
+        - name: "[CleanUP]: Deleting the Disk Fill Pod"
           shell: kubectl delete -f /chaoslib/litmus/disk_fill/disk_fill_pod.yml -n {{ c_ns }}
           args:
             executable: /bin/bash
 
-        - name: Confirm that the Disk Fill Pod is deleted successfully
+        - name: "[CleanUP]: Confirm that the Disk Fill Pod is deleted successfully"
           k8s_facts:
             kind: Pod
             namespace: "{{ c_ns }}"
@@ -195,14 +207,14 @@
 
     - block: 
 
-        - name: Deleting the Disk-fill Pod
+        - name: "[CleanUP]: Deleting the Disk-fill Pod"
           shell: >
             kubectl delete -f /chaoslib/litmus/disk_fill/disk_fill_pod.yml -n {{ c_ns }}
           args:
             executable: /bin/bash
           when: "chaos_pod_result.rc == 0 "
 
-        - name: Confirm that the Disk Fill Pod is deleted successfully
+        - name: "[CleanUP]: Confirm that the Disk Fill Pod is deleted successfully"
           k8s_facts:
             kind: Pod
             namespace: "{{ c_ns }}"

diff --git a/chaoslib/litmus/disk_fill/file_creation.yml b/chaoslib/litmus/disk_fill/file_creation.yml
@@ -1,15 +1,15 @@
-- name: Block size for file creation
+- name: "[Prepare]: Record the Block size for the file creation"
   set_fact:
     bs: 4K
 
-- name: Count for file creation 
+- name: "[Prepare]: Deriving the file count for creation"
   set_fact:
     fill_count: "{{ fill_storage|int / 4 }}"
 
-- name: Change count for file creation to integer
+- name: "[Prepare]: Converting the file count to integer"
   set_fact:
     fill_count_int: "{{ fill_count|int }}"
 
-- name: File creation
+- name: "[Inject]: Creating file inside the application pod"
   shell: >
     kubectl exec -it {{ disk_fill_pod.stdout }} -n {{ a_ns }} -- sh -c "cd /diskfill/{{ containerID.stdout }} && dd if=/dev/urandom of=/diskfill/{{ containerID.stdout }}/diskfill bs={{ bs }} count={{ fill_count_int }}"