From c1a7e21c666fc9384032fe10b5f2ccf584d1051f Mon Sep 17 00:00:00 2001
From: James Busche <jbusche@us.ibm.com>
Date: Tue, 5 Dec 2023 14:39:03 -0800
Subject: [PATCH 1/2] kwok and perf updates

Signed-off-by: James Busche <jbusche@us.ibm.com>
---
 test/perf-test/kwokmcadperf.sh                | 57 +++++++------
 test/perf-test/nodes.sh                       | 36 ++++----
 test/perf-test/perf.sh                        | 59 ++++++-------
 test/perf-test/preempt-exp-kwok.yaml          |  9 +-
 test/perf-test/preempt-exp.yaml               | 10 +--
 .../simulatingnodesandappwrappers.md          | 83 ++++++-------------
 6 files changed, 112 insertions(+), 142 deletions(-)

diff --git a/test/perf-test/kwokmcadperf.sh b/test/perf-test/kwokmcadperf.sh
index bb0e2ed37..6dc3d5cdf 100755
--- a/test/perf-test/kwokmcadperf.sh
+++ b/test/perf-test/kwokmcadperf.sh
@@ -10,7 +10,7 @@ function help() {
     echo
     echo "Preconditions: "
     echo "   - The script assumes you've logged into your cluster already. If not, it will tell you to login."
-    echo "   - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
+    echo "   - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
     echo "   - The script checks that you have the kwok-controller installed, otherwise it'll tell you to install it first."
     echo
     echo "Options:"
@@ -40,12 +40,15 @@ function check_mcad_installed_status() {
     res2="$?"
     kubectl get crd |grep appwrapper &> /dev/null
     res3="$?"
+    kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
+    res4="$?"
     set -e
     MCAD="$res2"
     CRD="$res3"
-      if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
+    CODEFLARE="$res4"
+      if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
       then
-        echo "You need Install MCAD Controller first before running this script"
+        echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
         exit 1
       else
         echo "Nice, MCAD Controller is installed"
@@ -95,65 +98,65 @@ echo
 check_kwok_installed_status
 
 echo
-read -p "How many fake KWOK appwrapper jobs do you want?" jobs
+read -p "How many fake KWOK appwrapper jobs do you want?" JOBS
 
 # Start the timer now
 SECONDS=0
 
-echo "jobs number is $jobs"
+echo "jobs number is $JOBS"
 export STARTTIME=`date +"%T"`
 echo " "
 echo "Jobs started at: $STARTTIME" |tee fake-job-$STARTTIME.log
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realjobs=$jobs-1))
-
-for num in $(eval echo "{0.."$realjobs"}")
+COUNTER=1
+while [ $COUNTER -le $JOBS ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting job $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting job $COUNTER"
+
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
-      darwin*) 
-        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
-      *) 
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$num/fake-defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+      darwin*)
+        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+      *)
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/fake-defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/preempt-exp-kwok.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original preempt-exp-kwok.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
       darwin*) 
-        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i '' "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
       *) 
-        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$next_num/fake-defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
+        sed -i "s/fake-defaultaw-schd-spec-with-timeout-$JOBS/fake-defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp-kwok.yaml ;;
     esac
 
 # Check for all jobs to report complete
-jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
+JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
 
-while [ $jobstatus -lt $jobs ]
+while [ $JOBSTATUS -lt $JOBS ]
 do
-   echo "Number of completed jobs is: " $jobstatus " and the goal is: " $jobs
+   echo "Number of completed jobs is: " $JOBSTATUS " and the goal is: " $JOBS
    sleep 10
-   jobstatus=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
+   JOBSTATUS=`kubectl get jobs -n default --no-headers --field-selector status.successful=1 |wc -l`
 done
 
 echo " "
 export FINISHTIME=`date +"%T"`
-echo "All $jobstatus jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
-echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
+echo "All $JOBSTATUS jobs finished: $FINISHTIME" |tee -a fake-job-$STARTTIME.log
+echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/fake-job-$STARTTIME.log
 echo " "
-echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log"
+echo "Test results are stored in this file: ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log"
 
 # Rename the log to show the number of jobs used
-mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$next_num-$STARTTIME.log
+mv ${SCRIPT_DIR}/fake-job-$STARTTIME.log ${SCRIPT_DIR}/fake-job-$JOBS-$STARTTIME.log
 
 #Ask if you want to auto-cleanup the appwrapper jobs
 echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"
diff --git a/test/perf-test/nodes.sh b/test/perf-test/nodes.sh
index dc0bcb3ad..410d20e41 100755
--- a/test/perf-test/nodes.sh
+++ b/test/perf-test/nodes.sh
@@ -65,51 +65,49 @@ echo "Checking whether we have a valid cluster login or not..."
 check_kubectl_login_status
 
 # Track whether you have the KWOK controller installed
-echo "Checking MCAD Controller installation status"
+echo "Checking KWOK Controller installation status"
 echo
 check_kwok_installed_status
 
 echo
-read -p "How many simulated KWOK nodes do you want?" nodes
+read -p "How many simulated KWOK nodes do you want?" NODES
 
-echo "Nodes number is $nodes"
+echo "Nodes number is $NODES"
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realnodes=$nodes-1))
-echo "The real number of nodes is $realnodes"
-
-for num in $(eval echo "{0.."$realnodes"}")
+COUNTER=1
+while [ $COUNTER -le $NODES ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting node $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting node $COUNTER"
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
-      darwin*) 
-        sed -i '' "s/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
-      *) 
-        sed -i "/kwok-node-$num/kwok-node-$next_num/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
+      darwin*)
+        sed -i '' "s/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ${SCRIPT_DIR}/node.yaml ;;
+      *)
+        sed -i "/kwok-node-$ORIG_COUNTER/kwok-node-$COUNTER/g" ${SCRIPT_DIR}/node.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/node.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original node.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
       darwin*) 
-        sed -i '' "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i '' "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
       *) 
-        sed -i "s/kwok-node-$next_num/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
+        sed -i "s/kwok-node-$NODES/kwok-node-0/g" ${SCRIPT_DIR}/node.yaml ;;
     esac
 
 # Check for all nodes to report complete
 echo "Waiting until all the simualted pods become ready:"
 kubectl wait --for=condition=Ready nodes --selector type=kwok --timeout=600s
 echo " "
-echo "Total amount of simulated nodes requested is: $nodes"
+echo "Total amount of simulated nodes requested is: $NODES"
 echo "Total number of created nodes is: "`kubectl get nodes --selector type=kwok -o name |wc -l`
 kubectl get nodes --selector type=kwok
 
diff --git a/test/perf-test/perf.sh b/test/perf-test/perf.sh
index a9d47801a..97ed91129 100755
--- a/test/perf-test/perf.sh
+++ b/test/perf-test/perf.sh
@@ -9,7 +9,7 @@ function help() {
     echo
     echo "Preconditions: "
     echo "   - The script assumes you've logged into your cluster already. If not, it will tell you to login."
-    echo "   - The script checks that you have the mcad-controller installed, otherwise it'll tell you to install it first."
+    echo "   - The script checks that you have the mcad-controller installed (or the newer CodeFlare Operator), otherwise it'll tell you to install it first."
     echo
     echo "Options:"
     echo "  -h       Print this help message"
@@ -38,12 +38,15 @@ function check_mcad_installed_status() {
     res2="$?"
     kubectl get crd |grep appwrapper &> /dev/null
     res3="$?"
+    kubectl get pod -A |grep codeflare-operator-manager &> /dev/null
+    res4="$?"
     set -e
     MCAD="$res2"
     CRD="$res3"
-      if [[ $MCAD == 1 ]] || [[ $CRD == 1 ]]
+    CODEFLARE="$res4"
+      if [[ $MCAD == 1 ]] && [[ $CODEFLARE == 1 ]] || [[ $CRD == 1 ]]
       then
-        echo "You need Install MCAD Controller first before running this script"
+        echo "You need Install the MCAD Controller or the latest CodeFlare Operator first before running this script"
         exit 1
       else
         echo "Nice, MCAD Controller is installed"
@@ -70,68 +73,68 @@ check_kubectl_login_status
 # Track whether you have the MCAD controller installed
 echo "Checking MCAD Controller installation status"
 echo
-#check_mcad_installed_status
+check_mcad_installed_status
 
 echo
-read -p "How many appwrapper jobs do you want?" jobs
+read -p "How many appwrapper jobs do you want?" JOBS
 
 # Start the timer now
 SECONDS=0
 
-echo "Appwrapper number is $jobs"
+echo "Appwrapper number is $JOBS"
 export STARTTIME=`date +"%T"`
 echo " "
 echo "Appwrappers started at: $STARTTIME" |tee job-$STARTTIME.log
 echo " "
 
-# This fixes the number of jobs to be one less so the for loop gets the right amount
-((realjobs=$jobs-1))
-
-for num in $(eval echo "{0.."$realjobs"}")
+COUNTER=1
+while [ $COUNTER -le $JOBS ]
 do
-    next_num=$(($num + 1))
-    echo "Submitting job $next_num"
+    ORIG_COUNTER=$(($COUNTER - 1))
+    echo "Submitting job $COUNTER"
+
 # Had to do this OSTYPE because sed acts differently on Linux versus Mac
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
-      darwin*) 
-        sed -i '' "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
-      *) 
-        sed -i "s/defaultaw-schd-spec-with-timeout-$num/defaultaw-schd-spec-with-timeout-$next_num/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+      darwin*)
+        sed -i '' "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+      *)
+        sed -i "s/defaultaw-schd-spec-with-timeout-$ORIG_COUNTER/defaultaw-schd-spec-with-timeout-$COUNTER/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
     esac
     kubectl apply -f ${SCRIPT_DIR}/preempt-exp.yaml
+COUNTER=$[$COUNTER +1]
 done
 
     # Let's reset the original preempt-exp.yaml file back to original value 
     case "$OSTYPE" in
       linux-gnu*)
-        sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
       darwin*) 
-        sed -i '' "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i '' "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
       *) 
-        sed -i "s/defaultaw-schd-spec-with-timeout-$next_num/defaultaw-schd-spec-with-timeout-1/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
+        sed -i "s/defaultaw-schd-spec-with-timeout-$JOBS/defaultaw-schd-spec-with-timeout-0/g" ${SCRIPT_DIR}/preempt-exp.yaml ;;
     esac
 
 # Check for all appwrappers to report complete
-jobstatus=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
+JOBSTATUS=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
 
-while [ $jobstatus -lt $jobs ]
+while [ $JOBSTATUS -lt $JOBS ]
 do
-   echo "Number of completed appwrappers is: " $jobstatus " and the goal is: " $jobs
+   echo "Number of completed appwrappers is: " $JOBSTATUS " and the goal is: " $JOBS
    sleep 10
-   jobstatus=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
+   JOBSTATUS=`kubectl get appwrappers -o=custom-columns=SUCCESS:.status.Succeeded -n default |grep 1 |wc -l`
 done
 
 echo " "
 export FINISHTIME=`date +"%T"`
-echo "All $jobstatus appwrappers finished: $FINISHTIME" |tee -a job-$STARTTIME.log
-echo "Total amount of time for $jobs appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
+echo "All $JOBSTATUS appwrappers finished: $FINISHTIME" |tee -a job-$STARTTIME.log
+echo "Total amount of time for $JOBS appwrappers is: $SECONDS seconds" |tee -a ${SCRIPT_DIR}/job-$STARTTIME.log
 echo " "
-echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log"
+echo "Test results are stored in this file: ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log"
 
 # Rename the log to show the number of jobs used
-mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$next_num-$STARTTIME.log
+mv ${SCRIPT_DIR}/job-$STARTTIME.log ${SCRIPT_DIR}/job-$JOBS-$STARTTIME.log
 
 #Ask if you want to auto-cleanup the appwrapper jobs
 echo "Do you want to cleanup the most recently created appwrappers? [Y/n]"
diff --git a/test/perf-test/preempt-exp-kwok.yaml b/test/perf-test/preempt-exp-kwok.yaml
index 0bab8c857..eee490656 100644
--- a/test/perf-test/preempt-exp-kwok.yaml
+++ b/test/perf-test/preempt-exp-kwok.yaml
@@ -1,7 +1,7 @@
 apiVersion: workload.codeflare.dev/v1beta1
 kind: AppWrapper
 metadata:
-  name: fake-defaultaw-schd-spec-with-timeout-1
+  name: fake-defaultaw-schd-spec-with-timeout-0
   namespace: default
 spec:
   schedulingSpec:
@@ -11,7 +11,6 @@ spec:
         growthType: "exponential" 
   priority: 9
   resources:
-    Items: []
     GenericItems:
     - replicas: 1
       completionstatus: Complete
@@ -30,7 +29,7 @@ spec:
         kind: Job
         metadata:
           namespace: default
-          name: fake-defaultaw-schd-spec-with-timeout-1
+          name: fake-defaultaw-schd-spec-with-timeout-0
         spec:
           parallelism: 1
           completions: 1
@@ -38,7 +37,7 @@ spec:
             metadata:
               namespace: default
               labels:
-                appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-1"
+                appwrapper.mcad.ibm.com: "fake-defaultaw-schd-spec-with-timeout-0"
             spec:
               affinity:
                 nodeAffinity:
@@ -56,6 +55,6 @@ spec:
                   operator: "Exists"
                   effect: "NoSchedule"
               containers:
-              - name: fake-defaultaw-schd-spec-with-timeout-1
+              - name: fake-defaultaw-schd-spec-with-timeout-0
                 image: fake-image
               restartPolicy: Never
diff --git a/test/perf-test/preempt-exp.yaml b/test/perf-test/preempt-exp.yaml
index 13d9e003e..14d6b5aac 100644
--- a/test/perf-test/preempt-exp.yaml
+++ b/test/perf-test/preempt-exp.yaml
@@ -1,7 +1,7 @@
 apiVersion: workload.codeflare.dev/v1beta1
 kind: AppWrapper
 metadata:
-  name: defaultaw-schd-spec-with-timeout-1
+  name: defaultaw-schd-spec-with-timeout-0
   namespace: default
 spec:
   schedulingSpec:
@@ -29,9 +29,9 @@ spec:
         kind: Job
         metadata:
           namespace: default
-          name: defaultaw-schd-spec-with-timeout-1
+          name: defaultaw-schd-spec-with-timeout-0
           labels:
-            appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-1
+            appwrapper.mcad.ibm.com: defaultaw-schd-spec-with-timeout-0
         spec:
           parallelism: 1
           completions: 1
@@ -39,10 +39,10 @@ spec:
             metadata:
               namespace: default
               labels:
-                appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-1"
+                appwrapper.mcad.ibm.com: "defaultaw-schd-spec-with-timeout-0"
             spec:
               containers:
-              - name: defaultaw-schd-spec-with-timeout-1
+              - name: defaultaw-schd-spec-with-timeout-0
                 image: ubi8-minimal:latest
                 command: [ "/bin/bash", "-c", "--" ]
                 args: [ "sleep 10" ]
diff --git a/test/perf-test/simulatingnodesandappwrappers.md b/test/perf-test/simulatingnodesandappwrappers.md
index 7fbe722bc..ce6f61700 100644
--- a/test/perf-test/simulatingnodesandappwrappers.md
+++ b/test/perf-test/simulatingnodesandappwrappers.md
@@ -80,40 +80,23 @@ kubectl get pods -n kube-system |grep mcad
 brew install jq
 ```
 
-### 2.2 Install the latest version of Kustomize
+### Step 2.2. Install KWOK in your KIND Cluster: 
+### 2.2.1 Variable Prep:
 ```
-brew install kustomize
+# KWOK repository
+KWOK_REPO=kubernetes-sigs/kwok
+# Get latest
+KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
 ```
-
-### Step 2.3. Install KWOK in your KIND Cluster: 
-### 2.3.1 Variable Prep:
-```
-export KWOK_WORK_DIR=$(mktemp -d)
-export KWOK_REPO=kubernetes-sigs/kwok
-export KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
-```
-### 2.3.2 Render kustomization yaml
+### 2.2.2 Deployment kwok and set up CRDs
 ```
-cat <<EOF > "${KWOK_WORK_DIR}/kustomization.yaml"
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-images:
-  - name: registry.k8s.io/kwok/kwok
-    newTag: "${KWOK_LATEST_RELEASE}"
-resources:
-  - "https://github.com/${KWOK_REPO}/kustomize/kwok?ref=${KWOK_LATEST_RELEASE}"
-EOF
+kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/kwok.yaml"
 ```
-### 2.3.3 Render it with the prepared variables.
+### 2.3.3 Set up default CRs of Stages (required)
 ```
-kubectl kustomize "${KWOK_WORK_DIR}" > "${KWOK_WORK_DIR}/kwok.yaml"
+kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/stage-fast.yaml"
 ```
-### Step 2.4 Install the KWOK Controller in kube-system namespace:
-### 2.4.1 Apply your rendered yaml file from step 1.3 above:
-```
-kubectl apply -f "${KWOK_WORK_DIR}/kwok.yaml"
-```
-### 2.4.2 Check to make sure the kwok controller started:
+### 2.4 Check to make sure the kwok controller started:
 ```
 kubectl get pods -n kube-system |grep kwok-controller
 ```
@@ -160,53 +143,37 @@ kubectl version --short=true
 kubectl get pods -n kube-system
 ```
 
-### 0.3 Requires that the MCAD controller is already installed
+### 0.3 Requires that the MCAD controller is already installed, either as a standalone controller or as part of the integrated CodeFlare operator starting with Codeflare Operator v1.0.0-rc.1
 ```
 kubectl get pods -A |grep mcad-controller
 ```
-
-### 0.4 Install podman, jq, etc...
+or
 ```
-yum install make podman git tree jq go bc -y
+kubectl get pod -A |grep codeflare-operator-manager
 ```
 
-### 0.5 Install the latest version of Kustomize
+### 0.4 Install podman, jq, etc...
 ```
-OS=$(uname)
-curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"  | bash
-mv kustomize /usr/local/bin
-kustomize version
+yum install make podman git tree jq go bc -y
 ```
 
 ## Step 1. Install KWOK in Cluster: 
 ### 1.1 Variable Prep:
 ```
-export KWOK_WORK_DIR=$(mktemp -d)
-export KWOK_REPO=kubernetes-sigs/kwok
-export KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
-```
-### 1.2 Render kustomization yaml
-```
-cat <<EOF > "${KWOK_WORK_DIR}/kustomization.yaml"
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-images:
-  - name: registry.k8s.io/kwok/kwok
-    newTag: "${KWOK_LATEST_RELEASE}"
-resources:
-  - "https://github.com/${KWOK_REPO}/kustomize/kwok?ref=${KWOK_LATEST_RELEASE}"
-EOF
+# KWOK repository
+KWOK_REPO=kubernetes-sigs/kwok
+# Get latest
+KWOK_LATEST_RELEASE=$(curl "https://api.github.com/repos/${KWOK_REPO}/releases/latest" | jq -r '.tag_name')
 ```
-### 1.3 Render it with the prepared variables.
+### 1.2 Deployment kwok and set up CRDs
 ```
-kubectl kustomize "${KWOK_WORK_DIR}" > "${KWOK_WORK_DIR}/kwok.yaml"
+kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/kwok.yaml"
 ```
-## Step 2. Install the KWOK Controller in kube-system namespace:
-### 2.1 Apply your rendered yaml file from step 1.3 above:
+### 1.3 Set up default CRs of Stages (required)
 ```
-kubectl apply -f "${KWOK_WORK_DIR}/kwok.yaml"
+kubectl apply -f "https://github.com/${KWOK_REPO}/releases/download/${KWOK_LATEST_RELEASE}/stage-fast.yaml"
 ```
-### 2.2 Check to make sure the kwok controller started:
+### 2. Check to make sure the kwok controller started:
 ```
 kubectl get pods -n kube-system |grep kwok-controller
 ```

From da2a192a985a775453b49ce349ec5eba6132a636 Mon Sep 17 00:00:00 2001
From: James Busche <jbusche@us.ibm.com>
Date: Mon, 18 Dec 2023 10:37:20 -0800
Subject: [PATCH 2/2] Improve doc

Signed-off-by: James Busche <jbusche@us.ibm.com>
---
 test/perf-test/simulatingnodesandappwrappers.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/perf-test/simulatingnodesandappwrappers.md b/test/perf-test/simulatingnodesandappwrappers.md
index ce6f61700..013a612b9 100644
--- a/test/perf-test/simulatingnodesandappwrappers.md
+++ b/test/perf-test/simulatingnodesandappwrappers.md
@@ -111,6 +111,8 @@ cd ../test/perf-test
 ``` 
 ./nodes.sh
 ```
+Note: If you want to change the resource configuration of your simulated worker nodes, you can edit the node.yaml file prior to running the nodes.sh script.
+
 ### 3.3 Check that the requested number of nodes started:
 ```
 kubectl get nodes --selector type=kwok