Add second half experiment

experiments/15-e2e-scenarios-v2/scale_up/COND

@@ -14,6 +14,20 @@ run_experiment(
   },
 )
 
+run_experiment(
+  name="brad_100g_2",
+  run="./run_workload_2.sh",
+  options={
+    # TODO: Ideally, configurations are shared. Only keep AWS secrets separate.
+    "config-file": "config/config_large_100.yml",
+    "planner-config-file": "config/planner.yml",
+    "schema-name": "imdb_extended_100g",
+    "ra-query-bank-file": IMDB_100GB_REGULAR_QUERY_BANK,
+    "num-front-ends": 28,
+    "dataset-type": "100gb",
+  },
+)
+
 run_command(
   name="brad_100g_debug",
   run="./run_workload_debug.sh",

experiments/15-e2e-scenarios-v2/scale_up/run_workload.sh

@@ -98,8 +98,6 @@ sleep $((20 * 60))  # 20 mins total; 62 mins cumulative
 
 log_workload_point "stopping_heavy_rana_8"
 kill -INT $heavy_rana_pid
-sleep 5
-kill -KILL $heavy_rana_pid  # To ensure we do not get stuck.
 wait $heavy_rana_pid
 
 log_workload_point "start_heavy_rana_10"
@@ -109,8 +107,6 @@ sleep $((10 * 60))  # 10 mins total; 72 mins cumulative
 
 log_workload_point "stopping_heavy_rana_10"
 kill -INT $heavy_rana_pid
-sleep 5
-kill -KILL $heavy_rana_pid  # To ensure we do not get stuck.
 wait $heavy_rana_pid
 
 log_workload_point "start_heavy_rana_20"

experiments/15-e2e-scenarios-v2/scale_up/run_workload_2.sh

@@ -0,0 +1,105 @@
+#! /bin/bash
+
+script_loc=$(cd $(dirname $0) && pwd -P)
+cd $script_loc
+source ../common.sh
+
+# Scenario:
+# - Start from 2x t4g.medium Aurora, Redshift off
+# - 4 T clients, increasing to 24 T clients by 4 every minute
+# We expect BRAD to scale up Aurora at this point, but not start Redshift (a replica should be fine for the analytical workload)
+# - Increase the analytical load + add new "heavier" queries - expect that these go to Athena
+# - Increase frequency of queries, expect Redshift to start (go straight to 2x dc2.large to avoid classic resize for practical purposes)
+
+# TODO: This executor file should be adapted to run against the baselines too
+# (TiDB / Serverless Redshift + Aurora)
+
+initial_queries="99,56,32,92,91,49,30,83,94,38,87,86,76,37,31,46"
+heavier_queries="58,61,62,64,69,73,74,51,57,60"
+
+function step_txns() {
+  local lo=$1
+  local hi=$2
+  local gap_minute=$3
+}
+
+# Arguments:
+# --config-file
+# --planner-config-file
+# --query-indexes
+extract_named_arguments $@
+
+function txn_sweep() {
+  local sweep=$1
+  local gap_minute=$2
+  local keep_last=$3
+
+  for t_clients in $sweep; do
+    start_txn_runner $t_clients  # Implicit: --dataset-type
+    txn_pid=$runner_pid
+
+    sleep $(($gap_minute * 60))
+    if [[ -z $keep_last ]] || [[ $t_clients != $keep_last ]]; then
+      kill -INT $txn_pid
+      wait $txn_pid
+    fi
+  done
+}
+
+function inner_cancel_experiment() {
+  if [ ! -z $heavy_rana_pid ]; then
+    cancel_experiment $rana_pid $txn_pid $heavy_rana_pid
+  else
+    cancel_experiment $rana_pid $txn_pid
+  fi
+}
+
+trap "inner_cancel_experiment" INT
+trap "inner_cancel_experiment" TERM
+
+start_brad $config_file $planner_config_file
+log_workload_point "brad_start_initiated"
+sleep 30
+
+# Start with 8 analytical clients.
+log_workload_point "start_rana_8"
+start_repeating_olap_runner 8 15 5 $initial_queries "ra_8"
+rana_pid=$runner_pid
+sleep 2
+
+# Start with 8 transactional clients; hold for 10 minutes to stabilize.
+log_workload_point "start_txn_8"
+start_txn_runner 4
+txn_pid=$runner_pid
+sleep $((10 * 60))
+
+# 20 minutes.
+log_workload_point "start_heavy_rana_8"
+start_repeating_olap_runner 8 15 1 $heavier_queries "ra_8_heavy" 8
+heavy_rana_pid=$runner_pid
+sleep $((20 * 60))  # 20 mins total; 30 mins cumulative
+
+log_workload_point "stopping_heavy_rana_8"
+kill -INT $heavy_rana_pid
+wait $heavy_rana_pid
+
+log_workload_point "start_heavy_rana_10"
+start_repeating_olap_runner 10 5 1 $heavier_queries "ra_10_heavy" 8
+heavy_rana_pid=$runner_pid
+sleep $((10 * 60))  # 10 mins total; 40 mins cumulative
+
+log_workload_point "stopping_heavy_rana_10"
+kill -INT $heavy_rana_pid
+wait $heavy_rana_pid
+
+log_workload_point "start_heavy_rana_20"
+start_repeating_olap_runner 20 5 1 $heavier_queries "ra_20_heavy" 8
+heavy_rana_pid=$runner_pid
+sleep $((30 * 60))  # 30 mins total; 70 mins cumulative
+
+log_workload_point "experiment_workload_done"
+
+# Shut down everything now.
+>&2 echo "Experiment done. Shutting down runners..."
+graceful_shutdown $rana_pid $heavy_rana_pid $txn_pid
+log_workload_point "shutdown_complete"