diff --git a/experiments/15-e2e-scenarios-v2/scale_up/COND b/experiments/15-e2e-scenarios-v2/scale_up/COND index 5721e424..0aac8306 100644 --- a/experiments/15-e2e-scenarios-v2/scale_up/COND +++ b/experiments/15-e2e-scenarios-v2/scale_up/COND @@ -14,6 +14,20 @@ run_experiment( }, ) +run_experiment( + name="brad_100g_2", + run="./run_workload_2.sh", + options={ + # TODO: Ideally, configurations are shared. Only keep AWS secrets separate. + "config-file": "config/config_large_100.yml", + "planner-config-file": "config/planner.yml", + "schema-name": "imdb_extended_100g", + "ra-query-bank-file": IMDB_100GB_REGULAR_QUERY_BANK, + "num-front-ends": 28, + "dataset-type": "100gb", + }, +) + run_command( name="brad_100g_debug", run="./run_workload_debug.sh", diff --git a/experiments/15-e2e-scenarios-v2/scale_up/run_workload.sh b/experiments/15-e2e-scenarios-v2/scale_up/run_workload.sh index 2dcc84dd..383c802e 100755 --- a/experiments/15-e2e-scenarios-v2/scale_up/run_workload.sh +++ b/experiments/15-e2e-scenarios-v2/scale_up/run_workload.sh @@ -98,8 +98,6 @@ sleep $((20 * 60)) # 20 mins total; 62 mins cumulative log_workload_point "stopping_heavy_rana_8" kill -INT $heavy_rana_pid -sleep 5 -kill -KILL $heavy_rana_pid # To ensure we do not get stuck. wait $heavy_rana_pid log_workload_point "start_heavy_rana_10" @@ -109,8 +107,6 @@ sleep $((10 * 60)) # 10 mins total; 72 mins cumulative log_workload_point "stopping_heavy_rana_10" kill -INT $heavy_rana_pid -sleep 5 -kill -KILL $heavy_rana_pid # To ensure we do not get stuck. wait $heavy_rana_pid log_workload_point "start_heavy_rana_20" diff --git a/experiments/15-e2e-scenarios-v2/scale_up/run_workload_2.sh b/experiments/15-e2e-scenarios-v2/scale_up/run_workload_2.sh new file mode 100755 index 00000000..5156155e --- /dev/null +++ b/experiments/15-e2e-scenarios-v2/scale_up/run_workload_2.sh @@ -0,0 +1,105 @@ +#! /bin/bash + +script_loc=$(cd $(dirname $0) && pwd -P) +cd $script_loc +source ../common.sh + +# Scenario: +# - Start from 2x t4g.medium Aurora, Redshift off +# - 4 T clients, increasing to 24 T clients by 4 every minute +# We expect BRAD to scale up Aurora at this point, but not start Redshift (a replica should be fine for the analytical workload) +# - Increase the analytical load + add new "heavier" queries - expect that these go to Athena +# - Increase frequency of queries, expect Redshift to start (go straight to 2x dc2.large to avoid classic resize for practical purposes) + +# TODO: This executor file should be adapted to run against the baselines too +# (TiDB / Serverless Redshift + Aurora) + +initial_queries="99,56,32,92,91,49,30,83,94,38,87,86,76,37,31,46" +heavier_queries="58,61,62,64,69,73,74,51,57,60" + +function step_txns() { + local lo=$1 + local hi=$2 + local gap_minute=$3 +} + +# Arguments: +# --config-file +# --planner-config-file +# --query-indexes +extract_named_arguments $@ + +function txn_sweep() { + local sweep=$1 + local gap_minute=$2 + local keep_last=$3 + + for t_clients in $sweep; do + start_txn_runner $t_clients # Implicit: --dataset-type + txn_pid=$runner_pid + + sleep $(($gap_minute * 60)) + if [[ -z $keep_last ]] || [[ $t_clients != $keep_last ]]; then + kill -INT $txn_pid + wait $txn_pid + fi + done +} + +function inner_cancel_experiment() { + if [ ! -z $heavy_rana_pid ]; then + cancel_experiment $rana_pid $txn_pid $heavy_rana_pid + else + cancel_experiment $rana_pid $txn_pid + fi +} + +trap "inner_cancel_experiment" INT +trap "inner_cancel_experiment" TERM + +start_brad $config_file $planner_config_file +log_workload_point "brad_start_initiated" +sleep 30 + +# Start with 8 analytical clients. +log_workload_point "start_rana_8" +start_repeating_olap_runner 8 15 5 $initial_queries "ra_8" +rana_pid=$runner_pid +sleep 2 + +# Start with 8 transactional clients; hold for 10 minutes to stabilize. +log_workload_point "start_txn_8" +start_txn_runner 4 +txn_pid=$runner_pid +sleep $((10 * 60)) + +# 20 minutes. +log_workload_point "start_heavy_rana_8" +start_repeating_olap_runner 8 15 1 $heavier_queries "ra_8_heavy" 8 +heavy_rana_pid=$runner_pid +sleep $((20 * 60)) # 20 mins total; 30 mins cumulative + +log_workload_point "stopping_heavy_rana_8" +kill -INT $heavy_rana_pid +wait $heavy_rana_pid + +log_workload_point "start_heavy_rana_10" +start_repeating_olap_runner 10 5 1 $heavier_queries "ra_10_heavy" 8 +heavy_rana_pid=$runner_pid +sleep $((10 * 60)) # 10 mins total; 40 mins cumulative + +log_workload_point "stopping_heavy_rana_10" +kill -INT $heavy_rana_pid +wait $heavy_rana_pid + +log_workload_point "start_heavy_rana_20" +start_repeating_olap_runner 20 5 1 $heavier_queries "ra_20_heavy" 8 +heavy_rana_pid=$runner_pid +sleep $((30 * 60)) # 30 mins total; 70 mins cumulative + +log_workload_point "experiment_workload_done" + +# Shut down everything now. +>&2 echo "Experiment done. Shutting down runners..." +graceful_shutdown $rana_pid $heavy_rana_pid $txn_pid +log_workload_point "shutdown_complete"