Display VDBE state on the dashboard (#515)

This PR also includes other various improvements.

config/system_config_demo.yml

@@ -0,0 +1,169 @@
+# This file contains configurations that are used by BRAD. These are default
+# values and should be customized for specific situations.
+
+# BRAD's front end servers will listen for client connections on this interface
+# and port. If `num_front_ends` is greater than one, subsequent front ends will
+# listen on successive ports (e.g., 6584, 6585, etc.).
+front_end_interface: "0.0.0.0"
+front_end_port: 6583
+num_front_ends: 1
+
+# If installed and enabled, BRAD will serve its UI from a webserver that listens
+# for connections on this network interface and port.
+ui_interface: "0.0.0.0"
+ui_port: 7583
+
+# Logging paths. If the value is in ALL_CAPS (with underscores), it is
+# interpreted as an environment variable (BRAD will log to the path stored in
+# the environment variable).
+
+# Where BRAD's daemon process will write its logs.
+daemon_log_file: /tmp
+
+# Where BRAD's front end processes will write their logs.
+front_end_log_path: /tmp
+
+# Where BRAD's blueprint planner will write debug logs.
+planner_log_path: /tmp
+
+# Where BRAD's metrics loggers will write their logs.
+metrics_log_path: /tmp
+
+# Probability that each transactional query will be logged.
+txn_log_prob: 0.01
+
+# Set to a non-zero value enable automatic data syncing. When this is set to 0,
+# automatic syncing is disabled.
+data_sync_period_seconds: 0
+
+# BRAD's front end servers will report their metrics at regular intervals.
+front_end_metrics_reporting_period_seconds: 30
+front_end_query_latency_buffer_size: 100
+
+# `default` means to use the policy encoded in the blueprint. Other values will
+# override the blueprint.
+routing_policy: default
+
+# Whether to disable table movement for benchmark purposes (i.e., keep all
+# tables on all engines.)
+disable_table_movement: true
+
+# Epoch length for metrics and forecasting. This is the granularity at which
+# metrics/forecasting will be performed.
+epoch_length:
+  weeks: 0
+  days: 0
+  hours: 0
+  minutes: 1
+
+# Blueprint planning strategy.
+strategy: fp_query_based_beam
+
+# Used to specify the period of time over which to use data for planning.
+# Currrently, this is a "look behind" window for the workload.
+planning_window:
+  weeks: 0
+  days: 0
+  hours: 1
+  minutes: 0
+
+# Used to aggregate metrics collected in the planning window.
+metrics_agg:
+  method: ewm         # 'mean' is another option
+  alpha: 0.86466472   # 1 - 1 / e^2
+
+# Used during planning.
+reinterpret_second_as: 1
+
+# The query distribution must change by at least this much for a new blueprint
+# to be accepted.
+query_dist_change_frac: 0.1
+
+# The search bound for the provisioning.
+max_provisioning_multiplier: 2.5
+
+# Flag options for blueprint planning.
+use_io_optimized_aurora: true
+use_recorded_routing_if_available: true
+ensure_tables_together_on_one_engine: true
+
+# Loads used to prime the system when no information is available.
+aurora_initialize_load_fraction: 0.25
+redshift_initialize_load_fraction: 0.25
+
+# BRAD will not reduce predicted load lower than these values. Raise these
+# values to be more conservative against mispredictions.
+aurora_min_load_removal_fraction: 0.8
+redshift_min_load_removal_fraction: 0.8
+
+# Blueprint planning performance ceilings.
+query_latency_p90_ceiling_s: 30.0
+txn_latency_p90_ceiling_s: 0.030
+
+# If set to true, BRAD will attempt to use the specified preset Redshift
+# clusters instead of resizing the main Redshift cluster.
+use_preset_redshift_clusters: false
+
+# Used for ordering blueprints during planning.
+comparator:
+  type: benefit_perf_ceiling  # or `perf_ceiling`
+
+  benefit_horizon:  # Only used by the `benefit_perf_ceiling` comparator
+    weeks: 0
+    days: 0
+    hours: 1
+    minutes: 0
+
+  penalty_threshold: 0.8  # Only used by the `benefit_perf_ceiling` comparator
+  penalty_power: 8  # Only used by the `benefit_perf_ceiling` comparator
+
+# Used for precomputed predictions.
+std_datasets:
+  - name: regular
+    path: workloads/IMDB_100GB/regular_test/
+  - name: adhoc
+    path: workloads/IMDB_100GB/adhoc_test/
+
+bootstrap_vdbe_path: config/vdbe_demo/imdb_extended_vdbes.json
+
+# Blueprint planning trigger configs.
+
+triggers:
+  enabled: false
+  check_period_s: 90  # Triggers are checked every X seconds.
+  check_period_offset_s: 360  # Wait 6 mins before starting.
+
+  # Triggers will not fire for at least this many minutes after a new blueprint
+  # takes effect. Usually this should be greater than zero to give BRAD
+  # sufficient time to observe the effect of the blueprint on the workload. BRAD
+  # may wait longer to ensure metrics are also available for this many minutes.
+  observe_new_blueprint_mins: 3
+
+  elapsed_time:
+    disabled: true
+    multiplier: 60  # Multiplier over `planning_window`.
+
+  redshift_cpu:
+    lo: 15
+    hi: 85
+    sustained_epochs: 3
+
+  aurora_cpu:
+    lo: 15
+    hi: 85
+    sustained_epochs: 3
+
+  variable_costs:
+    disabled: true
+    threshold: 1.0
+
+  query_latency_ceiling:
+    ceiling_s: 30.0
+    sustained_epochs: 3
+
+  txn_latency_ceiling:
+    ceiling_s: 0.030
+    sustained_epochs: 3
+
+  recent_change:
+    delay_epochs: 5

config/vdbe_demo/imdb_extended_vdbes.json

@@ -0,0 +1,220 @@
+{
+  "schema_name": "imdb_extended",
+  "engines": [
+    {
+      "name": "VDBE (T)",
+      "max_staleness_ms": 0,
+      "p90_latency_slo_ms": 30,
+      "interface": "postgresql",
+      "tables": [
+        {
+          "name": "theatres",
+          "writable": true
+        },
+        {
+          "name": "showings",
+          "writable": true
+        },
+        {
+          "name": "ticket_orders",
+          "writable": true
+        },
+        {
+          "name": "movie_info",
+          "writable": true
+        },
+        {
+          "name": "aka_title",
+          "writable": true
+        }
+      ],
+      "mapped_to": "aurora"
+    },
+    {
+      "name": "VDBE (A)",
+      "max_staleness_ms": 3600000,
+      "p90_latency_slo_ms": 30000,
+      "interface": "postgresql",
+      "tables": [
+        {
+          "name": "homes",
+          "writable": false
+        },
+        {
+          "name": "theatres",
+          "writable": false
+        },
+        {
+          "name": "showings",
+          "writable": false
+        },
+        {
+          "name": "ticket_orders",
+          "writable": false
+        },
+        {
+          "name": "aka_name",
+          "writable": false
+        },
+        {
+          "name": "aka_title",
+          "writable": false
+        },
+        {
+          "name": "cast_info",
+          "writable": false
+        },
+        {
+          "name": "char_name",
+          "writable": false
+        },
+        {
+          "name": "comp_cast_type",
+          "writable": false
+        },
+        {
+          "name": "company_name",
+          "writable": false
+        },
+        {
+          "name": "company_type",
+          "writable": false
+        },
+        {
+          "name": "complete_cast",
+          "writable": false
+        },
+        {
+          "name": "info_type",
+          "writable": false
+        },
+        {
+          "name": "keyword",
+          "writable": false
+        },
+        {
+          "name": "kind_type",
+          "writable": false
+        },
+        {
+          "name": "link_type",
+          "writable": false
+        },
+        {
+          "name": "movie_companies",
+          "writable": false
+        },
+        {
+          "name": "movie_info_idx",
+          "writable": false
+        },
+        {
+          "name": "movie_keyword",
+          "writable": false
+        },
+        {
+          "name": "movie_link",
+          "writable": false
+        },
+        {
+          "name": "name",
+          "writable": false
+        },
+        {
+          "name": "role_type",
+          "writable": false
+        },
+        {
+          "name": "title",
+          "writable": false
+        },
+        {
+          "name": "movie_info",
+          "writable": false
+        },
+        {
+          "name": "person_info",
+          "writable": false
+        }
+      ],
+      "mapped_to": "redshift"
+    }
+  ],
+  "tables": [
+    {
+      "name": "homes"
+    },
+    {
+      "name": "theatres"
+    },
+    {
+      "name": "showings"
+    },
+    {
+      "name": "ticket_orders"
+    },
+    {
+      "name": "aka_name"
+    },
+    {
+      "name": "aka_title"
+    },
+    {
+      "name": "cast_info"
+    },
+    {
+      "name": "char_name"
+    },
+    {
+      "name": "comp_cast_type"
+    },
+    {
+      "name": "company_name"
+    },
+    {
+      "name": "company_type"
+    },
+    {
+      "name": "complete_cast"
+    },
+    {
+      "name": "info_type"
+    },
+    {
+      "name": "keyword"
+    },
+    {
+      "name": "kind_type"
+    },
+    {
+      "name": "link_type"
+    },
+    {
+      "name": "movie_companies"
+    },
+    {
+      "name": "movie_info_idx"
+    },
+    {
+      "name": "movie_keyword"
+    },
+    {
+      "name": "movie_link"
+    },
+    {
+      "name": "name"
+    },
+    {
+      "name": "role_type"
+    },
+    {
+      "name": "title"
+    },
+    {
+      "name": "movie_info"
+    },
+    {
+      "name": "person_info"
+    }
+  ]
+}