diff --git a/config/system_config_demo.yml b/config/system_config_demo.yml index b2f3e0a6..720a4d7f 100644 --- a/config/system_config_demo.yml +++ b/config/system_config_demo.yml @@ -124,6 +124,8 @@ std_datasets: - name: adhoc path: workloads/IMDB_100GB/adhoc_test/ +bootstrap_vdbe_path: config/vdbe_demo/imdb_extended_vdbes.json + # Blueprint planning trigger configs. triggers: diff --git a/config/vdbe_demo/imdb_extended_vdbes.json b/config/vdbe_demo/imdb_extended_vdbes.json index ab28d850..f37a78cc 100644 --- a/config/vdbe_demo/imdb_extended_vdbes.json +++ b/config/vdbe_demo/imdb_extended_vdbes.json @@ -2,7 +2,7 @@ "schema_name": "imdb_extended", "engines": [ { - "name": "VDBE [T]", + "name": "VDBE (T)", "max_staleness_ms": 0, "p90_latency_slo_ms": 30, "interface": "postgresql", @@ -27,11 +27,12 @@ "name": "aka_title", "writable": true } - ] + ], + "mapped_to": "aurora" }, { - "name": "VDBE [A]", - "max_staleness_ms": 0, + "name": "VDBE (A)", + "max_staleness_ms": 3600000, "p90_latency_slo_ms": 30000, "interface": "postgresql", "tables": [ @@ -135,7 +136,8 @@ "name": "person_info", "writable": false } - ] + ], + "mapped_to": "redshift" } ], "tables": [ diff --git a/src/brad/config/file.py b/src/brad/config/file.py index fe781c23..7c26a0cf 100644 --- a/src/brad/config/file.py +++ b/src/brad/config/file.py @@ -294,6 +294,12 @@ def result_row_limit(self) -> Optional[int]: except KeyError: return None + def bootstrap_vdbe_path(self) -> Optional[pathlib.Path]: + try: + return pathlib.Path(self._raw["bootstrap_vdbe_path"]) + except KeyError: + return None + def _extract_log_path(self, config_key: str) -> Optional[pathlib.Path]: if config_key not in self._raw: return None diff --git a/src/brad/daemon/daemon.py b/src/brad/daemon/daemon.py index 56045c7b..1c37d947 100644 --- a/src/brad/daemon/daemon.py +++ b/src/brad/daemon/daemon.py @@ -70,6 +70,7 @@ from brad.row_list import RowList from brad.utils.time_periods import period_start, universal_now from brad.ui.manager import UiManager +from brad.vdbe.manager import VdbeManager logger = logging.getLogger(__name__) @@ -128,17 +129,30 @@ def __init__( self._system_event_logger = SystemEventLogger.create_if_requested(self._config) self._watchdog = BlueprintWatchdog(self._system_event_logger) + load_vdbe_path = self._config.bootstrap_vdbe_path() + if load_vdbe_path is not None: + self._vdbe_manager: Optional[VdbeManager] = VdbeManager.load_from( + load_vdbe_path + ) + else: + self._vdbe_manager = None + # This is used to hold references to internal command tasks we create. # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task self._internal_command_tasks: Set[asyncio.Task] = set() self._startup_timestamp = universal_now() - if self._start_ui and UiManager.is_supported(): + if ( + self._start_ui + and UiManager.is_supported() + and self._vdbe_manager is not None + ): self._ui_mgr: Optional[UiManager] = UiManager.create( self._config, self._monitor, self._blueprint_mgr, + self._vdbe_manager, self._system_event_logger, ) else: @@ -148,6 +162,10 @@ def __init__( "Cannot start the BRAD UI because it is not supported. " "Please make sure you install BRAD with the [ui] option." ) + if self._vdbe_manager is None: + logger.warning( + "Cannot start the BRAD UI because a VDBE definition is not available." + ) async def run_forever(self) -> None: """ diff --git a/src/brad/ui/manager.py b/src/brad/ui/manager.py index c9ec9ade..dfafda31 100644 --- a/src/brad/ui/manager.py +++ b/src/brad/ui/manager.py @@ -5,6 +5,7 @@ from brad.blueprint.manager import BlueprintManager from brad.daemon.system_event_logger import SystemEventLogger from brad.planner.abstract import BlueprintPlanner +from brad.vdbe.manager import VdbeManager class UiManager: @@ -33,11 +34,14 @@ def create( config: ConfigFile, monitor: Monitor, blueprint_mgr: BlueprintManager, + vdbe_mgr: VdbeManager, system_event_logger: Optional[SystemEventLogger], ) -> "UiManager": from brad.ui.manager_impl import UiManagerImpl - return cls(UiManagerImpl(config, monitor, blueprint_mgr, system_event_logger)) + return cls( + UiManagerImpl(config, monitor, blueprint_mgr, vdbe_mgr, system_event_logger) + ) # We hide away the implementation details to allow external code to import # `UiManager` without worrying about import errors (e.g., because the diff --git a/src/brad/ui/manager_impl.py b/src/brad/ui/manager_impl.py index 760fce03..b3854a29 100644 --- a/src/brad/ui/manager_impl.py +++ b/src/brad/ui/manager_impl.py @@ -22,15 +22,13 @@ TimestampedMetrics, DisplayableBlueprint, SystemState, - DisplayableVirtualEngine, - VirtualInfrastructure, - DisplayableTable, Status, ClientState, SetClientState, ) from brad.daemon.front_end_metrics import FrontEndMetric from brad.daemon.system_event_logger import SystemEventLogger, SystemEventRecord +from brad.vdbe.manager import VdbeManager logger = logging.getLogger(__name__) @@ -41,11 +39,13 @@ def __init__( config: ConfigFile, monitor: Monitor, blueprint_mgr: BlueprintManager, + vdbe_mgr: VdbeManager, system_event_logger: Optional[SystemEventLogger], ) -> None: self.config = config self.monitor = monitor self.blueprint_mgr = blueprint_mgr + self.vdbe_mgr = vdbe_mgr self.system_event_logger = system_event_logger self.planner: Optional[BlueprintPlanner] = None @@ -109,7 +109,6 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState: # TODO: Hardcoded virtualized infrasturcture and writers. txn_tables = ["theatres", "showings", "ticket_orders", "movie_info", "aka_title"] - txn_only = ["theatres", "showings", "ticket_orders"] if filter_tables_for_demo: # To improve how the UI looks in a screenshot, we filter out some tables @@ -134,47 +133,7 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState: ) dbp = DisplayableBlueprint.from_blueprint(blueprint) - vdbe1 = DisplayableVirtualEngine( - name="VDBE 1", - freshness="No staleness (SI)", - dialect="PostgreSQL SQL", - peak_latency_s=0.030, - tables=[ - DisplayableTable(name=name, is_writer=True, mapped_to=["Aurora"]) - for name in [ - "theatres", - "showings", - "ticket_orders", - "movie_info", - "aka_title", - ] - ], - ) - vdbe1.tables.sort(key=lambda t: t.name) - vdbe2 = DisplayableVirtualEngine( - name="VDBE 2", - freshness="≤ 10 minutes stale (SI)", - dialect="PostgreSQL SQL", - peak_latency_s=30.0, - tables=[ - DisplayableTable( - name=table.name, - is_writer=False, - mapped_to=_analytics_table_mapper_temp(table.name, blueprint), - ) - for table in blueprint.tables() - if table.name not in txn_only - ], - ) - vdbe2.tables.sort(key=lambda t: t.name) - for engine in dbp.engines: - if engine.name != "Aurora": - continue - for t in engine.tables: - if t.name in txn_tables: - t.is_writer = True - virtual_infra = VirtualInfrastructure(engines=[vdbe1, vdbe2]) - + virtual_infra = manager.vdbe_mgr.infra() status = _determine_current_status(manager) if status is Status.Transitioning: next_blueprint = manager.blueprint_mgr.get_transition_metadata().next_blueprint @@ -188,7 +147,6 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState: blueprint=dbp, next_blueprint=next_dbp, ) - _add_reverse_mapping_temp(system_state) return system_state diff --git a/src/brad/ui/models.py b/src/brad/ui/models.py index 7389b663..276a7d2e 100644 --- a/src/brad/ui/models.py +++ b/src/brad/ui/models.py @@ -4,6 +4,7 @@ from brad.blueprint import Blueprint from brad.config.engine import Engine +from brad.vdbe.models import VirtualInfrastructure class TimestampedMetrics(BaseModel): @@ -84,18 +85,6 @@ def from_blueprint(cls, blueprint: Blueprint) -> "DisplayableBlueprint": return cls(engines=engines) -class DisplayableVirtualEngine(BaseModel): - name: str - freshness: str - dialect: str - peak_latency_s: Optional[float] = None - tables: List[DisplayableTable] = [] - - -class VirtualInfrastructure(BaseModel): - engines: List[DisplayableVirtualEngine] - - class Status(enum.Enum): Running = "running" Planning = "planning" diff --git a/src/brad/vdbe/models.py b/src/brad/vdbe/models.py index 14965d50..42f644da 100644 --- a/src/brad/vdbe/models.py +++ b/src/brad/vdbe/models.py @@ -2,6 +2,8 @@ from typing import List from pydantic import BaseModel +from brad.config.engine import Engine + # This is a simple implementation of a Virtual Database Engine (VDBE) metadata # model meant for demonstration purposes only. @@ -27,6 +29,7 @@ class VirtualEngine(BaseModel): p90_latency_slo_ms: int interface: QueryInterface tables: List[VirtualTable] + mapped_to: Engine class VirtualInfrastructure(BaseModel): diff --git a/tools/serialize_vdbes.py b/tools/serialize_vdbes.py index 7eeaf17a..363c1aa3 100644 --- a/tools/serialize_vdbes.py +++ b/tools/serialize_vdbes.py @@ -8,6 +8,7 @@ SchemaTable, QueryInterface, ) +from brad.config.engine import Engine # Define your virtual infrastructure here. @@ -25,18 +26,20 @@ def to_serialize(schema: Dict[str, Any]) -> VirtualInfrastructure: ] a_tables = [VirtualTable(name=name, writable=False) for name in all_table_names] t_engine = VirtualEngine( - name="VDBE [T]", + name="VDBE (T)", max_staleness_ms=0, p90_latency_slo_ms=30, interface=QueryInterface.PostgreSQL, tables=t_tables, + mapped_to=Engine.Aurora, ) a_engine = VirtualEngine( - name="VDBE [A]", - max_staleness_ms=0, + name="VDBE (A)", + max_staleness_ms=60 * 60 * 1000, # 1 hour p90_latency_slo_ms=30 * 1000, interface=QueryInterface.PostgreSQL, tables=a_tables, + mapped_to=Engine.Redshift, ) return VirtualInfrastructure( schema_name=schema["schema_name"], diff --git a/ui/src/components/PhysDbView.jsx b/ui/src/components/PhysDbView.jsx index 509106a6..06eda4f8 100644 --- a/ui/src/components/PhysDbView.jsx +++ b/ui/src/components/PhysDbView.jsx @@ -47,11 +47,11 @@ function PhysDbView({ )}
- {sortedTables.map(({ name, is_writer, mapped_to }) => ( + {sortedTables.map(({ name, writable, mapped_to }) => ( ))} - {addedTablesList.map(({ name, is_writer }) => ( + {addedTablesList.map(({ name, writable }) => ( {}} diff --git a/ui/src/components/VdbeView.jsx b/ui/src/components/VdbeView.jsx index 63f12c02..3f23ea20 100644 --- a/ui/src/components/VdbeView.jsx +++ b/ui/src/components/VdbeView.jsx @@ -9,29 +9,51 @@ import { } from "../highlight"; import { useState, useCallback } from "react"; -function formatLatencySeconds(latencySeconds) { - const precision = 1; - if (latencySeconds < 1.0) { +function formatMilliseconds(milliseconds) { + const precision = 2; + if (milliseconds >= 1000 * 60 * 60) { + // Use hours. + const latencyHours = milliseconds / (1000 * 60 * 60); + return `${latencyHours.toFixed(precision)} hr`; + } else if (milliseconds >= 1000) { // Use milliseconds. - const latencyMs = latencySeconds * 1000; - return `${latencyMs.toFixed(precision)} ms`; + const latencySeconds = milliseconds / 1000; + return `${latencySeconds.toFixed(precision)} s`; + } + return `${milliseconds} ms`; +} + +function formatFreshness(maxStalenessMs) { + if (maxStalenessMs === 0) { + return "No staleness"; + } + return `Staleness ≤ ${formatMilliseconds(maxStalenessMs)}`; +} + +function formatDialect(queryInterface) { + if (queryInterface === "postgresql") { + return "PostgreSQL SQL"; + } else if (queryInterface === "athena") { + return "Athena SQL"; + } else if (queryInterface === "common") { + return "SQL-99"; } - return `${latencySeconds.toFixed(precision)} s`; } function VdbeView({ - name, - freshness, - dialect, - peak_latency_s, - tables, + vdbe, highlight, onTableHoverEnter, onTableHoverExit, workloadState, updateWorkloadNumClients, }) { - const vengName = name; + const vengName = vdbe.name; + const tables = vdbe.tables; + const freshness = formatFreshness(vdbe.max_staleness_ms); + const peakLatency = formatMilliseconds(vdbe.p90_latency_slo_ms); + const dialect = formatDialect(vdbe.interface); + const sortedTables = sortTablesToHoist(highlight, vengName, true, tables); const [showWorkloadAdjuster, setShowWorkloadAdjuster] = useState(false); @@ -58,18 +80,16 @@ function VdbeView({
  • 🌿: {freshness}
    • - {peak_latency_s && ( -
  • ⏱️: Query Latency ≤ {formatLatencySeconds(peak_latency_s)}
    • - )} +
  • ⏱️: p90 Query Latency ≤ {peakLatency}
  • 🗣: {dialect}
- {sortedTables.map(({ name, is_writer, mapped_to }) => ( + {sortedTables.map(({ name, writable, mapped_to }) => (
- {virtualInfra?.engines?.map(({ name, ...props }, index) => ( + {virtualInfra?.engines?.map((vdbe, index) => ( updateWorkloadNumClients(index, numClients) } - {...props} + vdbe={vdbe} /> ))}
diff --git a/ui/vite.config.js b/ui/vite.config.js index 9cc50ead..201bbada 100644 --- a/ui/vite.config.js +++ b/ui/vite.config.js @@ -1,7 +1,10 @@ import { defineConfig } from "vite"; import react from "@vitejs/plugin-react"; -// https://vitejs.dev/config/ -export default defineConfig({ +// https://vite.dev/config/ +export default defineConfig(({ mode }) => ({ plugins: [react()], -}); + build: { + minify: mode === "production", + }, +}));