Display VDBE state given by the daemon

config/system_config_demo.yml

@@ -124,6 +124,8 @@ std_datasets:
   - name: adhoc
     path: workloads/IMDB_100GB/adhoc_test/
 
+bootstrap_vdbe_path: config/vdbe_demo/imdb_extended_vdbes.json
+
 # Blueprint planning trigger configs.
 
 triggers:

config/vdbe_demo/imdb_extended_vdbes.json

@@ -2,7 +2,7 @@
   "schema_name": "imdb_extended",
   "engines": [
     {
-      "name": "VDBE [T]",
+      "name": "VDBE (T)",
       "max_staleness_ms": 0,
       "p90_latency_slo_ms": 30,
       "interface": "postgresql",
@@ -27,11 +27,12 @@
           "name": "aka_title",
           "writable": true
         }
-      ]
+      ],
+      "mapped_to": "aurora"
     },
     {
-      "name": "VDBE [A]",
-      "max_staleness_ms": 0,
+      "name": "VDBE (A)",
+      "max_staleness_ms": 3600000,
       "p90_latency_slo_ms": 30000,
       "interface": "postgresql",
       "tables": [
@@ -135,7 +136,8 @@
           "name": "person_info",
           "writable": false
         }
-      ]
+      ],
+      "mapped_to": "redshift"
     }
   ],
   "tables": [

src/brad/config/file.py

@@ -294,6 +294,12 @@ def result_row_limit(self) -> Optional[int]:
         except KeyError:
             return None
 
+    def bootstrap_vdbe_path(self) -> Optional[pathlib.Path]:
+        try:
+            return pathlib.Path(self._raw["bootstrap_vdbe_path"])
+        except KeyError:
+            return None
+
     def _extract_log_path(self, config_key: str) -> Optional[pathlib.Path]:
         if config_key not in self._raw:
             return None

src/brad/daemon/daemon.py

@@ -70,6 +70,7 @@
 from brad.row_list import RowList
 from brad.utils.time_periods import period_start, universal_now
 from brad.ui.manager import UiManager
+from brad.vdbe.manager import VdbeManager
 
 logger = logging.getLogger(__name__)
 
@@ -128,17 +129,30 @@ def __init__(
         self._system_event_logger = SystemEventLogger.create_if_requested(self._config)
         self._watchdog = BlueprintWatchdog(self._system_event_logger)
 
+        load_vdbe_path = self._config.bootstrap_vdbe_path()
+        if load_vdbe_path is not None:
+            self._vdbe_manager: Optional[VdbeManager] = VdbeManager.load_from(
+                load_vdbe_path
+            )
+        else:
+            self._vdbe_manager = None
+
         # This is used to hold references to internal command tasks we create.
         # https://docs.python.org/3/library/asyncio-task.html#asyncio.create_task
         self._internal_command_tasks: Set[asyncio.Task] = set()
 
         self._startup_timestamp = universal_now()
 
-        if self._start_ui and UiManager.is_supported():
+        if (
+            self._start_ui
+            and UiManager.is_supported()
+            and self._vdbe_manager is not None
+        ):
             self._ui_mgr: Optional[UiManager] = UiManager.create(
                 self._config,
                 self._monitor,
                 self._blueprint_mgr,
+                self._vdbe_manager,
                 self._system_event_logger,
             )
         else:
@@ -148,6 +162,10 @@ def __init__(
                     "Cannot start the BRAD UI because it is not supported. "
                     "Please make sure you install BRAD with the [ui] option."
                 )
+            if self._vdbe_manager is None:
+                logger.warning(
+                    "Cannot start the BRAD UI because a VDBE definition is not available."
+                )
 
     async def run_forever(self) -> None:
         """

src/brad/ui/manager.py

@@ -5,6 +5,7 @@
 from brad.blueprint.manager import BlueprintManager
 from brad.daemon.system_event_logger import SystemEventLogger
 from brad.planner.abstract import BlueprintPlanner
+from brad.vdbe.manager import VdbeManager
 
 
 class UiManager:
@@ -33,11 +34,14 @@ def create(
         config: ConfigFile,
         monitor: Monitor,
         blueprint_mgr: BlueprintManager,
+        vdbe_mgr: VdbeManager,
         system_event_logger: Optional[SystemEventLogger],
     ) -> "UiManager":
         from brad.ui.manager_impl import UiManagerImpl
 
-        return cls(UiManagerImpl(config, monitor, blueprint_mgr, system_event_logger))
+        return cls(
+            UiManagerImpl(config, monitor, blueprint_mgr, vdbe_mgr, system_event_logger)
+        )
 
     # We hide away the implementation details to allow external code to import
     # `UiManager` without worrying about import errors (e.g., because the

src/brad/ui/manager_impl.py

@@ -22,15 +22,13 @@
     TimestampedMetrics,
     DisplayableBlueprint,
     SystemState,
-    DisplayableVirtualEngine,
-    VirtualInfrastructure,
-    DisplayableTable,
     Status,
     ClientState,
     SetClientState,
 )
 from brad.daemon.front_end_metrics import FrontEndMetric
 from brad.daemon.system_event_logger import SystemEventLogger, SystemEventRecord
+from brad.vdbe.manager import VdbeManager
 
 logger = logging.getLogger(__name__)
 
@@ -41,11 +39,13 @@ def __init__(
         config: ConfigFile,
         monitor: Monitor,
         blueprint_mgr: BlueprintManager,
+        vdbe_mgr: VdbeManager,
         system_event_logger: Optional[SystemEventLogger],
     ) -> None:
         self.config = config
         self.monitor = monitor
         self.blueprint_mgr = blueprint_mgr
+        self.vdbe_mgr = vdbe_mgr
         self.system_event_logger = system_event_logger
         self.planner: Optional[BlueprintPlanner] = None
 
@@ -109,7 +109,6 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState:
 
     # TODO: Hardcoded virtualized infrasturcture and writers.
     txn_tables = ["theatres", "showings", "ticket_orders", "movie_info", "aka_title"]
-    txn_only = ["theatres", "showings", "ticket_orders"]
 
     if filter_tables_for_demo:
         # To improve how the UI looks in a screenshot, we filter out some tables
@@ -134,47 +133,7 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState:
         )
 
     dbp = DisplayableBlueprint.from_blueprint(blueprint)
-    vdbe1 = DisplayableVirtualEngine(
-        name="VDBE 1",
-        freshness="No staleness (SI)",
-        dialect="PostgreSQL SQL",
-        peak_latency_s=0.030,
-        tables=[
-            DisplayableTable(name=name, is_writer=True, mapped_to=["Aurora"])
-            for name in [
-                "theatres",
-                "showings",
-                "ticket_orders",
-                "movie_info",
-                "aka_title",
-            ]
-        ],
-    )
-    vdbe1.tables.sort(key=lambda t: t.name)
-    vdbe2 = DisplayableVirtualEngine(
-        name="VDBE 2",
-        freshness="≤ 10 minutes stale (SI)",
-        dialect="PostgreSQL SQL",
-        peak_latency_s=30.0,
-        tables=[
-            DisplayableTable(
-                name=table.name,
-                is_writer=False,
-                mapped_to=_analytics_table_mapper_temp(table.name, blueprint),
-            )
-            for table in blueprint.tables()
-            if table.name not in txn_only
-        ],
-    )
-    vdbe2.tables.sort(key=lambda t: t.name)
-    for engine in dbp.engines:
-        if engine.name != "Aurora":
-            continue
-        for t in engine.tables:
-            if t.name in txn_tables:
-                t.is_writer = True
-    virtual_infra = VirtualInfrastructure(engines=[vdbe1, vdbe2])
-
+    virtual_infra = manager.vdbe_mgr.infra()
     status = _determine_current_status(manager)
     if status is Status.Transitioning:
         next_blueprint = manager.blueprint_mgr.get_transition_metadata().next_blueprint
@@ -188,7 +147,6 @@ def get_system_state(filter_tables_for_demo: bool = False) -> SystemState:
         blueprint=dbp,
         next_blueprint=next_dbp,
     )
-    _add_reverse_mapping_temp(system_state)
     return system_state
 
 

src/brad/ui/models.py

@@ -4,6 +4,7 @@
 
 from brad.blueprint import Blueprint
 from brad.config.engine import Engine
+from brad.vdbe.models import VirtualInfrastructure
 
 
 class TimestampedMetrics(BaseModel):
@@ -84,18 +85,6 @@ def from_blueprint(cls, blueprint: Blueprint) -> "DisplayableBlueprint":
         return cls(engines=engines)
 
 
-class DisplayableVirtualEngine(BaseModel):
-    name: str
-    freshness: str
-    dialect: str
-    peak_latency_s: Optional[float] = None
-    tables: List[DisplayableTable] = []
-
-
-class VirtualInfrastructure(BaseModel):
-    engines: List[DisplayableVirtualEngine]
-
-
 class Status(enum.Enum):
     Running = "running"
     Planning = "planning"

src/brad/vdbe/models.py

@@ -2,6 +2,8 @@
 from typing import List
 from pydantic import BaseModel
 
+from brad.config.engine import Engine
+
 # This is a simple implementation of a Virtual Database Engine (VDBE) metadata
 # model meant for demonstration purposes only.
 
@@ -27,6 +29,7 @@ class VirtualEngine(BaseModel):
     p90_latency_slo_ms: int
     interface: QueryInterface
     tables: List[VirtualTable]
+    mapped_to: Engine
 
 
 class VirtualInfrastructure(BaseModel):

tools/serialize_vdbes.py

@@ -8,6 +8,7 @@
     SchemaTable,
     QueryInterface,
 )
+from brad.config.engine import Engine
 
 
 # Define your virtual infrastructure here.
@@ -25,18 +26,20 @@ def to_serialize(schema: Dict[str, Any]) -> VirtualInfrastructure:
     ]
     a_tables = [VirtualTable(name=name, writable=False) for name in all_table_names]
     t_engine = VirtualEngine(
-        name="VDBE [T]",
+        name="VDBE (T)",
         max_staleness_ms=0,
         p90_latency_slo_ms=30,
         interface=QueryInterface.PostgreSQL,
         tables=t_tables,
+        mapped_to=Engine.Aurora,
     )
     a_engine = VirtualEngine(
-        name="VDBE [A]",
-        max_staleness_ms=0,
+        name="VDBE (A)",
+        max_staleness_ms=60 * 60 * 1000,  # 1 hour
         p90_latency_slo_ms=30 * 1000,
         interface=QueryInterface.PostgreSQL,
         tables=a_tables,
+        mapped_to=Engine.Redshift,
     )
     return VirtualInfrastructure(
         schema_name=schema["schema_name"],

ui/src/components/PhysDbView.jsx

@@ -47,11 +47,11 @@ function PhysDbView({
         </div>
       )}
       <div class="db-table-set">
-        {sortedTables.map(({ name, is_writer, mapped_to }) => (
+        {sortedTables.map(({ name, writable, mapped_to }) => (
           <TableView
             key={name}
             name={name}
-            isWriter={is_writer}
+            isWriter={writable}
             color="blue"
             highlightClass={highlightTableViewClass(
               highlight,
@@ -65,11 +65,11 @@ function PhysDbView({
             onTableHoverExit={onTableHoverExit}
           />
         ))}
-        {addedTablesList.map(({ name, is_writer }) => (
+        {addedTablesList.map(({ name, writable }) => (
           <TableView
             key={name}
             name={name}
-            isWriter={is_writer}
+            isWriter={writable}
             color="blue"
             highlightClass="dim"
             onTableHoverEnter={() => {}}