diff --git a/config/vdbe_demo/imdb_extended_vdbes.json b/config/vdbe_demo/imdb_extended_vdbes.json new file mode 100644 index 00000000..ab28d850 --- /dev/null +++ b/config/vdbe_demo/imdb_extended_vdbes.json @@ -0,0 +1,218 @@ +{ + "schema_name": "imdb_extended", + "engines": [ + { + "name": "VDBE [T]", + "max_staleness_ms": 0, + "p90_latency_slo_ms": 30, + "interface": "postgresql", + "tables": [ + { + "name": "theatres", + "writable": true + }, + { + "name": "showings", + "writable": true + }, + { + "name": "ticket_orders", + "writable": true + }, + { + "name": "movie_info", + "writable": true + }, + { + "name": "aka_title", + "writable": true + } + ] + }, + { + "name": "VDBE [A]", + "max_staleness_ms": 0, + "p90_latency_slo_ms": 30000, + "interface": "postgresql", + "tables": [ + { + "name": "homes", + "writable": false + }, + { + "name": "theatres", + "writable": false + }, + { + "name": "showings", + "writable": false + }, + { + "name": "ticket_orders", + "writable": false + }, + { + "name": "aka_name", + "writable": false + }, + { + "name": "aka_title", + "writable": false + }, + { + "name": "cast_info", + "writable": false + }, + { + "name": "char_name", + "writable": false + }, + { + "name": "comp_cast_type", + "writable": false + }, + { + "name": "company_name", + "writable": false + }, + { + "name": "company_type", + "writable": false + }, + { + "name": "complete_cast", + "writable": false + }, + { + "name": "info_type", + "writable": false + }, + { + "name": "keyword", + "writable": false + }, + { + "name": "kind_type", + "writable": false + }, + { + "name": "link_type", + "writable": false + }, + { + "name": "movie_companies", + "writable": false + }, + { + "name": "movie_info_idx", + "writable": false + }, + { + "name": "movie_keyword", + "writable": false + }, + { + "name": "movie_link", + "writable": false + }, + { + "name": "name", + "writable": false + }, + { + "name": "role_type", + "writable": false + }, + { + "name": "title", + "writable": false + }, + { + "name": "movie_info", + "writable": false + }, + { + "name": "person_info", + "writable": false + } + ] + } + ], + "tables": [ + { + "name": "homes" + }, + { + "name": "theatres" + }, + { + "name": "showings" + }, + { + "name": "ticket_orders" + }, + { + "name": "aka_name" + }, + { + "name": "aka_title" + }, + { + "name": "cast_info" + }, + { + "name": "char_name" + }, + { + "name": "comp_cast_type" + }, + { + "name": "company_name" + }, + { + "name": "company_type" + }, + { + "name": "complete_cast" + }, + { + "name": "info_type" + }, + { + "name": "keyword" + }, + { + "name": "kind_type" + }, + { + "name": "link_type" + }, + { + "name": "movie_companies" + }, + { + "name": "movie_info_idx" + }, + { + "name": "movie_keyword" + }, + { + "name": "movie_link" + }, + { + "name": "name" + }, + { + "name": "role_type" + }, + { + "name": "title" + }, + { + "name": "movie_info" + }, + { + "name": "person_info" + } + ] +} diff --git a/setup.py b/setup.py index c321b6d7..34653397 100644 --- a/setup.py +++ b/setup.py @@ -45,6 +45,7 @@ "ddsketch", "tqdm", "psycopg[binary]", + "pydantic", ] DEV_REQUIRES = [ @@ -68,7 +69,6 @@ UI_REQUIRES = [ "fastapi", "uvicorn[standard]", - "pydantic", "requests", "types-requests", ] diff --git a/src/brad/vdbe/__init__.py b/src/brad/vdbe/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/brad/vdbe/models.py b/src/brad/vdbe/models.py new file mode 100644 index 00000000..14965d50 --- /dev/null +++ b/src/brad/vdbe/models.py @@ -0,0 +1,35 @@ +import enum +from typing import List +from pydantic import BaseModel + +# This is a simple implementation of a Virtual Database Engine (VDBE) metadata +# model meant for demonstration purposes only. + + +class VirtualTable(BaseModel): + name: str + writable: bool + + +class SchemaTable(BaseModel): + name: str + + +class QueryInterface(enum.Enum): + Common = "common" + PostgreSQL = "postgresql" + Athena = "athena" + + +class VirtualEngine(BaseModel): + name: str + max_staleness_ms: int + p90_latency_slo_ms: int + interface: QueryInterface + tables: List[VirtualTable] + + +class VirtualInfrastructure(BaseModel): + schema_name: str + engines: List[VirtualEngine] + tables: List[SchemaTable] diff --git a/tools/serialize_vdbes.py b/tools/serialize_vdbes.py new file mode 100644 index 00000000..7eeaf17a --- /dev/null +++ b/tools/serialize_vdbes.py @@ -0,0 +1,72 @@ +import argparse +import yaml +from typing import Any, Dict +from brad.vdbe.models import ( + VirtualInfrastructure, + VirtualEngine, + VirtualTable, + SchemaTable, + QueryInterface, +) + + +# Define your virtual infrastructure here. +def to_serialize(schema: Dict[str, Any]) -> VirtualInfrastructure: + all_table_names = [tbl["table_name"] for tbl in schema["tables"]] + t_tables = [ + VirtualTable(name=name, writable=True) + for name in [ + "theatres", + "showings", + "ticket_orders", + "movie_info", + "aka_title", + ] + ] + a_tables = [VirtualTable(name=name, writable=False) for name in all_table_names] + t_engine = VirtualEngine( + name="VDBE [T]", + max_staleness_ms=0, + p90_latency_slo_ms=30, + interface=QueryInterface.PostgreSQL, + tables=t_tables, + ) + a_engine = VirtualEngine( + name="VDBE [A]", + max_staleness_ms=0, + p90_latency_slo_ms=30 * 1000, + interface=QueryInterface.PostgreSQL, + tables=a_tables, + ) + return VirtualInfrastructure( + schema_name=schema["schema_name"], + engines=[t_engine, a_engine], + tables=[SchemaTable(name=name) for name in all_table_names], + ) + + +def main(): + parser = argparse.ArgumentParser( + description="Tool used to serialize VDBE defintions." + ) + parser.add_argument("--out-file", type=str, help="Output file path.", required=True) + parser.add_argument("--compact", action="store_true", help="Compact JSON output.") + parser.add_argument( + "--schema-file", type=str, required=True, help="Schema file path." + ) + args = parser.parse_args() + + with open(args.schema_file, "r", encoding="utf-8") as f: + schema = yaml.load(f, Loader=yaml.Loader) + + infra = to_serialize(schema) + indent = None if args.compact else 2 + out_str = infra.model_dump_json(indent=indent) + + with open(args.out_file, "w", encoding="utf-8") as f: + f.write(out_str) + f.write("\n") + + +if __name__ == "__main__": + main()