Add new interface definitions (#514)

config/vdbes/quickflix.yml

@@ -0,0 +1,50 @@
+schema_name: quickflix
+
+vdbes:
+  - name: order_flow
+    tables:
+      - name: tickets
+        writable: true
+      - name: customers
+        writable: true
+    max_staleness_ms: 0
+    query_interface: SQL_POSTGRESQL
+
+  - name: transform
+    tables:
+      - name: tickets
+      - name: customers
+      - name: sales_history
+        writable: true
+    max_staleness_ms: 3600000  # 1 hour
+    query_interface: SQL_AWS_REDSHIFT
+
+  - name: analysis
+    tables:
+      - name: sales_history
+    max_staleness_ms: 0
+    query_interface: SQL_AWS_REDSHIFT
+
+tables:
+  - name: tickets
+    columns:
+      - name: t_customer_id
+        type: INT_64
+      - name: t_movie_id
+        type: INT_64
+
+  - name: customers
+    columns:
+      - name: c_customer_id
+        type: INT_64
+      - name: c_name
+        type: STRING
+
+  - name: sales_history
+    columns:
+      - name: sh_customer_id
+        type: INT_64
+      - name: sh_movie_id
+        type: INT_64
+      - name: sh_name
+        type: STRING

proto/interface.proto

@@ -0,0 +1,20 @@
+// This file is the entrypoint for the virtual database engine (VDBE) interface
+// definitions. We have split the definitions across multiple files to make them
+// managable to read.
+
+syntax = "proto3";
+
+package vdbe;
+
+import "interface/blueprint.proto";
+import "interface/schema.proto";
+import "interface/vdbe.proto";
+
+// All the information needed to describe a data infrastructure deployment.
+message SystemState {
+  // Used to uniquely identify the system state.
+  string schema_name = 1;
+  repeated Table tables = 2;
+  repeated VirtualEngine vdbes = 3;
+  Blueprint blueprint = 4;
+}

proto/interface/blueprint.proto

@@ -0,0 +1,43 @@
+// This file contains definitions for the blueprint (physical realization of a
+// set of VDBEs).
+
+syntax = "proto3";
+
+package vdbe;
+
+message Blueprint {
+  Provisioning aurora = 101;
+  Provisioning redshift = 102;
+  RoutingPolicy policy = 201;
+  repeated PhysicalSnapshot snapshots = 301;
+}
+
+message RoutingPolicy {
+  // A serialized form of the routing policy.
+  bytes policy = 1;
+}
+
+message PhysicalSnapshot {
+  // The VDBEs that are mapped to this physical snapshot.
+  repeated string vdbes = 1;
+  // The tables in this physical snapshot.
+  repeated string tables = 2;
+  // Where the data in this physical snapshot resides.
+  Engine location = 3;
+}
+
+// Used to indicate the location of data.
+enum Engine {
+  ENGINE_UNKNOWN = 0;
+  // The data is stored in Aurora.
+  ENGINE_AURORA = 1;
+  // The data is stored in Redshift.
+  ENGINE_REDSHIFT = 2;
+  // The data is stored on S3 in an Iceberg table and registered with Athena.
+  ENGINE_ATHENA = 3;
+}
+
+message Provisioning {
+  string instance_type = 1;
+  uint32 num_nodes = 2;
+}

proto/interface/schema.proto

@@ -0,0 +1,26 @@
+// This file contains definitions for table schema information.
+
+syntax = "proto3";
+
+package vdbe;
+
+// Represents a relational table.
+message Table {
+  string name = 1;
+  repeated TableColumn columns = 2;
+}
+
+// Represents a column in a relational table.
+message TableColumn {
+  string name = 1;
+  DataType type = 2;
+  bool nullable = 3;
+}
+
+// The data types we currently support.
+enum DataType {
+  DT_UNKNOWN = 0;
+  DT_INT_32 = 1;
+  DT_INT_64 = 2;
+  DT_STRING = 3;
+}

proto/interface/vdbe.proto

@@ -0,0 +1,33 @@
+// This file contains definitions for virtual database engines (VDBEs).
+
+syntax = "proto3";
+
+package vdbe;
+
+message VirtualEngine {
+  string name = 1;
+  QueryInterface qi = 2;
+  repeated VirtualTable tables = 3;
+  uint64 max_staleness_ms = 4;
+  // TODO: Meaningful representation for performance properties.
+  // TODO: Flag set for feature support.
+}
+
+message VirtualTable {
+  // Name must reference a table that has a schema definition.
+  string name = 1;
+  bool writable = 2;
+}
+
+// These are the query interfaces we currently support.
+enum QueryInterface {
+  QI_UNKNOWN = 0;
+
+  // 1xx - Open source SQL DBMSes.
+  QI_SQL_POSTGRESQL = 101;
+  QI_SQL_MYSQL = 102;
+
+  // 2xx - AWS SQL DBMSes.
+  QI_SQL_AWS_REDSHIFT = 201;
+  QI_SQL_AWS_ATHENA = 202;
+}

src/brad/proto_gen/interface/blueprint_pb2.pyi

@@ -0,0 +1,54 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class Engine(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = []  # type: ignore
+    ENGINE_UNKNOWN: _ClassVar[Engine]
+    ENGINE_AURORA: _ClassVar[Engine]
+    ENGINE_REDSHIFT: _ClassVar[Engine]
+    ENGINE_ATHENA: _ClassVar[Engine]
+ENGINE_UNKNOWN: Engine
+ENGINE_AURORA: Engine
+ENGINE_REDSHIFT: Engine
+ENGINE_ATHENA: Engine
+
+class Blueprint(_message.Message):
+    __slots__ = ["aurora", "redshift", "policy", "snapshots"]
+    AURORA_FIELD_NUMBER: _ClassVar[int]
+    REDSHIFT_FIELD_NUMBER: _ClassVar[int]
+    POLICY_FIELD_NUMBER: _ClassVar[int]
+    SNAPSHOTS_FIELD_NUMBER: _ClassVar[int]
+    aurora: Provisioning
+    redshift: Provisioning
+    policy: RoutingPolicy
+    snapshots: _containers.RepeatedCompositeFieldContainer[PhysicalSnapshot]
+    def __init__(self, aurora: _Optional[_Union[Provisioning, _Mapping]] = ..., redshift: _Optional[_Union[Provisioning, _Mapping]] = ..., policy: _Optional[_Union[RoutingPolicy, _Mapping]] = ..., snapshots: _Optional[_Iterable[_Union[PhysicalSnapshot, _Mapping]]] = ...) -> None: ...
+
+class RoutingPolicy(_message.Message):
+    __slots__ = ["policy"]
+    POLICY_FIELD_NUMBER: _ClassVar[int]
+    policy: bytes
+    def __init__(self, policy: _Optional[bytes] = ...) -> None: ...
+
+class PhysicalSnapshot(_message.Message):
+    __slots__ = ["vdbes", "tables", "location"]
+    VDBES_FIELD_NUMBER: _ClassVar[int]
+    TABLES_FIELD_NUMBER: _ClassVar[int]
+    LOCATION_FIELD_NUMBER: _ClassVar[int]
+    vdbes: _containers.RepeatedScalarFieldContainer[str]
+    tables: _containers.RepeatedScalarFieldContainer[str]
+    location: Engine
+    def __init__(self, vdbes: _Optional[_Iterable[str]] = ..., tables: _Optional[_Iterable[str]] = ..., location: _Optional[_Union[Engine, str]] = ...) -> None: ...
+
+class Provisioning(_message.Message):
+    __slots__ = ["instance_type", "num_nodes"]
+    INSTANCE_TYPE_FIELD_NUMBER: _ClassVar[int]
+    NUM_NODES_FIELD_NUMBER: _ClassVar[int]
+    instance_type: str
+    num_nodes: int
+    def __init__(self, instance_type: _Optional[str] = ..., num_nodes: _Optional[int] = ...) -> None: ...

src/brad/proto_gen/interface/schema_pb2.pyi

@@ -0,0 +1,36 @@
+from google.protobuf.internal import containers as _containers
+from google.protobuf.internal import enum_type_wrapper as _enum_type_wrapper
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import message as _message
+from typing import ClassVar as _ClassVar, Iterable as _Iterable, Mapping as _Mapping, Optional as _Optional, Union as _Union
+
+DESCRIPTOR: _descriptor.FileDescriptor
+
+class DataType(int, metaclass=_enum_type_wrapper.EnumTypeWrapper):
+    __slots__ = []  # type: ignore
+    DT_UNKNOWN: _ClassVar[DataType]
+    DT_INT_32: _ClassVar[DataType]
+    DT_INT_64: _ClassVar[DataType]
+    DT_STRING: _ClassVar[DataType]
+DT_UNKNOWN: DataType
+DT_INT_32: DataType
+DT_INT_64: DataType
+DT_STRING: DataType
+
+class Table(_message.Message):
+    __slots__ = ["name", "columns"]
+    NAME_FIELD_NUMBER: _ClassVar[int]
+    COLUMNS_FIELD_NUMBER: _ClassVar[int]
+    name: str
+    columns: _containers.RepeatedCompositeFieldContainer[TableColumn]
+    def __init__(self, name: _Optional[str] = ..., columns: _Optional[_Iterable[_Union[TableColumn, _Mapping]]] = ...) -> None: ...
+
+class TableColumn(_message.Message):
+    __slots__ = ["name", "type", "nullable"]
+    NAME_FIELD_NUMBER: _ClassVar[int]
+    TYPE_FIELD_NUMBER: _ClassVar[int]
+    NULLABLE_FIELD_NUMBER: _ClassVar[int]
+    name: str
+    type: DataType
+    nullable: bool
+    def __init__(self, name: _Optional[str] = ..., type: _Optional[_Union[DataType, str]] = ..., nullable: bool = ...) -> None: ...