datacube: fix type conversion for caching (#3873)

finos · Feb 7, 2025 · 95c26ec · 95c26ec
1 parent 47ca81b
commit 95c26ec
Show file tree

Hide file tree

Showing 9 changed files with 235 additions and 85 deletions.
diff --git a/.changeset/twenty-frogs-whisper.md b/.changeset/twenty-frogs-whisper.md
@@ -0,0 +1,5 @@
+---
+'@finos/legend-application-data-cube': patch
+'@finos/legend-data-cube': patch
+'@finos/legend-graph': patch
+---
diff --git a/packages/legend-application-data-cube/package.json b/packages/legend-application-data-cube/package.json
@@ -54,6 +54,7 @@
     "@finos/legend-storage": "workspace:*",
     "@types/react": "19.0.8",
     "@types/react-dom": "19.0.3",
+    "apache-arrow": "19.0.0",
     "mobx": "6.13.6",
     "mobx-react-lite": "4.1.0",
     "react": "19.0.0",

diff --git a/...es/legend-application-data-cube/src/components/query-builder/LegendDataCubeQuerySaver.tsx b/...es/legend-application-data-cube/src/components/query-builder/LegendDataCubeQuerySaver.tsx
@@ -30,7 +30,11 @@ export const LegendDataCubeQuerySaver = observer(() => {
   const builder = guaranteeNonNullable(store.builder);
 
   useEffect(() => {
-    setName(builder.persistentQuery?.name ?? DEFAULT_REPORT_NAME);
+    setName(
+      builder.persistentQuery?.name ??
+        builder.query.configuration?.name ??
+        DEFAULT_REPORT_NAME,
+    );
   }, [builder]);
 
   return (

diff --git a/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts b/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts
@@ -18,13 +18,20 @@ import * as duckdb from '@duckdb/duckdb-wasm';
 import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm';
 import duckdb_wasm_next from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm';
 import {
+  INTERNAL__TDSColumn,
   PRIMITIVE_TYPE,
+  TDSBuilder,
   TDSExecutionResult,
   TDSRow,
   TabularDataSet,
 } from '@finos/legend-graph';
-import { assertNonNullable, guaranteeNonNullable } from '@finos/legend-shared';
+import {
+  assertNonNullable,
+  guaranteeNonNullable,
+  UnsupportedOperationError,
+} from '@finos/legend-shared';
 import type { CachedDataCubeSource } from '@finos/legend-data-cube';
+import { Type } from 'apache-arrow';
 
 export class LegendDataCubeDataCubeCacheManager {
   private static readonly DUCKDB_DEFAULT_SCHEMA_NAME = 'main'; // See https://duckdb.org/docs/sql/statements/use.html
@@ -69,7 +76,7 @@ export class LegendDataCubeDataCubeCacheManager {
       `Can't initialize cache manager: DuckDB main worker not initialized`,
     );
     const worker = new Worker(bundle.mainWorker);
-    const logger = new duckdb.ConsoleLogger();
+    const logger = new duckdb.ConsoleLogger(duckdb.LogLevel.WARNING);
     const database = new duckdb.AsyncDuckDB(logger, worker);
     await database.instantiate(bundle.mainModule, bundle.pthreadWorker);
     this._database = database;
@@ -97,16 +104,31 @@ export class LegendDataCubeDataCubeCacheManager {
           break;
         }
         case PRIMITIVE_TYPE.INTEGER: {
-          colType = 'INTEGER';
+          colType = 'FLOAT';
+          break;
+        }
+        case PRIMITIVE_TYPE.DECIMAL: {
+          colType = 'DECIMAL';
+          break;
+        }
+        case PRIMITIVE_TYPE.FLOAT: {
+          colType = 'FLOAT';
           break;
         }
+        case PRIMITIVE_TYPE.STRICTDATE:
+        case PRIMITIVE_TYPE.DATETIME:
         case PRIMITIVE_TYPE.DATE: {
-          colType = 'TIMESTAMP';
+          colType = 'STRING';
           break;
         }
-        case PRIMITIVE_TYPE.STRING:
-        default: {
+        case PRIMITIVE_TYPE.STRING: {
           colType = 'VARCHAR';
+          break;
+        }
+        default: {
+          throw new UnsupportedOperationError(
+            `Can't initialize cache: failed to find matching DuckDB type for Pure type '${col.type}'`,
+          );
         }
       }
       columns.push(`"${col.name}" ${colType}`);
@@ -139,9 +161,12 @@ export class LegendDataCubeDataCubeCacheManager {
 
   async runSQLQuery(sql: string) {
     const connection = await this.database.connect();
-    const result = (await connection.query(sql)).toArray();
-    const columnNames = Object.keys(result.at(0));
-    const rows = result.map((row) => {
+    const result = await connection.query(sql);
+    await connection.close();
+
+    const data = result.toArray();
+    const columnNames = result.schema.fields.map((field) => field.name);
+    const rows = data.map((row) => {
       const values = new TDSRow();
       values.values = columnNames.map(
         (column) => row[column] as string | number | boolean | null,
@@ -153,6 +178,62 @@ export class LegendDataCubeDataCubeCacheManager {
     tds.columns = columnNames;
     tds.rows = rows;
     tdsExecutionResult.result = tds;
+    const builder = new TDSBuilder();
+    builder.columns = result.schema.fields.map((field) => {
+      const col = new INTERNAL__TDSColumn();
+      col.name = field.name;
+      switch (field.typeId) {
+        case Type.Binary: {
+          col.type = PRIMITIVE_TYPE.BINARY;
+          break;
+        }
+        case Type.Bool: {
+          col.type = PRIMITIVE_TYPE.BOOLEAN;
+          break;
+        }
+        case Type.Date:
+        case Type.DateDay:
+        case Type.DateMillisecond: {
+          col.type = PRIMITIVE_TYPE.DATE;
+          break;
+        }
+        case Type.Utf8:
+        case Type.LargeUtf8: {
+          col.type = PRIMITIVE_TYPE.STRING;
+          break;
+        }
+        case Type.Decimal: {
+          col.type = PRIMITIVE_TYPE.DECIMAL;
+          break;
+        }
+        case Type.Int:
+        case Type.Int8:
+        case Type.Uint8:
+        case Type.Int16:
+        case Type.Uint16:
+        case Type.Int32:
+        case Type.Uint32:
+        case Type.Int64:
+        case Type.Uint64: {
+          col.type = PRIMITIVE_TYPE.INTEGER;
+          break;
+        }
+        case Type.Float:
+        case Type.Float16:
+        case Type.Float32:
+        case Type.Float64: {
+          col.type = PRIMITIVE_TYPE.FLOAT;
+          break;
+        }
+        default: {
+          throw new UnsupportedOperationError(
+            `Can't find matching Pure type for Arrow type ID '${field.typeId}' ${Type.Utf8}`,
+          );
+        }
+      }
+      return col;
+    });
+    tdsExecutionResult.builder = builder;
     return tdsExecutionResult;
   }
 

diff --git a/packages/legend-application-data-cube/src/stores/LegendDataCubeDataCubeEngine.ts b/packages/legend-application-data-cube/src/stores/LegendDataCubeDataCubeEngine.ts
@@ -48,7 +48,6 @@ import {
   V1_deserializeExecutionPlan,
   V1_SQLExecutionNode,
   V1_SimpleExecutionPlan,
-  V1_Binary,
   V1_ClassInstance,
   V1_ClassInstanceType,
   V1_Column,
@@ -63,15 +62,18 @@ import {
   V1_PackageableRuntime,
   V1_PureModelContextData,
   V1_RelationStoreAccessor,
-  type V1_RelationalDataType,
   V1_RelationalDatabaseConnection,
   V1_Schema,
   V1_StoreConnections,
   V1_Table,
   V1_TestAuthenticationStrategy,
   V1_VarChar,
+  V1_Bit,
+  V1_Decimal,
+  V1_Float,
   PackageableElementPointerType,
   DatabaseType,
+  PRIMITIVE_TYPE,
 } from '@finos/legend-graph';
 import {
   _elementPtr,
@@ -536,7 +538,43 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     table.columns = result.result.builder.columns.map((col) => {
       const column = new V1_Column();
       column.name = col.name;
-      column.type = this._getColumnType(col.type);
+      switch (col.type as string) {
+        case PRIMITIVE_TYPE.BOOLEAN: {
+          column.type = new V1_Bit();
+          break;
+        }
+        case PRIMITIVE_TYPE.NUMBER: {
+          column.type = new V1_Double();
+          break;
+        }
+        case PRIMITIVE_TYPE.INTEGER: {
+          column.type = new V1_Integer();
+          break;
+        }
+        case PRIMITIVE_TYPE.FLOAT: {
+          column.type = new V1_Float();
+          break;
+        }
+        case PRIMITIVE_TYPE.DECIMAL: {
+          column.type = new V1_Decimal();
+          break;
+        }
+        case PRIMITIVE_TYPE.DATE:
+        case PRIMITIVE_TYPE.STRICTDATE:
+        case PRIMITIVE_TYPE.DATETIME: {
+          column.type = new V1_Date();
+          break;
+        }
+        case PRIMITIVE_TYPE.STRING: {
+          column.type = new V1_VarChar();
+          break;
+        }
+        default: {
+          throw new UnsupportedOperationError(
+            `Can't initialize cache: failed to find matching relational data type for Pure type '${col.type}' when synthesizing table definition`,
+          );
+        }
+      }
       return column;
     });
 
@@ -596,27 +634,6 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     return cachedSource;
   }
 
-  // TODO: need a better way to infer datatype from tds builder
-  private _getColumnType(type: string | undefined): V1_RelationalDataType {
-    if (type === undefined) {
-      throw Error('Unsupported data type');
-    }
-    switch (type) {
-      case 'string':
-        return new V1_VarChar();
-      case 'integer':
-        return new V1_Integer();
-      case 'date':
-        return new V1_Date();
-      case 'boolean':
-        return new V1_Binary();
-      case 'number':
-        return new V1_Double();
-      default:
-        return new V1_VarChar();
-    }
-  }
-
   override async disposeCache(source: CachedDataCubeSource) {
     await this._cacheManager.disposeCache(source);
   }

diff --git a/packages/legend-data-cube/src/components/DataCubeTitleBar.tsx b/packages/legend-data-cube/src/components/DataCubeTitleBar.tsx
@@ -35,9 +35,9 @@ export const DataCubeTitleBar = observer(
 
     return (
       <div className="flex h-7 justify-between bg-neutral-100">
-        <div className="flex select-none items-center pl-1 pr-2 text-lg font-medium">
-          <DataCubeIcon.Cube className="mr-1 h-4 w-4" />
-          <div>{title}</div>
+        <div className="flex items-center overflow-hidden pl-1 pr-2 text-lg font-medium">
+          <DataCubeIcon.Cube className="mr-1 h-4 w-4 flex-shrink-0" />
+          <div className="text-ellipsis whitespace-nowrap">{title}</div>
         </div>
         <div className="flex">
           {children}

diff --git a/packages/legend-graph/src/STO_Relational_Exports.ts b/packages/legend-graph/src/STO_Relational_Exports.ts
@@ -87,6 +87,7 @@ export {
   V1_serializeMapper,
   V1_deserializeMapper,
 } from './graph-manager/protocol/pure/v1/transformation/pureProtocol/serializationHelpers/V1_PostProcessorSerializationHelper.js';
+export * from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/model/V1_RelationalDataType.js';
 export * from './graph-manager/extensions/STO_Relational_PureGraphManagerPlugin_Extension.js';
 
 export * from './graph/metamodel/pure/data/RelationalCSVData.js';

diff --git a/packages/legend-graph/src/index.ts b/packages/legend-graph/src/index.ts
@@ -220,14 +220,6 @@ export { V1_Database } from './graph-manager/protocol/pure/v1/model/packageableE
 export { V1_Schema } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/model/V1_Schema.js';
 export { V1_Table } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/model/V1_Table.js';
 export { V1_Column } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/model/V1_Column.js';
-export {
-  V1_Binary,
-  V1_Date,
-  V1_Double,
-  V1_Integer,
-  V1_RelationalDataType,
-  V1_VarChar,
-} from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/model/V1_RelationalDataType.js';
 export { V1_RelationalDatabaseConnection } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/connection/V1_RelationalDatabaseConnection.js';
 export { V1_DuckDBDatasourceSpecification } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/connection/V1_DatasourceSpecification.js';
 export { V1_TestAuthenticationStrategy } from './graph-manager/protocol/pure/v1/model/packageableElements/store/relational/connection/V1_AuthenticationStrategy.js';