datacube: improve UX around caching (#3868)

* datacube: improve UX around caching

* datacube: passing clientVersion to initializeCache

* datacube: force VX_X_X for generatePlan in caching flow

.changeset/kind-badgers-exercise.md

@@ -0,0 +1,6 @@
+---
+'@finos/legend-application-pure-ide': patch
+'@finos/legend-application-studio': patch
+'@finos/legend-data-cube': patch
+'@finos/legend-shared': patch
+---

.changeset/seven-eyes-begin.md

@@ -0,0 +1,8 @@
+---
+'@finos/legend-application-data-cube': patch
+'@finos/legend-application-pure-ide': patch
+'@finos/legend-application-studio': patch
+'@finos/legend-query-builder': patch
+'@finos/legend-data-cube': patch
+'@finos/legend-shared': patch
+---

packages/legend-application-data-cube/src/stores/LegendDataCubeDataCubeEngine.ts

@@ -85,8 +85,9 @@ import {
   RawAdhocQueryDataCubeSource,
   _lambda,
   _defaultPrimitiveTypeValue,
-  type DataCubeExecutionOptions,
   CachedDataCubeSource,
+  type DataCubeExecutionOptions,
+  type DataCubeCacheInitializationOptions,
 } from '@finos/legend-data-cube';
 import {
   isNonNullable,
@@ -430,11 +431,16 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
       );
     } else if (source instanceof CachedDataCubeSource) {
       // get the execute plan to extract the generated SQL to run against cached DB
-      const executionPlan = await this.generateExecutionPlan(
+      const executionPlan = await this._generateExecutionPlan(
         query,
         source.model,
         [],
-        options,
+        // NOTE: for caching, we're using DuckDB, but its protocol models
+        // are not available in the latest production protocol version V1_33_0, so
+        // we have to force using VX_X_X
+        // once we either cut another protocol version or backport the DuckDB models
+        // to V1_33_0, we will can remove this
+        { ...options, clientVersion: PureClientVersion.VX_X_X },
       );
       const sql = guaranteeNonNullable(
         executionPlan instanceof V1_SimpleExecutionPlan
@@ -500,98 +506,11 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     return undefined;
   }
 
-  // ---------------------------------- UTILITIES ----------------------------------
-
-  private async _getQueryRelationType(
-    query: PlainObject<V1_Lambda>,
-    source: DataCubeSource,
-  ) {
-    if (source instanceof AdhocQueryDataCubeSource) {
-      return this._getLambdaRelationType(query, source.model);
-    } else if (source instanceof LegendQueryDataCubeSource) {
-      return this._getLambdaRelationType(query, source.model);
-    } else if (source instanceof CachedDataCubeSource) {
-      return this._getLambdaRelationType(query, serialize(source.model));
-    }
-    throw new UnsupportedOperationError(
-      `Can't get relation type for lambda with unsupported source`,
-    );
-  }
-
-  private async _getLambdaRelationType(
-    lambda: PlainObject<V1_Lambda>,
-    model: PlainObject<V1_PureModelContext>,
-  ) {
-    const relationType = deserialize(
-      V1_relationTypeModelSchema,
-      await this._engineServerClient.lambdaRelationType({
-        lambda,
-        model,
-      }),
-    );
-    return {
-      columns: relationType.columns.map((column) => ({
-        name: column.name,
-        type: V1_getGenericTypeFullPath(column.genericType),
-      })),
-    };
-  }
-
-  private async _runQuery(
-    query: V1_Lambda,
-    model: PlainObject<V1_PureModelContext>,
-    parameterValues?: V1_ParameterValue[] | undefined,
-    options?: DataCubeExecutionOptions | undefined,
-  ): Promise<ExecutionResult> {
-    return V1_buildExecutionResult(
-      V1_deserializeExecutionResult(
-        (await this._engineServerClient.runQuery({
-          clientVersion:
-            options?.clientVersion ??
-            // eslint-disable-next-line no-process-env
-            (process.env.NODE_ENV === 'development'
-              ? PureClientVersion.VX_X_X
-              : undefined),
-          function: this.serializeValueSpecification(query),
-          model,
-          context: serialize(
-            V1_rawBaseExecutionContextModelSchema,
-            new V1_RawBaseExecutionContext(),
-          ),
-          parameterValues: (parameterValues ?? []).map((parameterValue) =>
-            serialize(V1_parameterValueModelSchema, parameterValue),
-          ),
-        })) as PlainObject<V1_ExecutionResult>,
-      ),
-    );
-  }
-
-  private async generateExecutionPlan(
-    query: V1_Lambda,
-    model: V1_PureModelContext,
-    parameterValues?: V1_ParameterValue[] | undefined,
-    options?: DataCubeExecutionOptions | undefined,
-  ): Promise<V1_ExecutionPlan> {
-    return V1_deserializeExecutionPlan(
-      await this._engineServerClient.generatePlan({
-        clientVersion: PureClientVersion.VX_X_X,
-        function: this.serializeValueSpecification(query),
-        model: serialize(model),
-        context: serialize(
-          V1_rawBaseExecutionContextModelSchema,
-          new V1_RawBaseExecutionContext(),
-        ),
-        parameterValues: (parameterValues ?? []).map((parameterValue) =>
-          serialize(V1_parameterValueModelSchema, parameterValue),
-        ),
-      }),
-    );
-  }
-
   // ---------------------------------- CACHING --------------------------------------
 
   override async initializeCache(
     source: DataCubeSource,
+    options?: DataCubeCacheInitializationOptions | undefined,
   ): Promise<CachedDataCubeSource | undefined> {
     const cacheQuery = guaranteeNonNullable(
       this.buildExecutionContext(source),
@@ -601,7 +520,7 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     const result = await this.executeQuery(
       _lambda([], [cacheQuery]),
       source,
-      undefined,
+      options,
     );
     const {
       schema: schemaName,
@@ -677,10 +596,6 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     return cachedSource;
   }
 
-  override async disposeCache(source: CachedDataCubeSource) {
-    await this._cacheManager.disposeCache(source);
-  }
-
   // TODO: need a better way to infer datatype from tds builder
   private _getColumnType(type: string | undefined): V1_RelationalDataType {
     if (type === undefined) {
@@ -702,6 +617,103 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
     }
   }
 
+  override async disposeCache(source: CachedDataCubeSource) {
+    await this._cacheManager.disposeCache(source);
+  }
+
+  // ---------------------------------- UTILITIES ----------------------------------
+
+  private async _getQueryRelationType(
+    query: PlainObject<V1_Lambda>,
+    source: DataCubeSource,
+  ) {
+    if (source instanceof AdhocQueryDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof LegendQueryDataCubeSource) {
+      return this._getLambdaRelationType(query, source.model);
+    } else if (source instanceof CachedDataCubeSource) {
+      return this._getLambdaRelationType(query, serialize(source.model));
+    }
+    throw new UnsupportedOperationError(
+      `Can't get relation type for lambda with unsupported source`,
+    );
+  }
+
+  private async _getLambdaRelationType(
+    lambda: PlainObject<V1_Lambda>,
+    model: PlainObject<V1_PureModelContext>,
+  ) {
+    const relationType = deserialize(
+      V1_relationTypeModelSchema,
+      await this._engineServerClient.lambdaRelationType({
+        lambda,
+        model,
+      }),
+    );
+    return {
+      columns: relationType.columns.map((column) => ({
+        name: column.name,
+        type: V1_getGenericTypeFullPath(column.genericType),
+      })),
+    };
+  }
+
+  private async _runQuery(
+    query: V1_Lambda,
+    model: PlainObject<V1_PureModelContext>,
+    parameterValues?: V1_ParameterValue[] | undefined,
+    options?: DataCubeExecutionOptions | undefined,
+  ): Promise<ExecutionResult> {
+    return V1_buildExecutionResult(
+      V1_deserializeExecutionResult(
+        (await this._engineServerClient.runQuery({
+          clientVersion:
+            options?.clientVersion ??
+            // eslint-disable-next-line no-process-env
+            (process.env.NODE_ENV === 'development'
+              ? PureClientVersion.VX_X_X
+              : undefined),
+          function: this.serializeValueSpecification(query),
+          model,
+          context: serialize(
+            V1_rawBaseExecutionContextModelSchema,
+            new V1_RawBaseExecutionContext(),
+          ),
+          parameterValues: (parameterValues ?? []).map((parameterValue) =>
+            serialize(V1_parameterValueModelSchema, parameterValue),
+          ),
+        })) as PlainObject<V1_ExecutionResult>,
+      ),
+    );
+  }
+
+  private async _generateExecutionPlan(
+    query: V1_Lambda,
+    model: V1_PureModelContext,
+    parameterValues?: V1_ParameterValue[] | undefined,
+    options?: DataCubeExecutionOptions | undefined,
+  ): Promise<V1_ExecutionPlan> {
+    return V1_deserializeExecutionPlan(
+      await this._engineServerClient.generatePlan({
+        clientVersion:
+          options?.clientVersion ??
+          // eslint-disable-next-line no-process-env
+          (process.env.NODE_ENV === 'development'
+            ? PureClientVersion.VX_X_X
+            : undefined),
+        function: this.serializeValueSpecification(query),
+        model: serialize(model),
+        context: serialize(
+          V1_rawBaseExecutionContextModelSchema,
+          new V1_RawBaseExecutionContext(),
+        ),
+        parameterValues: (parameterValues ?? []).map((parameterValue) =>
+          serialize(V1_parameterValueModelSchema, parameterValue),
+        ),
+      }),
+    );
+  }
+
   // ---------------------------------- APPLICATION ----------------------------------
 
   override logDebug(message: string, ...data: unknown[]) {

packages/legend-application-pure-ide/src/components/editor-group/GenericFileEditor.tsx

@@ -21,11 +21,7 @@ import type { FileEditorState } from '../../stores/FileEditorState.js';
 import { useApplicationStore, useCommands } from '@finos/legend-application';
 import { clsx, Dialog, WordWrapIcon } from '@finos/legend-art';
 import { usePureIDEStore } from '../PureIDEStoreProvider.js';
-import {
-  last,
-  guaranteeNonNullable,
-  returnUndefOnError,
-} from '@finos/legend-shared';
+import { at, returnUndefOnError } from '@finos/legend-shared';
 import {
   CODE_EDITOR_THEME,
   getBaseCodeEditorOptions,
@@ -42,8 +38,8 @@ const getPositionFromGoToLinePromptInputValue = (
     return [1, undefined];
   }
   return [
-    returnUndefOnError(() => parseInt(guaranteeNonNullable(parts[0]))) ?? 1,
-    returnUndefOnError(() => parseInt(guaranteeNonNullable(last(parts)))),
+    returnUndefOnError(() => parseInt(at(parts, 0))) ?? 1,
+    returnUndefOnError(() => parseInt(at(parts, -1))),
   ];
 };
 

packages/legend-application-pure-ide/src/stores/FileEditorState.ts

@@ -26,12 +26,7 @@ import {
   CODE_EDITOR_LANGUAGE,
 } from '@finos/legend-code-editor';
 import { DIRECTORY_PATH_DELIMITER } from '@finos/legend-graph';
-import {
-  assertErrorThrown,
-  last,
-  guaranteeNonNullable,
-  type GeneratorFn,
-} from '@finos/legend-shared';
+import { at, assertErrorThrown, type GeneratorFn } from '@finos/legend-shared';
 import {
   action,
   computed,
@@ -63,7 +58,7 @@ import { LEGEND_PURE_IDE_PURE_FILE_EDITOR_COMMAND_KEY } from '../__lib__/LegendP
 import type { TabState } from '@finos/legend-lego/application';
 
 const getFileEditorLanguage = (filePath: string): string => {
-  const extension = last(filePath.split('.'));
+  const extension = filePath.split('.').at(-1);
   switch (extension) {
     case 'pure':
       return CODE_EDITOR_LANGUAGE.PURE;
@@ -230,9 +225,7 @@ export class FileEditorState
   }
 
   get fileName(): string {
-    return guaranteeNonNullable(
-      last(this.filePath.split(DIRECTORY_PATH_DELIMITER)),
-    );
+    return at(this.filePath.split(DIRECTORY_PATH_DELIMITER), -1);
   }
 
   override match(tab: TabState): boolean {

packages/legend-application-studio/src/components/editor/panel-group/SQLPlaygroundPanel.tsx

@@ -76,7 +76,6 @@ import {
 } from '@finos/legend-lego/data-grid';
 import {
   at,
-  last,
   isNonNullable,
   isNumber,
   isString,
@@ -303,7 +302,7 @@ const PlaygroundSQLCodeEditor = observer(() => {
           const lines = currentValue.split('\n');
           const position = playgroundState.sqlEditor.getPosition() ?? {
             lineNumber: lines.length,
-            column: last(lines)?.length ?? 0,
+            column: lines.at(-1)?.length ?? 0,
           };
           playgroundState.sqlEditor.executeEdits('', [
             {

packages/legend-application-studio/src/stores/editor/panel-group/SQLPlaygroundPanelState.ts

@@ -18,7 +18,6 @@ import {
   type GeneratorFn,
   assertErrorThrown,
   LogEvent,
-  last,
   ActionState,
 } from '@finos/legend-shared';
 import { observable, makeObservable, flow, flowResult, action } from 'mobx';
@@ -118,7 +117,7 @@ export class SQLPlaygroundPanelState implements CommandRegistrar {
       const lines = this.sqlText.split('\n');
       moveCursorToPosition(val, {
         lineNumber: lines.length,
-        column: last(lines)?.length ?? 0,
+        column: lines.at(-1)?.length ?? 0,
       });
     }
   }

packages/legend-data-cube/src/__lib__/DataCubeSetting.ts

@@ -20,6 +20,7 @@ export enum DataCubeSettingKey {
   DEBUGGER__ACTION__RELOAD = 'dataCube.debugger.action.reload',
   GRID_CLIENT__ROW_BUFFER = 'dataCube.grid.rowBuffer',
   GRID_CLIENT__PURGE_CLOSED_ROW_NODES = 'dataCube.grid.purgeClosedRowNodes',
+  GRID_CLIENT__SHOW_CACHE_PERFORMANCE_WARNING = 'dataCube.grid.showCachePerformanceWarning',
   GRID_CLIENT__SUPPRESS_LARGE_DATASET_WARNING = 'dataCube.grid.suppressLargeDatasetWarning',
   GRID_CLIENT__ACTION__RETRY_FAILED_DATA_FETCHES = 'dataCube.grid.action.retryFailedDataFetches',
 }