Skip to content

Commit

Permalink
datacube: improve debugging around execution
Browse files Browse the repository at this point in the history
  • Loading branch information
akphi committed Feb 14, 2025
1 parent e127942 commit ff1b3b9
Show file tree
Hide file tree
Showing 6 changed files with 316 additions and 100 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ import {
CachedDataCubeSource,
type DataCubeExecutionOptions,
type DataCubeCacheInitializationOptions,
DataCubeExecutionError,
} from '@finos/legend-data-cube';
import {
isNonNullable,
Expand Down Expand Up @@ -418,68 +419,81 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
const queryCodePromise = this.getValueSpecificationCode(query);
let result: ExecutionResult;
const startTime = performance.now();
if (source instanceof AdhocQueryDataCubeSource) {
result = await this._runQuery(query, source.model, undefined, options);
} else if (source instanceof LegendQueryDataCubeSource) {
query.parameters = source.lambda.parameters;
result = await this._runQuery(
query,
source.model,
source.parameterValues,
options,
);
} else if (source instanceof CachedDataCubeSource) {
// get the execute plan to extract the generated SQL to run against cached DB
const executionPlan = await this._generateExecutionPlan(
query,
source.model,
[],
// NOTE: for caching, we're using DuckDB, but its protocol models
// are not available in the latest production protocol version V1_33_0, so
// we have to force using VX_X_X
// once we either cut another protocol version or backport the DuckDB models
// to V1_33_0, we will can remove this
{ ...options, clientVersion: PureClientVersion.VX_X_X },

try {
if (source instanceof AdhocQueryDataCubeSource) {
result = await this._runQuery(query, source.model, undefined, options);
} else if (source instanceof LegendQueryDataCubeSource) {
query.parameters = source.lambda.parameters;
result = await this._runQuery(
query,
source.model,
source.parameterValues,
options,
);
} else if (source instanceof CachedDataCubeSource) {
// get the execute plan to extract the generated SQL to run against cached DB
const executionPlan = await this._generateExecutionPlan(
query,
source.model,
[],
// NOTE: for caching, we're using DuckDB, but its protocol models
// are not available in the latest production protocol version V1_33_0, so
// we have to force using VX_X_X
// once we either cut another protocol version or backport the DuckDB models
// to V1_33_0, we will can remove this
{ ...options, clientVersion: PureClientVersion.VX_X_X },
);
const sql = guaranteeNonNullable(
executionPlan instanceof V1_SimpleExecutionPlan
? executionPlan.rootExecutionNode.executionNodes
.filter(filterByType(V1_SQLExecutionNode))
.at(-1)?.sqlQuery
: undefined,
`Can't process execution plan: failed to extract generated SQL`,
);
const endTime = performance.now();
return {
executedQuery: await queryCodePromise,
executedSQL: sql,
result: await this._cacheManager.runSQLQuery(sql),
executionTime: endTime - startTime,
};
} else {
throw new UnsupportedOperationError(
`Can't execute query with unsupported source`,
);
}
assertType(
result,
TDSExecutionResult,
`Can't process execution result: expected tabular data set format`,
);
const endTime = performance.now();
const queryCode = await queryCodePromise;
const sql = guaranteeNonNullable(
executionPlan instanceof V1_SimpleExecutionPlan
? executionPlan.rootExecutionNode.executionNodes
.filter(filterByType(V1_SQLExecutionNode))
.at(-1)?.sqlQuery
result.activities?.[0] instanceof RelationalExecutionActivities
? result.activities[0].sql
: undefined,
`Can't process execution plan: failed to extract generated SQL`,
`Can't process execution result: failed to extract generated SQL`,
);
const endTime = performance.now();
return {
executedQuery: await queryCodePromise,
result: result,
executedQuery: queryCode,
executedSQL: sql,
result: await this._cacheManager.runSQLQuery(sql),
executionTime: endTime - startTime,
};
} else {
throw new UnsupportedOperationError(
`Can't execute query with unsupported source`,
);
} catch (error) {
assertErrorThrown(error);
if (error instanceof DataCubeExecutionError) {
try {
error.queryCode = await this.getValueSpecificationCode(query, true);
} catch {
// ignore
}
}
throw error;
}
assertType(
result,
TDSExecutionResult,
`Can't process execution result: expected tabular data set format`,
);
const endTime = performance.now();
const queryCode = await queryCodePromise;
const sql = guaranteeNonNullable(
result.activities?.[0] instanceof RelationalExecutionActivities
? result.activities[0].sql
: undefined,
`Can't process execution result: failed to extract generated SQL`,
);
return {
result: result,
executedQuery: queryCode,
executedSQL: sql,
executionTime: endTime - startTime,
};
}

override buildExecutionContext(source: DataCubeSource) {
Expand Down Expand Up @@ -672,27 +686,37 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
parameterValues?: V1_ParameterValue[] | undefined,
options?: DataCubeExecutionOptions | undefined,
): Promise<ExecutionResult> {
return V1_buildExecutionResult(
V1_deserializeExecutionResult(
(await this._engineServerClient.runQuery({
clientVersion:
options?.clientVersion ??
// eslint-disable-next-line no-process-env
(process.env.NODE_ENV === 'development'
? PureClientVersion.VX_X_X
: undefined),
function: this.serializeValueSpecification(query),
model,
context: serialize(
V1_rawBaseExecutionContextModelSchema,
new V1_RawBaseExecutionContext(),
),
parameterValues: (parameterValues ?? []).map((parameterValue) =>
serialize(V1_parameterValueModelSchema, parameterValue),
),
})) as PlainObject<V1_ExecutionResult>,
const input = {
clientVersion:
options?.clientVersion ??
// eslint-disable-next-line no-process-env
(process.env.NODE_ENV === 'development'
? PureClientVersion.VX_X_X
: undefined),
function: this.serializeValueSpecification(query),
model,
context: serialize(
V1_rawBaseExecutionContextModelSchema,
new V1_RawBaseExecutionContext(),
),
);
parameterValues: (parameterValues ?? []).map((parameterValue) =>
serialize(V1_parameterValueModelSchema, parameterValue),
),
};
try {
return V1_buildExecutionResult(
V1_deserializeExecutionResult(
(await this._engineServerClient.runQuery(
input,
)) as PlainObject<V1_ExecutionResult>,
),
);
} catch (err) {
assertErrorThrown(err);
const error = new DataCubeExecutionError(err.message);
error.executeInput = input;
throw error;
}
}

private async _generateExecutionPlan(
Expand All @@ -701,25 +725,33 @@ export class LegendDataCubeDataCubeEngine extends DataCubeEngine {
parameterValues?: V1_ParameterValue[] | undefined,
options?: DataCubeExecutionOptions | undefined,
): Promise<V1_ExecutionPlan> {
return V1_deserializeExecutionPlan(
await this._engineServerClient.generatePlan({
clientVersion:
options?.clientVersion ??
// eslint-disable-next-line no-process-env
(process.env.NODE_ENV === 'development'
? PureClientVersion.VX_X_X
: undefined),
function: this.serializeValueSpecification(query),
model,
context: serialize(
V1_rawBaseExecutionContextModelSchema,
new V1_RawBaseExecutionContext(),
),
parameterValues: (parameterValues ?? []).map((parameterValue) =>
serialize(V1_parameterValueModelSchema, parameterValue),
),
}),
);
const input = {
clientVersion:
options?.clientVersion ??
// eslint-disable-next-line no-process-env
(process.env.NODE_ENV === 'development'
? PureClientVersion.VX_X_X
: undefined),
function: this.serializeValueSpecification(query),
model,
context: serialize(
V1_rawBaseExecutionContextModelSchema,
new V1_RawBaseExecutionContext(),
),
parameterValues: (parameterValues ?? []).map((parameterValue) =>
serialize(V1_parameterValueModelSchema, parameterValue),
),
};
try {
return V1_deserializeExecutionPlan(
await this._engineServerClient.generatePlan(input),
);
} catch (err) {
assertErrorThrown(err);
const error = new DataCubeExecutionError(err.message);
error.executeInput = input;
throw error;
}
}

// ---------------------------------- APPLICATION ----------------------------------
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/**
* Copyright (c) 2020-present, Goldman Sachs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import { DataCubeIcon } from '@finos/legend-art';
import { CODE_EDITOR_LANGUAGE } from '@finos/legend-code-editor';
import { DATE_TIME_FORMAT } from '@finos/legend-graph';
import {
ContentType,
downloadFileUsingDataURI,
formatDate,
} from '@finos/legend-shared';
import type { DataCubeExecutionError } from '../../stores/core/DataCubeEngine.js';
import {
FormButton,
FormCheckbox,
FormCodeEditor,
} from './DataCubeFormUtils.js';
import { useState } from 'react';

export function DataCubeExecutionErrorAlert(props: {
error: DataCubeExecutionError;
message: string;
text?: string | undefined;
onClose: () => void;
}) {
const { error, message, text, onClose } = props;
const [showDebugInfo, setShowDebugInfo] = useState(false);
const queryCode = error.queryCode;
const executeInput = error.executeInput;
let prompt = undefined;
if (executeInput !== undefined && queryCode !== undefined) {
prompt = `Check the execute input and the query code below to debug or report issue`;
} else if (queryCode !== undefined) {
prompt = `Check the query code below to debug or report issue`;
} else if (executeInput !== undefined) {
prompt = `Check the execute input below to debug or report issue`;
}

return (
<>
<div className="h-[calc(100%_-_40px)] w-full pt-2">
<div className="h-full w-full overflow-auto">
<div className="flex w-full p-6 pb-4">
<div className="mr-3">
<DataCubeIcon.AlertError className="flex-shrink-0 stroke-[0.5px] text-[40px] text-red-500" />
</div>
<div>
<div className="whitespace-break-spaces text-lg">{message}</div>
<div className="mt-1 whitespace-break-spaces text-neutral-500">
{text}
</div>
</div>
</div>
{showDebugInfo && (
<>
<div className="h-[1px] w-full bg-neutral-300" />
{prompt !== undefined && (
<div className="pl-5 pt-1">{prompt}</div>
)}
{queryCode !== undefined && (
<div className="h-40 justify-center px-4 pt-1">
<div className="h-full w-full">
<FormCodeEditor
value={queryCode}
isReadOnly={true}
title="Query Code"
language={CODE_EDITOR_LANGUAGE.PURE}
hidePadding={true}
/>
</div>
</div>
)}
{executeInput !== undefined && (
<div className="h-40 justify-center px-4 pt-1">
<div className="h-full w-full">
<FormCodeEditor
value={JSON.stringify(executeInput, null, 2)}
isReadOnly={true}
title="Execute Input"
language={CODE_EDITOR_LANGUAGE.JSON}
hidePadding={true}
/>
</div>
</div>
)}
</>
)}
</div>
</div>
<div className="flex h-10 items-center justify-between border border-t-neutral-300 px-2">
<div className="flex h-full items-center pl-1">
<FormCheckbox
label="Show debug info?"
checked={showDebugInfo}
onChange={() => setShowDebugInfo(!showDebugInfo)}
/>
</div>
<div className="flex">
<FormButton onClick={onClose}>OK</FormButton>
{showDebugInfo && (
<FormButton
className="ml-2"
onClick={() =>
downloadFileUsingDataURI(
`DEBUG__ExecuteInput__${formatDate(
new Date(Date.now()),
DATE_TIME_FORMAT,
)}.json`,
JSON.stringify(executeInput, null, 2),
ContentType.APPLICATION_JSON,
)
}
>
Download Execute Input
</FormButton>
)}
</div>
</div>
</>
);
}
Original file line number Diff line number Diff line change
Expand Up @@ -776,7 +776,7 @@ export const FormCodeEditor: React.FC<{
return (
<div className="h-full w-full border border-neutral-200">
{!hideActionBar && (
<div className="flex h-5 w-full items-center justify-between border-b border-neutral-200">
<div className="flex h-5 w-full items-center justify-between border-b border-neutral-200 bg-white">
<div className="pl-1 text-sm text-neutral-500">{title ?? ''}</div>
<button
tabIndex={-1}
Expand Down
Loading

0 comments on commit ff1b3b9

Please sign in to comment.