diff --git a/.changeset/tidy-ads-judge.md b/.changeset/tidy-ads-judge.md new file mode 100644 index 0000000000..eba4897e82 --- /dev/null +++ b/.changeset/tidy-ads-judge.md @@ -0,0 +1,6 @@ +--- +'@finos/legend-application-data-cube': patch +'@finos/legend-shared': patch +--- + +Fixing Caching for CSV diff --git a/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts b/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts index 03a6dc08a2..c1c9ce01be 100644 --- a/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts +++ b/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts @@ -27,6 +27,7 @@ import { } from '@finos/legend-graph'; import { assertNonNullable, + csvStringify, guaranteeNonNullable, UnsupportedOperationError, } from '@finos/legend-shared'; @@ -92,32 +93,22 @@ export class LegendDataCubeDataCubeCacheManager { const connection = await this.database.connect(); - const columnString = result.builder.columns - .map((col) => col.name) - .join(','); + const columnNames: string[] = []; + result.builder.columns.forEach((col) => columnNames.push(col.name)); - const dataString: string[] = [columnString]; + const data = result.result.rows.map((row) => row.values); - result.result.rows.forEach((row) => { - const updatedRows = row.values.map((val) => { - if (val !== null && typeof val === 'string') { - return `'${val.replaceAll(`'`, `''`)}'`; - } else if (val === null) { - return `NULL`; - } - return val; - }); - dataString.push(`${updatedRows.join(',')}`); + const csv = csvStringify([columnNames, ...data], { + escapeChar: `'`, + quoteChar: `'`, }); - const csvString = dataString.join('\n'); - - await this._database?.registerFileText(csvFileName, csvString); + await this._database?.registerFileText(csvFileName, csv); await connection.insertCSVFromPath(csvFileName, { schema: schema, name: table, - create: false, + create: true, header: true, detect: true, escape: `'`, diff --git a/packages/legend-shared/src/format/FormatterUtils.ts b/packages/legend-shared/src/format/FormatterUtils.ts index 095177880e..8cd3b1d51c 100644 --- a/packages/legend-shared/src/format/FormatterUtils.ts +++ b/packages/legend-shared/src/format/FormatterUtils.ts @@ -21,7 +21,7 @@ import { parse as losslessParse, isSafeNumber as lossIsSafeNumber, } from 'lossless-json'; -import CSVParser from 'papaparse'; +import CSVParser, { type UnparseConfig } from 'papaparse'; import { assertNonNullable } from '../error/AssertionUtils.js'; export const capitalize = (value: string): string => @@ -152,8 +152,10 @@ export const parseCSVString = (value: string): string[] | undefined => { } }; -export const csvStringify = (value: unknown[]): string => - CSVParser.unparse(value); +export const csvStringify = ( + value: unknown[], + config?: UnparseConfig, +): string => CSVParser.unparse(value, config); /** * One very common use case is that we get the JSON as response from the server than we will convert this to a string and persist