Datacube: fixing caching for csv (#3891)

finos · Feb 12, 2025 · 8a89338 · 8a89338
1 parent c0a5458
commit 8a89338
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 21 deletions.
diff --git a/.changeset/tidy-ads-judge.md b/.changeset/tidy-ads-judge.md
@@ -0,0 +1,6 @@
+---
+'@finos/legend-application-data-cube': patch
+'@finos/legend-shared': patch
+---
+
+Fixing Caching for CSV
diff --git a/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts b/packages/legend-application-data-cube/src/stores/LegendDataCubeCacheManager.ts
@@ -27,6 +27,7 @@ import {
 } from '@finos/legend-graph';
 import {
   assertNonNullable,
+  csvStringify,
   guaranteeNonNullable,
   UnsupportedOperationError,
 } from '@finos/legend-shared';
@@ -92,32 +93,22 @@ export class LegendDataCubeDataCubeCacheManager {
 
     const connection = await this.database.connect();
 
-    const columnString = result.builder.columns
-      .map((col) => col.name)
-      .join(',');
+    const columnNames: string[] = [];
+    result.builder.columns.forEach((col) => columnNames.push(col.name));
 
-    const dataString: string[] = [columnString];
+    const data = result.result.rows.map((row) => row.values);
 
-    result.result.rows.forEach((row) => {
-      const updatedRows = row.values.map((val) => {
-        if (val !== null && typeof val === 'string') {
-          return `'${val.replaceAll(`'`, `''`)}'`;
-        } else if (val === null) {
-          return `NULL`;
-        }
-        return val;
-      });
-      dataString.push(`${updatedRows.join(',')}`);
+    const csv = csvStringify([columnNames, ...data], {
+      escapeChar: `'`,
+      quoteChar: `'`,
     });
 
-    const csvString = dataString.join('\n');
-
-    await this._database?.registerFileText(csvFileName, csvString);
+    await this._database?.registerFileText(csvFileName, csv);
 
     await connection.insertCSVFromPath(csvFileName, {
       schema: schema,
       name: table,
-      create: false,
+      create: true,
       header: true,
       detect: true,
       escape: `'`,

diff --git a/packages/legend-shared/src/format/FormatterUtils.ts b/packages/legend-shared/src/format/FormatterUtils.ts
@@ -21,7 +21,7 @@ import {
   parse as losslessParse,
   isSafeNumber as lossIsSafeNumber,
 } from 'lossless-json';
-import CSVParser from 'papaparse';
+import CSVParser, { type UnparseConfig } from 'papaparse';
 import { assertNonNullable } from '../error/AssertionUtils.js';
 
 export const capitalize = (value: string): string =>
@@ -152,8 +152,10 @@ export const parseCSVString = (value: string): string[] | undefined => {
   }
 };
 
-export const csvStringify = (value: unknown[]): string =>
-  CSVParser.unparse(value);
+export const csvStringify = (
+  value: unknown[],
+  config?: UnparseConfig,
+): string => CSVParser.unparse(value, config);
 
 /**
  * One very common use case is that we get the JSON as response from the server than we will convert this to a string and persist