Implement separate result data cache

packages/ui/src/models/ClobbrUICachedLog.ts

@@ -0,0 +1,4 @@
+export interface ClobbrUICachedLog {
+  index: number;
+  data?: { [key: string]: string } | string;
+}

packages/ui/src/models/ClobbrUIResult.ts

@@ -3,6 +3,7 @@ import { ClobbrLogItem } from '@clobbr/api/src/models/ClobbrLog';
 
 export interface ClobbrUIResult {
   id: string;
+  cachedId: string;
   startDate?: string;
   endDate?: string;
   resultDurations: Array<number>;

packages/ui/src/results/ResultHistory/ResultHistoryResponses/ResultHistoryResponse.tsx

@@ -8,6 +8,7 @@ import { getDurationColorClass } from 'shared/util/getDurationColorClass';
 
 import { ClobbrUIResult } from 'models/ClobbrUIResult';
 import { ResultHistoryResponseViewer } from 'results/ResultHistory/ResultHistoryResponses/ResultHistoryResponseViewer';
+import { getResultLogsKey } from 'shared/util/getResultLogsKey';
 
 export const ResultHistoryResponse = ({
   className,
@@ -31,6 +32,9 @@ export const ResultHistoryResponse = ({
   };
 
   const openLog = isNumber(openLogIndex) ? result.logs[openLogIndex] : null;
+  const logKey = getResultLogsKey({
+    cachedId: result.cachedId
+  });
 
   return (
     <div className={clsx('flex', className)}>
@@ -97,7 +101,7 @@ export const ResultHistoryResponse = ({
       </ul>
 
       {openLog ? (
-        <ResultHistoryResponseViewer log={openLog} />
+        <ResultHistoryResponseViewer log={openLog} logKey={logKey} />
       ) : (
         <Typography variant="caption" className="text-center p-6">
           Select an item to view the response.

packages/ui/src/results/ResultHistory/ResultHistoryResponses/ResultHistoryResponseViewer.tsx

@@ -4,6 +4,7 @@ import MonacoEditor from '@uiw/react-monacoeditor';
 import * as monaco from 'monaco-editor';
 import { GlobalStore } from 'app/globalContext';
 import { ClobbrLogItem } from '@clobbr/api/src/models/ClobbrLog';
+import { ClobbrUICachedLog } from 'models/ClobbrUICachedLog';
 import { useCopyToClipboard } from 'react-use';
 
 import {
@@ -16,6 +17,9 @@ import {
 import { MONACO_OPTIONS } from 'shared/monaco/options';
 import { isString } from 'lodash-es';
 import { Button } from '@mui/material';
+import { getDb } from 'storage/storage';
+import { EDbStores } from 'storage/EDbStores';
+import { SK } from 'storage/storageKeys';
 
 declare type IMonacoEditor = typeof monaco;
 
@@ -40,10 +44,12 @@ const onEditorMount = (
 
 export const ResultHistoryResponseViewer = ({
   className,
-  log
+  log,
+  logKey
 }: {
   className?: string;
   log: ClobbrLogItem;
+  logKey: string;
 }) => {
   const [state, copyToClipboard] = useCopyToClipboard();
   const [copied, setCopied] = useState(false);
@@ -64,10 +70,17 @@ export const ResultHistoryResponseViewer = ({
 
   const parseResponse = useCallback(async () => {
     if (log.failed) {
-      // Failed items
-      // Try to parse as JSON
+      /* Failed items */
       try {
-        const string = JSON.stringify(log.error, null, 2);
+        // Try to parse as JSON
+        const string = JSON.stringify(
+          {
+            ...log.metas,
+            error: log.error
+          },
+          null,
+          2
+        );
         setFormattedResponse(string);
         setEditorLanguage('json');
         return;
@@ -76,23 +89,48 @@ export const ResultHistoryResponseViewer = ({
       }
 
       try {
-        setEditorLanguage('plaintext');
-        setFormattedResponse(log.error?.toString() || '');
+        // Try to parse as plaintext
+        setEditorLanguage('json');
+        setFormattedResponse(
+          JSON.stringify(
+            {
+              ...log.metas,
+              error: log.error?.toString()
+            },
+            null,
+            2
+          )
+        );
         return;
       } catch (e) {
         console.warn('Could not parse error as plaintext', e);
       }
     } else {
-      // Successful items
-      if (!log.metas.data) {
+      /* Successful items */
+      const resultDb = getDb(EDbStores.RESULT_LOGS_STORE_NAME);
+      const data = await resultDb.getItem(
+        `${SK.RESULT_RESPONSE.ITEM}-${logKey}`
+      );
+
+      // Get response data from DB.
+      if (!data) {
+        setFormattedResponse('');
+        return;
+      }
+
+      const cachedResponse = data.find(
+        (item: ClobbrUICachedLog) => item.index === log.metas.index
+      );
+
+      if (!cachedResponse) {
         setFormattedResponse('');
         return;
       }
 
-      if (isString(log.metas.data)) {
-        // Try to parse as XML
+      if (isString(cachedResponse.data)) {
         try {
-          const xmlString = await formatXml(log.metas.data);
+          // Try to parse as XML
+          const xmlString = await formatXml(cachedResponse.data);
 
           if (xmlString) {
             setFormattedResponse(xmlString);
@@ -104,23 +142,17 @@ export const ResultHistoryResponseViewer = ({
         }
       }
 
-      // Try to parse as JSON
       try {
-        const string = JSON.stringify(log.metas.data, null, 2);
+        // Try to parse as JSON
+        const string = JSON.stringify(cachedResponse.data, null, 2);
         setFormattedResponse(string);
         setEditorLanguage('json');
         return;
       } catch (e) {
         console.warn('Could not parse as JSON', e);
       }
     }
-  }, [
-    log.error,
-    log.failed,
-    log.metas.data,
-    setEditorLanguage,
-    setFormattedResponse
-  ]);
+  }, [log.error, log.failed, log.metas, logKey]);
 
   useEffect(() => {
     const formatResponse = async () => {

packages/ui/src/shared/util/getResultLogsKey.ts

@@ -0,0 +1,10 @@
+/**
+ * Get the key for the result logs cache.
+ */
+export const getResultLogsKey = ({
+  cachedId
+}: {
+  cachedId: string;
+}): string => {
+  return cachedId;
+};

packages/ui/src/shared/util/toValidStorageKey.ts

@@ -0,0 +1,8 @@
+export const toValidStorageKeyName = (key: string) => {
+  return key
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace(/^[._]/, '')
+    .replace(/[^a-z0-9-~]+/g, '-');
+};

packages/ui/src/state/useResultState.ts

@@ -1,15 +1,63 @@
-import { uniq } from 'lodash-es';
+import { isPlainObject, uniq } from 'lodash-es';
 import { ClobbrLogItem } from '@clobbr/api/src/models/ClobbrLog';
 import { useState } from 'react';
 import { v4 as uuidv4 } from 'uuid';
 import { formatISO } from 'date-fns';
 import { ClobbrUIResult } from 'models/ClobbrUIResult';
 import { ClobbrUIResultListItem } from 'models/ClobbrUIResultListItem';
 import { Everbs } from 'shared/enums/http';
+import { getResultLogsKey } from 'shared/util/getResultLogsKey';
 import { ClobbrUIHeaderItem } from 'models/ClobbrUIHeaderItem';
 import useStateRef from 'react-usestateref';
 import { ClobbrUIProperties } from 'models/ClobbrUIProperties';
 import { useThrottle } from 'react-use';
+import { AxiosError } from 'axios';
+import { getDb } from 'storage/storage';
+import { SK } from 'storage/storageKeys';
+import { EDbStores } from 'storage/EDbStores';
+
+const unbloatLogs = (log: ClobbrLogItem) => {
+  if (log.failed && log.error && isPlainObject(log.error)) {
+    try {
+      const error = log.error as AxiosError;
+      error.config = {
+        url: error.config.url,
+        method: error.config.method,
+        headers: error.config.headers,
+        data: error.config.data
+      };
+    } catch (error) {
+      console.error('Failed to prune error object', error);
+    }
+  }
+
+  delete log.metas.data;
+  return log;
+};
+
+const writeLogResponsesToStorage = async ({
+  cachedId,
+  logs
+}: {
+  cachedId: string;
+  logs: Array<ClobbrLogItem>;
+}) => {
+  const resultDb = getDb(EDbStores.RESULT_LOGS_STORE_NAME);
+  const id = getResultLogsKey({
+    cachedId
+  });
+
+  resultDb.setItem(
+    `${SK.RESULT_RESPONSE.ITEM}-${id}`,
+    logs.map((log) => {
+      let logData = log.failed ? log.error : log.metas.data;
+      return {
+        index: log.metas.index,
+        data: logData
+      };
+    })
+  );
+};
 
 export const useResultState = ({ initialState }: { [key: string]: any }) => {
   const [editing, setEditing] = useState(false);
@@ -92,18 +140,23 @@ export const useResultState = ({ initialState }: { [key: string]: any }) => {
     properties: ClobbrUIProperties;
   }) => {
     const runId = uuidv4();
+    const cachedId = uuidv4();
     const currentList = listRef.current;
-    const logsWithoutMetaResponseData = logs.map((log) => {
-      delete log.metas.data;
-      return log;
+
+    writeLogResponsesToStorage({
+      cachedId,
+      logs
     });
 
+    const logsWithoutBloat = logs.map(unbloatLogs);
+
     const result: ClobbrUIResult = {
       id: runId,
+      cachedId,
       startDate: formatISO(new Date()),
       endDate: undefined,
       resultDurations,
-      logs: logsWithoutMetaResponseData,
+      logs: logsWithoutBloat,
       parallel,
       iterations,
       headers,
@@ -194,10 +247,6 @@ export const useResultState = ({ initialState }: { [key: string]: any }) => {
     logs: Array<ClobbrLogItem>;
   }) => {
     const currentList = listRef.current;
-    const logsWithoutMetaResponseData = logs.map((log) => {
-      // delete log.metas.data; // might want to make deleting data an option this could blow up in some cases.
-      return log;
-    });
 
     const existingListItem = currentList.find((i) => i.id === itemId);
     const isComplete = existingListItem?.iterations === logs.length;
@@ -208,7 +257,13 @@ export const useResultState = ({ initialState }: { [key: string]: any }) => {
       return false;
     }
 
-    const resultDurations = logsWithoutMetaResponseData
+    writeLogResponsesToStorage({
+      cachedId: existingListItem.latestResult.cachedId,
+      logs
+    });
+
+    const logsWithoutBloat = logs.map(unbloatLogs);
+    const resultDurations = logsWithoutBloat
       .filter((l) => typeof l.metas.duration === 'number')
       .map((l) => {
         return l.metas.duration as number;
@@ -217,7 +272,7 @@ export const useResultState = ({ initialState }: { [key: string]: any }) => {
     const nextResult: ClobbrUIResult = {
       ...existingListItem.latestResult,
       endDate,
-      logs: logsWithoutMetaResponseData,
+      logs: logsWithoutBloat,
       resultDurations
     };
 

packages/ui/src/storage/EDbStores.ts

@@ -1,4 +1,5 @@
 export const enum EDbStores {
   MAIN_STORE_NAME = 'main',
-  RESULT_STORE_NAME = 'result'
+  RESULT_STORE_NAME = 'result',
+  RESULT_LOGS_STORE_NAME = 'resultLogs'
 }

packages/ui/src/storage/storage.ts

@@ -23,13 +23,21 @@ const resultDb = localforage.createInstance({
   description: 'Holds result data'
 });
 
+const resultResponseDb = localforage.createInstance({
+  name: DB_NAME,
+  storeName: EDbStores.RESULT_LOGS_STORE_NAME,
+  description: 'Holds result response data'
+});
+
 export const getDb = (type: EDbStores) => {
   const _getDb = () => {
     switch (type) {
       case EDbStores.MAIN_STORE_NAME:
         return mainDb;
       case EDbStores.RESULT_STORE_NAME:
         return resultDb;
+      case EDbStores.RESULT_LOGS_STORE_NAME:
+        return resultResponseDb;
       default:
         return mainDb;
     }

packages/ui/src/storage/storageKeys.ts

@@ -2,6 +2,9 @@ export const SK = {
   RESULT: {
     LIST: 'result.resultList'
   },
+  RESULT_RESPONSE: {
+    ITEM: 'result.response.item'
+  },
   PREFERENCES: {
     THEME: 'preferences.theme',
     STICKY_SEARCH: 'preferences.stickySearch',