ENG-800 handle update embeddings button (#510)

sid597 · web-flow · commit 40d929dd82aa · 2025-11-17T02:42:10.000+05:30
* handle update embeddings button * 1. Fix query to get imported nodes edit data 2. remove isabled arg 3. Fix getAllDiscourseNodesSince to get nodes per node type, use map to store and get non duplicate data 4. getNodesByType * remove success banner for suggestive mode, send error email on fail, fix on load check if we need to initialize by default (#527) * remove success banner for suggestive mode, send error email on fail, fix on load check if we need to initialize by default * address coderabbit review * address review * address review * remove console log * use logged in client * remove supabase based guarding * remove unused
diff --git a/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx b/apps/roam/src/components/settings/DiscourseNodeSuggestiveRules.tsx
@@ -12,6 +12,10 @@ import TextPanel from "roamjs-components/components/ConfigPanels/TextPanel";
 import getSubTree from "roamjs-components/util/getSubTree";
 import { DiscourseNode } from "~/utils/getDiscourseNodes";
 import extractRef from "roamjs-components/util/extractRef";
+import { getAllDiscourseNodesSince } from "~/utils/getAllDiscourseNodesSince";
+import { upsertNodesToSupabaseAsContentWithEmbeddings } from "~/utils/syncDgNodesToSupabase";
+import { discourseNodeBlockToLocalConcept } from "~/utils/conceptConversion";
+import { getLoggedInClient, getSupabaseContext } from "~/utils/supabaseContext";
 
 const BlockRenderer = ({ uid }: { uid: string }) => {
   const containerRef = useRef<HTMLDivElement>(null);
@@ -64,10 +68,34 @@ const DiscourseNodeSuggestiveRules = ({
     (e: React.ChangeEvent<HTMLInputElement>) => {
       const newValue = e.target.value;
       setEmbeddingRef(newValue);
+      node.embeddingRef = newValue;
     },
-    [],
+    [node],
   );
-
+  const [isUpdating, setIsUpdating] = useState(false);
+
+  const handleUpdateEmbeddings = async (): Promise<void> => {
+    setIsUpdating(true);
+    try {
+      const blockNodesSince = await getAllDiscourseNodesSince(
+        new Date(0).toISOString(),
+        [node],
+      );
+      const supabaseClient = await getLoggedInClient();
+      if (!supabaseClient) return;
+
+      const context = await getSupabaseContext();
+      if (context && blockNodesSince) {
+        await upsertNodesToSupabaseAsContentWithEmbeddings(
+          blockNodesSince,
+          supabaseClient,
+          context,
+        );
+      }
+    } finally {
+      setIsUpdating(false);
+    }
+  };
   return (
     <div className="flex flex-col gap-4 p-4">
       <BlocksPanel
@@ -82,11 +110,10 @@ const DiscourseNodeSuggestiveRules = ({
       <TextPanel
         title="Embedding Block Ref"
         description="Copy block ref from template which you want to be embedded and ranked."
-        order={0}
+        order={1}
         uid={node.embeddingRefUid || ""}
         parentUid={parentUid}
-        value={node.embeddingRef || ""}
-        defaultValue={node.embeddingRef || ""}
+        defaultValue={embeddingRef || ""}
         options={{
           placeholder: "((block-uid))",
           onChange: handleEmbeddingRefChange,
@@ -103,19 +130,18 @@ const DiscourseNodeSuggestiveRules = ({
       <FlagPanel
         title="First Child"
         description="If the block is the first child of the embedding block ref, it will be embedded and ranked."
-        order={1}
+        order={2}
         uid={node.isFirstChild?.uid || ""}
         parentUid={parentUid}
         value={node.isFirstChild?.value || false}
       />
 
-      {/* TODO: Add a button to update embeddings in seperate PR */}
       <Button
         text="Update Embeddings"
         intent={Intent.NONE}
-        onClick={() => console.log("Not implemented")}
+        onClick={() => void handleUpdateEmbeddings()}
+        loading={isUpdating}
         className="w-52"
-        disabled
       />
     </div>
   );
diff --git a/apps/roam/src/components/settings/SuggestiveModeSettings.tsx b/apps/roam/src/components/settings/SuggestiveModeSettings.tsx
@@ -111,6 +111,10 @@ const SuggestiveModeSettings = () => {
                       });
                       try {
                         await createOrUpdateDiscourseEmbedding();
+                        await createBlock({
+                          parentUid: settings.settingsUid,
+                          node: { text: "(BETA) Suggestive Mode Enabled" },
+                        });
                       } catch (e) {
                         console.error("Failed to generate embeddings", e);
                         renderToast({
diff --git a/apps/roam/src/index.ts b/apps/roam/src/index.ts
@@ -32,6 +32,10 @@ import {
   setSyncActivity,
 } from "./utils/syncDgNodesToSupabase";
 import { initPluginTimer } from "./utils/pluginTimer";
+import { getUidAndBooleanSetting } from "./utils/getExportSettings";
+import getBasicTreeByParentUid from "roamjs-components/queries/getBasicTreeByParentUid";
+import getPageUidByPageTitle from "roamjs-components/queries/getPageUidByPageTitle";
+import { DISCOURSE_CONFIG_PAGE_TITLE } from "./utils/renderNodeConfigPage";
 
 const initPostHog = () => {
   posthog.init("phc_SNMmBqwNfcEpNduQ41dBUjtGNEUEKAy6jTn63Fzsrax", {
@@ -126,7 +130,16 @@ export default runExtension(async (onloadArgs) => {
   document.addEventListener("input", discourseNodeSearchTriggerListener);
   document.addEventListener("selectionchange", nodeCreationPopoverListener);
 
-  await initializeSupabaseSync();
+  const isSuggestiveModeEnabled = getUidAndBooleanSetting({
+    tree: getBasicTreeByParentUid(
+      getPageUidByPageTitle(DISCOURSE_CONFIG_PAGE_TITLE),
+    ),
+    text: "(BETA) Suggestive Mode Enabled",
+  }).value;
+
+  if (isSuggestiveModeEnabled) {
+    initializeSupabaseSync();
+  }
 
   const { extensionAPI } = onloadArgs;
   window.roamjs.extension.queryBuilder = {
diff --git a/apps/roam/src/utils/conceptConversion.ts b/apps/roam/src/utils/conceptConversion.ts
@@ -26,7 +26,7 @@ const getNodeExtraData = (
       [?block :create/user ?author_id]
       [?author_id :user/uid ?author_uid]
       [?block :create/time ?created]
-      [?block :edit/time ?last_modified]
+      [(get-else $ ?block :edit/time ?created) ?last_modified]
       [(get-else $ ?block :block/page ?block) ?page_id]
       [?page_id :block/uid ?page_uid]
   ]`,
diff --git a/apps/roam/src/utils/discourseConfigRef.ts b/apps/roam/src/utils/discourseConfigRef.ts
@@ -45,6 +45,7 @@ type FormattedConfigTree = {
   suggestiveMode: SuggestiveModeConfigWithUids;
   leftSidebar: LeftSidebarConfig;
   leftSidebarEnabled: BooleanSetting;
+  suggestiveModeEnabled: BooleanSetting;
 };
 
 export const getFormattedConfigTree = (): FormattedConfigTree => {
@@ -77,6 +78,10 @@ export const getFormattedConfigTree = (): FormattedConfigTree => {
       tree: configTreeRef.tree,
       text: "(BETA) Left Sidebar",
     }),
+    suggestiveModeEnabled: getUidAndBooleanSetting({
+      tree: configTreeRef.tree,
+      text: "(BETA) Suggestive Mode",
+    }),
   };
 };
 export default configTreeRef;
diff --git a/apps/roam/src/utils/getAllDiscourseNodesSince.ts b/apps/roam/src/utils/getAllDiscourseNodesSince.ts
@@ -1,5 +1,4 @@
-import getDiscourseNodes, { type DiscourseNode } from "./getDiscourseNodes";
-import findDiscourseNode from "./findDiscourseNode";
+import { type DiscourseNode } from "./getDiscourseNodes";
 import getDiscourseNodeFormatExpression from "./getDiscourseNodeFormatExpression";
 import { extractRef } from "roamjs-components/util";
 
@@ -23,10 +22,10 @@ export type DiscourseNodesSinceResult = {
   blockNodes: RoamDiscourseNodeData[];
 };
 
-export const getDiscourseNodeTypeWithSettingsBlockNodes = (
+export const getDiscourseNodeTypeWithSettingsBlockNodes = async (
   node: DiscourseNode,
   sinceMs: number,
-): RoamDiscourseNodeData[] => {
+): Promise<RoamDiscourseNodeData[]> => {
   const regex = getDiscourseNodeFormatExpression(node.format);
   const regexPattern = regex.source.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
   const firstChildUid = extractRef(node.embeddingRef);
@@ -40,30 +39,32 @@ export const getDiscourseNodeTypeWithSettingsBlockNodes = (
         [(re-find ?title-regex ?node-title)]
         [?node :block/uid ?nodeUid]
         [?node :create/time ?nodeCreateTime]
-        [?node :edit/time ?nodeEditTime]
+        [(get-else $ ?node :edit/time ?nodeCreateTime) ?nodeEditTime]
         [?s :block/uid ?firstChildUid]
         [?s :block/string ?firstChildString]
         [?bg :block/page ?node]
         [?bg :block/string ?firstChildString]
         [?bg :block/children ?child]
         [?child :block/order 0]
         [?child :block/string ?childString]
-        [?child :edit/time ?childEditTime]
+        [(get-else $ ?child :edit/time ?nodeCreateTime) ?childEditTime]
         [?child :create/user ?user-eid]
         [?user-eid :user/uid ?author_local_id]
-        [?child :edit/user ?eu]
-        [?eu :user/display-name ?author_name]
+        [(get-else $ ?child :edit/user ?user-eid) ?eu]
+        [(get-else $ ?eu :user/display-name "Anonymous User") ?author_name]
         [or 
          [(> ?childEditTime ?since)]
          [(> ?nodeEditTime ?since)]]
         ]`;
 
-  const blockNode = window.roamAlphaAPI.data.q(
-    queryBlock,
-    String(firstChildUid),
-    String(node.type),
-    sinceMs,
-  ) as unknown as RoamDiscourseNodeData[];
+  const blockNode = (await Promise.resolve(
+    window.roamAlphaAPI.data.backend.q(
+      queryBlock,
+      String(firstChildUid),
+      String(node.type),
+      sinceMs,
+    ),
+  )) as unknown as RoamDiscourseNodeData[];
   return blockNode;
 };
 
@@ -72,65 +73,60 @@ export const getAllDiscourseNodesSince = async (
   nodeTypes: DiscourseNode[],
 ): Promise<RoamDiscourseNodeData[]> => {
   const sinceMs = new Date(since).getTime();
-  const result: RoamDiscourseNodeData[] = [];
+  const resultMap = new Map<string, RoamDiscourseNodeData>();
 
-  const query = `[
+  await Promise.all(
+    nodeTypes.map(async (node) => {
+      const regex = getDiscourseNodeFormatExpression(node.format);
+      const regexPattern = regex.source
+        .replace(/\\/g, "\\\\")
+        .replace(/"/g, '\\"');
 
-    :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name
-    :keys text source_local_id created last_modified author_local_id author_name
-    :in $ ?since 
-    :where
-      [?node :node/title ?node-title]
-      [?node :block/uid ?uid]
-      [?node :create/time ?nodeCreateTime]
-      [?node :create/user ?user-eid]
-      [?user-eid :user/uid ?author_local_id]
-      [(get-else $ ?user-eid :user/display-name "Unknown-Creator") ?author_name]
-      [(get-else $ ?node :edit/time 0) ?nodeEditTime]
-      [(get-else $ ?node :edit/time ?nodeCreateTime) ?filterTime]
-      [(> ?filterTime ?since)]
-  ]`;
-  const allNodes = (await Promise.resolve(
-    window.roamAlphaAPI.data.backend.q(query, sinceMs),
-  )) as unknown as RoamDiscourseNodeData[];
+      const query = `[
+        :find ?node-title ?uid ?nodeCreateTime ?nodeEditTime ?author_local_id ?author_name ?type
+        :keys text source_local_id created last_modified author_local_id author_name type
+        :in $ ?since ?type
+        :where
+          [(re-pattern "${regexPattern}") ?title-regex]
+          [?node :node/title ?node-title]
+          [(re-find ?title-regex ?node-title)]
+          [?node :block/uid ?uid]
+          [?node :create/time ?nodeCreateTime]
+          [?node :create/user ?user-eid]
+          [?user-eid :user/uid ?author_local_id]
+          [(get-else $ ?user-eid :user/display-name "Anonymous User") ?author_name]
+          [(get-else $ ?node :edit/time ?nodeCreateTime) ?nodeEditTime]
+          [(> ?nodeEditTime ?since)]
+      ]`;
 
-  const discourseNodes = getDiscourseNodes();
+      const nodesOfType = (await Promise.resolve(
+        window.roamAlphaAPI.data.backend.q(query, sinceMs, String(node.type)),
+      )) as unknown as RoamDiscourseNodeData[];
 
-  result.push(
-    ...allNodes.flatMap((entity) => {
-      if (!entity.source_local_id) {
-        return [];
-      }
-      const node = findDiscourseNode(entity.source_local_id, discourseNodes);
-      if (
-        !node ||
-        node.backedBy === "default" ||
-        !entity.text ||
-        entity.text.trim() === ""
-      ) {
-        return [];
+      nodesOfType.forEach((n) => {
+        if (n.source_local_id) {
+          resultMap.set(n.source_local_id, n);
+        }
+      });
+
+      const hasBlockSettings =
+        node.embeddingRef && extractRef(node.embeddingRef);
+      if (hasBlockSettings) {
+        const blockNodes = await getDiscourseNodeTypeWithSettingsBlockNodes(
+          node,
+          sinceMs,
+        );
+        if (blockNodes) {
+          blockNodes.forEach((blockNode) => {
+            if (blockNode.source_local_id) {
+              resultMap.set(blockNode.source_local_id, blockNode);
+            }
+          });
+        }
       }
-      return [
-        {
-          ...entity,
-          type: node.type,
-        },
-      ];
     }),
   );
-
-  if (nodeTypes.length > 0) {
-    for (const node of nodeTypes) {
-      const blockNode = getDiscourseNodeTypeWithSettingsBlockNodes(
-        node,
-        sinceMs,
-      );
-      if (blockNode) {
-        result.push(...blockNode);
-      }
-    }
-  }
-  return result;
+  return Array.from(resultMap.values());
 };
 
 export const nodeTypeSince = async (
diff --git a/apps/roam/src/utils/getDiscourseNodes.ts b/apps/roam/src/utils/getDiscourseNodes.ts
@@ -112,7 +112,7 @@ const getDiscourseNodes = (relations = getDiscourseRelations()) => {
       const embeddingBlockRef = getSubTree({
         tree: suggestiveRules.children,
         key: "Embedding Block Ref",
-      }).children?.[0];
+      });
 
       return {
         format: getSettingValueFromTree({ tree: children, key: "format" }),
@@ -134,7 +134,7 @@ const getDiscourseNodes = (relations = getDiscourseRelations()) => {
           key: "description",
         }),
         template: getSubTree({ tree: children, key: "template" }).children,
-        embeddingRef: embeddingBlockRef?.text,
+        embeddingRef: embeddingBlockRef?.children?.[0]?.text,
         embeddingRefUid: embeddingBlockRef?.uid,
         isFirstChild: getUidAndBooleanSetting({
           tree: suggestiveRules.children,
diff --git a/apps/roam/src/utils/syncDgNodesToSupabase.ts b/apps/roam/src/utils/syncDgNodesToSupabase.ts
