Human Annotation Sidebar (#6410)

* merge

* sidebar work

* fix bounding box

* keep image pixelated (#6409)

Co-authored-by: Gavin McDonald <[email protected]>

* cleanup events, and schema creation

---------

Co-authored-by: Gavin <[email protected]>
Co-authored-by: Gavin McDonald <[email protected]>

Author: 3 people

app/packages/core/src/components/Modal/Lighter/LighterSampleRenderer.tsx

@@ -1,27 +1,19 @@
 /**
  * Copyright 2017-2025, Voxel51, Inc.
  */
-import { editing } from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/Edit";
-import {
-  labelAtoms,
-  labels,
-} from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/useLabels";
 import {
   ImageOptions,
   ImageOverlay,
   overlayFactory,
   useLighter,
   useLighterSetupWithPixi,
 } from "@fiftyone/lighter";
-import { FROM_FO, loadOverlays } from "@fiftyone/looker/src/overlays";
-import DetectionOverlay from "@fiftyone/looker/src/overlays/detection";
+import type { Sample } from "@fiftyone/state";
 import * as fos from "@fiftyone/state";
-import { Sample, getSampleSrc } from "@fiftyone/state";
-import { getDefaultStore, useSetAtom } from "jotai";
+import { getSampleSrc } from "@fiftyone/state";
 import React, { useEffect, useLayoutEffect, useRef, useState } from "react";
 import { useRecoilValue } from "recoil";
 import { singletonCanvas } from "./SharedCanvas";
-import { convertLegacyToLighterDetection } from "./looker-lighter-bridge";
 import { useBridge } from "./useBridge";
 
 export interface LighterSampleRendererProps {
@@ -34,10 +26,10 @@ export interface LighterSampleRendererProps {
 /**
  * Lighter unit sample renderer with PixiJS renderer.
  */
-export const LighterSampleRenderer: React.FC<LighterSampleRendererProps> = ({
+export const LighterSampleRenderer = ({
   className = "",
   sample,
-}) => {
+}: LighterSampleRendererProps) => {
   const containerRef = useRef<HTMLDivElement>(null);
 
   // we have this hack to force a re-render on layout effect, so that containerRef.current is defined
@@ -51,12 +43,6 @@ export const LighterSampleRenderer: React.FC<LighterSampleRendererProps> = ({
   // Get access to the lighter instance
   const { scene, isReady, addOverlay } = useLighter();
 
-  const setEditing = useSetAtom(editing);
-
-  const schema = fos.useAssertedRecoilValue(
-    fos.fieldSchema({ space: fos.State.SPACE.SAMPLE })
-  );
-
   /**
    * This effect is responsible for loading the sample and adding the overlays to the scene.
    */
@@ -82,70 +68,10 @@ export const LighterSampleRenderer: React.FC<LighterSampleRendererProps> = ({
       scene.setCanonicalMedia(mediaOverlay);
     }
 
-    const overlays = loadOverlays(sample.sample, schema, false);
-
-    for (const overlay of overlays) {
-      if (overlay instanceof DetectionOverlay) {
-        // Convert legacy overlay to lighter overlay with relative coordinates
-        const lighterOverlay = convertLegacyToLighterDetection(
-          overlay,
-          sample.id
-        );
-        addOverlay(lighterOverlay, false);
-      }
-    }
-
     return () => {
       scene.destroy();
     };
-  }, [isReady, addOverlay, sample, scene, schema]);
-
-  useEffect(() => {
-    if (!scene || !sample) return;
-
-    const store = getDefaultStore();
-
-    // hack: this is how we execute it once
-    let areOverlaysAdded = false;
-
-    const unsub = store.sub(labels, () => {
-      const labelAtomsList = store.get(labelAtoms);
-
-      if (areOverlaysAdded) return;
-
-      for (const atom of labelAtomsList) {
-        const label = store.get(atom);
-        if (!FROM_FO[label.type]) {
-          continue;
-        }
-        const overlay = FROM_FO[label.type](label.path, label.data)[0];
-        if (overlay instanceof DetectionOverlay) {
-          // Convert legacy overlay to lighter overlay with relative coordinates
-          const lighterOverlay = convertLegacyToLighterDetection(
-            overlay,
-            sample.id
-          );
-
-          addOverlay(lighterOverlay, false);
-        }
-        areOverlaysAdded = true;
-      }
-    });
-
-    return () => {
-      unsub();
-      areOverlaysAdded = false;
-    };
-  }, [scene, sample, addOverlay, labelAtoms]);
-
-  /**
-   * This effect runs cleanup when the component unmounts
-   */
-  useEffect(() => {
-    return () => {
-      setEditing(null);
-    };
-  }, []);
+  }, [isReady, addOverlay, sample, scene]);
 
   return (
     <div
@@ -174,7 +100,7 @@ const LighterSetupImpl = (props: {
     fos.lookerOptions({ modal: true, withFilter: false })
   );
 
-  const canvas = singletonCanvas.getCanvas(containerRef.current!);
+  const canvas = singletonCanvas.getCanvas(containerRef.current);
 
   const { scene } = useLighterSetupWithPixi(canvas, options);
 

app/packages/core/src/components/Modal/Lighter/SharedCanvas.ts

@@ -53,7 +53,7 @@ class SingletonCanvas {
    * The canvas element itself is preserved for reuse.
    */
   detach(): void {
-    if (this.canvas && this.canvas.parentNode) {
+    if (this.canvas?.parentNode) {
       this.canvas.parentNode.removeChild(this.canvas);
       this.isAttached = false;
       this.container = null;

app/packages/core/src/components/Modal/Lighter/looker-lighter-bridge.ts

@@ -2,12 +2,9 @@
  * Copyright 2017-2025, Voxel51, Inc.
  */
 
-import { Scene2D } from "@fiftyone/lighter";
-import { colorScheme, colorSeed } from "@fiftyone/state";
+import type { Scene2D } from "@fiftyone/lighter";
 import { useEffect } from "react";
-import { useRecoilValue } from "recoil";
-import { useOverlayPersistence } from "./useOverlayPersistence";
-import { useLighterTooltipEventHandler } from "./useLighterTooltipEventHandler";
+import useColorMappingContext from "./useColorMappingContext";
 
 /**
  * Hook that bridges FiftyOne state management system with Lighter.
@@ -17,11 +14,9 @@ import { useLighterTooltipEventHandler } from "./useLighterTooltipEventHandler";
  * 2. We trigger certain events into "FiftyOne state" world based on user interactions in Lighter.
  */
 export const useBridge = (scene: Scene2D | null) => {
-  const currentColorScheme = useRecoilValue(colorScheme);
-  const currentColorSeed = useRecoilValue(colorSeed);
-
-  useLighterTooltipEventHandler(scene);
-  useOverlayPersistence(scene);
+  // useLighterTooltipEventHandler(scene);
+  // useOverlayPersistence(scene);
+  const context = useColorMappingContext();
 
   // Effect to update scene with color scheme changes
   useEffect(() => {
@@ -30,14 +25,11 @@ export const useBridge = (scene: Scene2D | null) => {
     }
 
     // Update the scene's color mapping context
-    scene.updateColorMappingContext({
-      colorScheme: currentColorScheme,
-      seed: currentColorSeed,
-    });
+    scene.updateColorMappingContext(context);
 
     // Mark all overlays as dirty to trigger re-rendering with new colors
-    scene.getAllOverlays().forEach((overlay) => {
+    for (const overlay of scene.getAllOverlays()) {
       overlay.markDirty();
-    });
-  }, [scene, currentColorScheme, currentColorSeed]);
+    }
+  }, [scene, context]);
 };

app/packages/core/src/components/Modal/Lighter/useBridge.ts

@@ -0,0 +1,15 @@
+import { colorScheme, colorSeed } from "@fiftyone/state";
+import { useMemo } from "react";
+import { useRecoilValue } from "recoil";
+
+export default function useColorMappingContext() {
+  const currentColorScheme = useRecoilValue(colorScheme);
+  const currentColorSeed = useRecoilValue(colorSeed);
+  return useMemo(
+    () => ({
+      colorScheme: currentColorScheme,
+      seed: currentColorSeed,
+    }),
+    [currentColorScheme, currentColorSeed]
+  );
+}

app/packages/core/src/components/Modal/Lighter/useColorMappingContext.ts

@@ -2,11 +2,12 @@
  * Copyright 2017-2025, Voxel51, Inc.
  */
 
+import type { Scene2D } from "@fiftyone/lighter";
+import { LIGHTER_EVENTS } from "@fiftyone/lighter";
+import type { Hoverable } from "@fiftyone/lighter/src/types";
 import * as fos from "@fiftyone/state";
 import { useEffect } from "react";
 import { useRecoilCallback } from "recoil";
-import { LIGHTER_EVENTS, Scene2D } from "@fiftyone/lighter";
-import type { Hoverable } from "@fiftyone/lighter/src/types";
 
 /**
  * Hook that handles tooltip events for lighter overlays.

app/packages/core/src/components/Modal/Lighter/useLighterTooltipEventHandler.ts

@@ -2,15 +2,10 @@
  * Copyright 2017-2025, Voxel51, Inc.
  */
 
-import {
-  addLabel,
-  removeLabel,
-} from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/useLabels";
+import type { OverlayEventDetail, Scene2D } from "@fiftyone/lighter";
+import { LIGHTER_EVENTS } from "@fiftyone/lighter";
 import { useOperatorExecutor } from "@fiftyone/operators";
-import { DETECTION } from "@fiftyone/utilities";
-import { useSetAtom } from "jotai";
 import { useCallback, useEffect } from "react";
-import { LIGHTER_EVENTS, OverlayEventDetail, Scene2D } from "@fiftyone/lighter";
 
 /**
  * Hook that handles overlay persistence events.
@@ -19,9 +14,6 @@ export const useOverlayPersistence = (scene: Scene2D | null) => {
   const addBoundingBox = useOperatorExecutor("add_bounding_box");
   const removeBoundingBox = useOperatorExecutor("remove_bounding_box");
 
-  const addLabelEffect = useSetAtom(addLabel);
-  const removeLabelEffect = useSetAtom(removeLabel);
-
   const handlePersistOverlay = useCallback(
     (
       event: CustomEvent<
@@ -45,14 +37,6 @@ export const useOverlayPersistence = (scene: Scene2D | null) => {
             label: overlay.label ?? "",
             bounding_box: bbox,
           });
-
-          addLabelEffect({
-            id: overlay.id,
-            data: overlay,
-            path: overlay.field,
-            expandedPath: overlay.field + ".detections",
-            type: DETECTION,
-          });
         } catch (error) {
           console.error("Error adding bounding box", error);
         }
@@ -64,7 +48,7 @@ export const useOverlayPersistence = (scene: Scene2D | null) => {
         );
       }
     },
-    [addBoundingBox, addLabelEffect]
+    [addBoundingBox]
   );
 
   const handleRemoveOverlay = useCallback(
@@ -81,13 +65,11 @@ export const useOverlayPersistence = (scene: Scene2D | null) => {
           path,
           sample_id: sampleId,
         });
-
-        removeLabelEffect(id);
       } catch (error) {
         console.error("Error removing bounding box", error);
       }
     },
-    [removeBoundingBox, removeLabelEffect]
+    [removeBoundingBox]
   );
 
   useEffect(() => {
@@ -102,5 +84,5 @@ export const useOverlayPersistence = (scene: Scene2D | null) => {
       scene.off(LIGHTER_EVENTS.DO_PERSIST_OVERLAY, handlePersistOverlay);
       scene.off(LIGHTER_EVENTS.DO_REMOVE_OVERLAY, handleRemoveOverlay);
     };
-  }, [scene]);
+  }, [handlePersistOverlay, handleRemoveOverlay, scene]);
 };