From a160a69dc3af2a7a28c7025cf50e5c26d61f4f85 Mon Sep 17 00:00:00 2001 From: Benjamin Kane Date: Tue, 7 Oct 2025 16:46:04 -0400 Subject: [PATCH 01/11] merge --- .../Modal/Lighter/LighterSampleRenderer.tsx | 86 +------ .../components/Modal/Lighter/SharedCanvas.ts | 2 +- .../Modal/Lighter/looker-lighter-bridge.ts | 35 --- .../src/components/Modal/Lighter/useBridge.ts | 22 +- .../Modal/Lighter/useColorMappingContext.ts | 15 ++ .../Lighter/useLighterTooltipEventHandler.ts | 5 +- .../Modal/Lighter/useOverlayPersistence.ts | 5 +- .../Modal/Sidebar/Annotate/Actions.tsx | 36 ++- .../Modal/Sidebar/Annotate/Annotate.tsx | 4 +- .../Annotate/Edit/AnnotationSchema.tsx | 90 +++++-- .../Modal/Sidebar/Annotate/Edit/Edit.tsx | 3 + .../Modal/Sidebar/Annotate/Edit/Field.tsx | 16 +- .../Modal/Sidebar/Annotate/Edit/Footer.tsx | 8 +- .../Modal/Sidebar/Annotate/Edit/Position.tsx | 140 +++++++++++ .../Modal/Sidebar/Annotate/Edit/state.ts | 137 +++++++---- .../Modal/Sidebar/Annotate/Edit/useCreate.ts | 132 ++++++++++- .../Annotate/Edit/useSpatialAttribute.ts | 222 ------------------ .../Modal/Sidebar/Annotate/Icons.tsx | 13 +- .../Modal/Sidebar/Annotate/LabelEntry.tsx | 11 +- .../Modal/Sidebar/Annotate/state.ts | 11 - .../Modal/Sidebar/Annotate/useColor.ts | 16 +- .../Modal/Sidebar/Annotate/useLabels.ts | 5 +- .../Modal/Sidebar/Annotate/useLoadSchemas.ts | 33 ++- .../SchemaIO/components/AutocompleteView.tsx | 1 + .../src/commands/UpdateLabelCommand.ts | 34 +++ app/packages/lighter/src/index.ts | 3 + .../InteractiveDetectionHandler.ts | 2 - .../lighter/src/overlay/BaseOverlay.ts | 21 +- .../lighter/src/overlay/BoundingBoxOverlay.ts | 57 +---- .../src/overlay/ClassificationOverlay.ts | 196 +--------------- .../lighter/src/react/useLighterSetup.ts | 4 +- .../lighter/src/renderer/Renderer2D.ts | 2 +- app/packages/looker/src/overlays/base.ts | 4 +- app/packages/looker/src/overlays/polyline.ts | 17 +- app/packages/state/src/recoil/sidebar.ts | 16 +- fiftyone/core/annotation.py | 4 +- 36 files changed, 647 insertions(+), 761 deletions(-) delete mode 100644 app/packages/core/src/components/Modal/Lighter/looker-lighter-bridge.ts create mode 100644 app/packages/core/src/components/Modal/Lighter/useColorMappingContext.ts create mode 100644 app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/Position.tsx delete mode 100644 app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/useSpatialAttribute.ts create mode 100644 app/packages/lighter/src/commands/UpdateLabelCommand.ts diff --git a/app/packages/core/src/components/Modal/Lighter/LighterSampleRenderer.tsx b/app/packages/core/src/components/Modal/Lighter/LighterSampleRenderer.tsx index 09b4efb1a1a..b0790449b0b 100644 --- a/app/packages/core/src/components/Modal/Lighter/LighterSampleRenderer.tsx +++ b/app/packages/core/src/components/Modal/Lighter/LighterSampleRenderer.tsx @@ -1,11 +1,6 @@ /** * Copyright 2017-2025, Voxel51, Inc. */ -import { editing } from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/Edit"; -import { - labelAtoms, - labels, -} from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/useLabels"; import { ImageOptions, ImageOverlay, @@ -13,15 +8,12 @@ import { useLighter, useLighterSetupWithPixi, } from "@fiftyone/lighter"; -import { FROM_FO, loadOverlays } from "@fiftyone/looker/src/overlays"; -import DetectionOverlay from "@fiftyone/looker/src/overlays/detection"; +import type { Sample } from "@fiftyone/state"; import * as fos from "@fiftyone/state"; -import { Sample, getSampleSrc } from "@fiftyone/state"; -import { getDefaultStore, useSetAtom } from "jotai"; +import { getSampleSrc } from "@fiftyone/state"; import React, { useEffect, useLayoutEffect, useRef, useState } from "react"; import { useRecoilValue } from "recoil"; import { singletonCanvas } from "./SharedCanvas"; -import { convertLegacyToLighterDetection } from "./looker-lighter-bridge"; import { useBridge } from "./useBridge"; export interface LighterSampleRendererProps { @@ -34,10 +26,10 @@ export interface LighterSampleRendererProps { /** * Lighter unit sample renderer with PixiJS renderer. */ -export const LighterSampleRenderer: React.FC = ({ +export const LighterSampleRenderer = ({ className = "", sample, -}) => { +}: LighterSampleRendererProps) => { const containerRef = useRef(null); // we have this hack to force a re-render on layout effect, so that containerRef.current is defined @@ -51,12 +43,6 @@ export const LighterSampleRenderer: React.FC = ({ // Get access to the lighter instance const { scene, isReady, addOverlay } = useLighter(); - const setEditing = useSetAtom(editing); - - const schema = fos.useAssertedRecoilValue( - fos.fieldSchema({ space: fos.State.SPACE.SAMPLE }) - ); - /** * This effect is responsible for loading the sample and adding the overlays to the scene. */ @@ -82,70 +68,10 @@ export const LighterSampleRenderer: React.FC = ({ scene.setCanonicalMedia(mediaOverlay); } - const overlays = loadOverlays(sample.sample, schema, false); - - for (const overlay of overlays) { - if (overlay instanceof DetectionOverlay) { - // Convert legacy overlay to lighter overlay with relative coordinates - const lighterOverlay = convertLegacyToLighterDetection( - overlay, - sample.id - ); - addOverlay(lighterOverlay, false); - } - } - return () => { scene.destroy(); }; - }, [isReady, addOverlay, sample, scene, schema]); - - useEffect(() => { - if (!scene || !sample) return; - - const store = getDefaultStore(); - - // hack: this is how we execute it once - let areOverlaysAdded = false; - - const unsub = store.sub(labels, () => { - const labelAtomsList = store.get(labelAtoms); - - if (areOverlaysAdded) return; - - for (const atom of labelAtomsList) { - const label = store.get(atom); - if (!FROM_FO[label.type]) { - continue; - } - const overlay = FROM_FO[label.type](label.path, label.data)[0]; - if (overlay instanceof DetectionOverlay) { - // Convert legacy overlay to lighter overlay with relative coordinates - const lighterOverlay = convertLegacyToLighterDetection( - overlay, - sample.id - ); - - addOverlay(lighterOverlay, false); - } - areOverlaysAdded = true; - } - }); - - return () => { - unsub(); - areOverlaysAdded = false; - }; - }, [scene, sample, addOverlay, labelAtoms]); - - /** - * This effect runs cleanup when the component unmounts - */ - useEffect(() => { - return () => { - setEditing(null); - }; - }, []); + }, [isReady, addOverlay, sample, scene]); return (
, - sampleId: string -) => { - // Get relative coordinates [0-1] from the legacy overlay - const [relativeX, relativeY, relativeWidth, relativeHeight] = - overlay.label.bounding_box; - - // Create a lighter bounding box with relative coordinates - // The scene will handle the coordinate transformation automatically - const lighterOverlay = overlayFactory.create( - "bounding-box", - { - label: overlay.label as BoundingBoxLabel, - relativeBounds: { - x: relativeX, - y: relativeY, - width: relativeWidth, - height: relativeHeight, - }, - draggable: true, - selectable: true, - field: overlay.field, - sampleId, - } - ); - - return lighterOverlay; -}; diff --git a/app/packages/core/src/components/Modal/Lighter/useBridge.ts b/app/packages/core/src/components/Modal/Lighter/useBridge.ts index cf8bcba69e9..657735c2abe 100644 --- a/app/packages/core/src/components/Modal/Lighter/useBridge.ts +++ b/app/packages/core/src/components/Modal/Lighter/useBridge.ts @@ -2,12 +2,11 @@ * Copyright 2017-2025, Voxel51, Inc. */ -import { Scene2D } from "@fiftyone/lighter"; -import { colorScheme, colorSeed } from "@fiftyone/state"; +import type { Scene2D } from "@fiftyone/lighter"; import { useEffect } from "react"; -import { useRecoilValue } from "recoil"; -import { useOverlayPersistence } from "./useOverlayPersistence"; +import useColorMappingContext from "./useColorMappingContext"; import { useLighterTooltipEventHandler } from "./useLighterTooltipEventHandler"; +import { useOverlayPersistence } from "./useOverlayPersistence"; /** * Hook that bridges FiftyOne state management system with Lighter. @@ -17,11 +16,9 @@ import { useLighterTooltipEventHandler } from "./useLighterTooltipEventHandler"; * 2. We trigger certain events into "FiftyOne state" world based on user interactions in Lighter. */ export const useBridge = (scene: Scene2D | null) => { - const currentColorScheme = useRecoilValue(colorScheme); - const currentColorSeed = useRecoilValue(colorSeed); - useLighterTooltipEventHandler(scene); useOverlayPersistence(scene); + const context = useColorMappingContext(); // Effect to update scene with color scheme changes useEffect(() => { @@ -30,14 +27,11 @@ export const useBridge = (scene: Scene2D | null) => { } // Update the scene's color mapping context - scene.updateColorMappingContext({ - colorScheme: currentColorScheme, - seed: currentColorSeed, - }); + scene.updateColorMappingContext(context); // Mark all overlays as dirty to trigger re-rendering with new colors - scene.getAllOverlays().forEach((overlay) => { + for (const overlay of scene.getAllOverlays()) { overlay.markDirty(); - }); - }, [scene, currentColorScheme, currentColorSeed]); + } + }, [scene, context]); }; diff --git a/app/packages/core/src/components/Modal/Lighter/useColorMappingContext.ts b/app/packages/core/src/components/Modal/Lighter/useColorMappingContext.ts new file mode 100644 index 00000000000..32b87610afc --- /dev/null +++ b/app/packages/core/src/components/Modal/Lighter/useColorMappingContext.ts @@ -0,0 +1,15 @@ +import { colorScheme, colorSeed } from "@fiftyone/state"; +import { useMemo } from "react"; +import { useRecoilValue } from "recoil"; + +export default function useColorMappingContext() { + const currentColorScheme = useRecoilValue(colorScheme); + const currentColorSeed = useRecoilValue(colorSeed); + return useMemo( + () => ({ + colorScheme: currentColorScheme, + seed: currentColorSeed, + }), + [currentColorScheme, currentColorSeed] + ); +} diff --git a/app/packages/core/src/components/Modal/Lighter/useLighterTooltipEventHandler.ts b/app/packages/core/src/components/Modal/Lighter/useLighterTooltipEventHandler.ts index fde494a4983..3da9a701212 100644 --- a/app/packages/core/src/components/Modal/Lighter/useLighterTooltipEventHandler.ts +++ b/app/packages/core/src/components/Modal/Lighter/useLighterTooltipEventHandler.ts @@ -2,11 +2,12 @@ * Copyright 2017-2025, Voxel51, Inc. */ +import type { Scene2D } from "@fiftyone/lighter"; +import { LIGHTER_EVENTS } from "@fiftyone/lighter"; +import type { Hoverable } from "@fiftyone/lighter/src/types"; import * as fos from "@fiftyone/state"; import { useEffect } from "react"; import { useRecoilCallback } from "recoil"; -import { LIGHTER_EVENTS, Scene2D } from "@fiftyone/lighter"; -import type { Hoverable } from "@fiftyone/lighter/src/types"; /** * Hook that handles tooltip events for lighter overlays. diff --git a/app/packages/core/src/components/Modal/Lighter/useOverlayPersistence.ts b/app/packages/core/src/components/Modal/Lighter/useOverlayPersistence.ts index 62afa15142a..cf680ec0151 100644 --- a/app/packages/core/src/components/Modal/Lighter/useOverlayPersistence.ts +++ b/app/packages/core/src/components/Modal/Lighter/useOverlayPersistence.ts @@ -6,11 +6,12 @@ import { addLabel, removeLabel, } from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/useLabels"; +import type { OverlayEventDetail, Scene2D } from "@fiftyone/lighter"; +import { LIGHTER_EVENTS } from "@fiftyone/lighter"; import { useOperatorExecutor } from "@fiftyone/operators"; import { DETECTION } from "@fiftyone/utilities"; import { useSetAtom } from "jotai"; import { useCallback, useEffect } from "react"; -import { LIGHTER_EVENTS, OverlayEventDetail, Scene2D } from "@fiftyone/lighter"; /** * Hook that handles overlay persistence events. @@ -102,5 +103,5 @@ export const useOverlayPersistence = (scene: Scene2D | null) => { scene.off(LIGHTER_EVENTS.DO_PERSIST_OVERLAY, handlePersistOverlay); scene.off(LIGHTER_EVENTS.DO_REMOVE_OVERLAY, handleRemoveOverlay); }; - }, [scene]); + }, [handlePersistOverlay, handleRemoveOverlay, scene]); }; diff --git a/app/packages/core/src/components/Modal/Sidebar/Annotate/Actions.tsx b/app/packages/core/src/components/Modal/Sidebar/Annotate/Actions.tsx index 850a002704b..d938a98da06 100644 --- a/app/packages/core/src/components/Modal/Sidebar/Annotate/Actions.tsx +++ b/app/packages/core/src/components/Modal/Sidebar/Annotate/Actions.tsx @@ -1,11 +1,11 @@ import { useLighter } from "@fiftyone/lighter"; -import { CLASSIFICATION, DETECTION } from "@fiftyone/utilities"; -import { atom, useAtomValue } from "jotai"; -import React from "react"; +import { is3DDataset } from "@fiftyone/state"; +import { CLASSIFICATION, DETECTION, POLYLINE } from "@fiftyone/utilities"; +import { PolylineOutlined } from "@mui/icons-material"; +import { useRecoilValue } from "recoil"; import styled from "styled-components"; import { ItemLeft, ItemRight } from "./Components"; import useCreate from "./Edit/useCreate"; -import { classificationFields, detectionFields } from "./state"; import useShowModal from "./useShowModal"; const ActionsDiv = styled.div` @@ -127,16 +127,10 @@ const Move = () => { ); }; -const hasClassifications = atom((get) => get(classificationFields)); - const Classification = () => { const create = useCreate(CLASSIFICATION); - const enabled = useAtomValue(hasClassifications); return ( - create() : undefined} - > + { ); }; -const hasDetections = atom((get) => get(detectionFields)); - const Detection = () => { const create = useCreate(DETECTION); - const enabled = useAtomValue(hasDetections); return ( - create() : undefined} - className={enabled ? "" : "disabled"} - > + { ); }; +const Polyline = () => { + const create = useCreate(POLYLINE); + return ( + + + + ); +}; + export const Undo = () => { const { undo, canUndo: enabled } = useLighter(); @@ -231,6 +228,7 @@ const Schema = () => { }; const Actions = () => { + const is3D = useRecoilValue(is3DDataset); return ( <> @@ -238,7 +236,7 @@ const Actions = () => { - + {is3D ? : } diff --git a/app/packages/core/src/components/Modal/Sidebar/Annotate/Annotate.tsx b/app/packages/core/src/components/Modal/Sidebar/Annotate/Annotate.tsx index 65b23237fcd..2accea0a94a 100644 --- a/app/packages/core/src/components/Modal/Sidebar/Annotate/Annotate.tsx +++ b/app/packages/core/src/components/Modal/Sidebar/Annotate/Annotate.tsx @@ -1,4 +1,5 @@ import { LoadingSpinner } from "@fiftyone/components"; +import { lighterSceneAtom } from "@fiftyone/lighter"; import { EntryKind } from "@fiftyone/state"; import { Typography } from "@mui/material"; import { atom, useAtomValue } from "jotai"; @@ -83,8 +84,9 @@ const Annotate = () => { const showImport = useAtomValue(showImportPage); const loading = useAtomValue(schemas) === null; const editing = useAtomValue(isEditing); + const scene = useAtomValue(lighterSceneAtom); - if (loading) { + if (loading || !scene) { return ; } diff --git a/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/AnnotationSchema.tsx b/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/AnnotationSchema.tsx index 56dc874ffb8..b75691692b4 100644 --- a/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/AnnotationSchema.tsx +++ b/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/AnnotationSchema.tsx @@ -1,7 +1,12 @@ +import { + BoundingBoxOverlay, + UpdateLabelCommand, + useLighter, +} from "@fiftyone/lighter"; import { useAtom, useAtomValue } from "jotai"; -import React, { useMemo } from "react"; +import { useMemo } from "react"; import { SchemaIOComponent } from "../../../../../plugins/SchemaIO"; -import { current, currentSchema } from "./state"; +import { currentData, currentSchema } from "./state"; const createInput = (name: string) => { return { @@ -14,20 +19,36 @@ const createInput = (name: string) => { }; }; +const createTags = (name: string, choices: string[]) => { + return { + type: "array", + view: { + name: "AutocompleteView", + label: name, + component: "AutocompleteView", + allow_user_input: false, + choices: choices.map((choice) => ({ + name: "Choice", + label: choice, + value: choice, + })), + }, + required: true, + }; +}; + const createSelect = (name: string, choices: string[]) => { return { - [name]: { - type: "string", - view: { - name: "DropdownView", - label: "Classes", - component: "DropdownView", - choices: choices.map((choice) => ({ - name: "Choice", - label: choice, - value: choice, - })), - }, + type: "string", + view: { + name: "DropdownView", + label: name, + component: "DropdownView", + choices: choices.map((choice) => ({ + name: "Choice", + label: choice, + value: choice, + })), }, }; }; @@ -40,35 +61,54 @@ const useSchema = () => { const attributes = config?.attributes; + properties.label = createSelect("label", config?.classes ?? []); + for (const attr in attributes) { - properties[attr] = createInput(attr); - } + if (attributes[attr].type === "input") { + properties[attr] = createInput(attr); + } - console.log(properties); + if (attributes[attr].type === "tags") { + properties[attr] = createTags(attr, attributes[attr].values); + } + + if (attributes[attr].type === "text") { + throw "text"; + } + } return { type: "object", view: { component: "ObjectView", }, - properties: { - classes: createSelect("Classes", config?.classes), - ...properties, - }, + properties, }; }, [config]); }; const AnnotationSchema = () => { const schema = useSchema(); - const [label, setLabel] = useAtom(current); + const [data, save] = useAtom(currentData); + const lighter = useLighter(); + return (
{ - console.log(a); + data={data} + onChange={(changes) => { + save(changes); + const overlay = lighter.getOverlay(data?._id); + + if (overlay instanceof BoundingBoxOverlay) { + lighter.scene?.executeCommand( + new UpdateLabelCommand(overlay, overlay.label, { + ...overlay.label, + ...changes, + }) + ); + } }} />
diff --git a/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/Edit.tsx b/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/Edit.tsx index fde582a898a..c1f242c6d74 100644 --- a/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/Edit.tsx +++ b/app/packages/core/src/components/Modal/Sidebar/Annotate/Edit/Edit.tsx @@ -1,3 +1,4 @@ +import { DETECTION } from "@fiftyone/utilities"; import { useAtom, useAtomValue } from "jotai"; import React from "react"; import styled from "styled-components"; @@ -5,6 +6,7 @@ import AnnotationSchema from "./AnnotationSchema"; import Field from "./Field"; import Footer from "./Footer"; import Header from "./Header"; +import Position from "./Position"; import { current, currentField } from "./state"; const ContentContainer = styled.div` @@ -40,6 +42,7 @@ export default function Edit() {
+ {label.type === DETECTION && } {field && }