-
-
+
+
{icon}
{label}
-
- {value.toLocaleString()}
-
-
+
+
{value.toLocaleString()}
+
+
{
transition: "width 0.4s cubic-bezier(.4,2,.6,1)",
}}
/>
-
+
);
return (
-
-
+
+
FPS{" "}
-
+
{perfStats.fps.toFixed(1)}
-
-
+
+
{
margin: "4px 0 2px 0",
}}
/>
-
{
value={perfStats.calls}
barKey="calls"
/>
- {
value={perfStats.triangles}
barKey="triangles"
/>
- {
value={perfStats.points}
barKey="points"
/>
- {
value={perfStats.geometries}
barKey="geometries"
/>
- {
value={perfStats.textures}
barKey="textures"
/>
- {
value={perfStats.programs}
barKey="programs"
/>
-
+
);
};
export const StatusBar = ({
cameraRef,
}: {
- cameraRef: RefObject
;
+ cameraRef: RefObject;
}) => {
+ const theme = useTheme();
const containerRef = useRef(null);
const [showPerfStatus, setShowPerfStatus] = useRecoilState(isStatusBarOnAtom);
const setActiveNode = useSetRecoilState(activeNodeAtom);
+ const segmentPolylineState = useRecoilState(segmentPolylineStateAtom)[0];
const springProps = useSpring({
transform: showPerfStatus ? "translateY(10%)" : "translateY(0%)",
@@ -339,44 +391,38 @@ export const StatusBar = ({
return (
{!showPerfStatus && (
-
+
-
+
+ )}
+
+ {segmentPolylineState.isActive && (
+
+
+
+ Snap to first vertex to close • Double click to finish • Escape to
+ cancel
+
+
)}
{showPerfStatus && (
<>
-
+
-
-
+
)}
diff --git a/app/packages/looker-3d/src/annotation/AnnotationPlane.tsx b/app/packages/looker-3d/src/annotation/AnnotationPlane.tsx
new file mode 100644
index 00000000000..b529dc3516a
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/AnnotationPlane.tsx
@@ -0,0 +1,347 @@
+import { Line, useCursor } from "@react-three/drei";
+import { useFrame } from "@react-three/fiber";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useRecoilState, useRecoilValue } from "recoil";
+import * as THREE from "three";
+import { useFo3dContext } from "../fo3d/context";
+import { Transformable } from "../labels/shared/TransformControls";
+import {
+ annotationPlaneAtom,
+ currentArchetypeSelectedForTransformAtom,
+ segmentPolylineStateAtom,
+ transformModeAtom,
+} from "../state";
+
+interface AnnotationPlaneProps {
+ showTransformControls?: boolean;
+ viewType?: "top" | "bottom" | "right" | "left" | "front" | "back";
+ panelType?: "side" | "main";
+}
+
+export const AnnotationPlane = ({
+ showTransformControls = true,
+ viewType = "top",
+ panelType = "main",
+}: AnnotationPlaneProps) => {
+ const [annotationPlane, setAnnotationPlane] =
+ useRecoilState(annotationPlaneAtom);
+
+ const isSegmenting = useRecoilValue(segmentPolylineStateAtom).isActive;
+ const transformMode = useRecoilValue(transformModeAtom);
+
+ const [
+ currentArchetypeSelectedForTransform,
+ setCurrentArchetypeSelectedForTransform,
+ ] = useRecoilState(currentArchetypeSelectedForTransformAtom);
+
+ const isSelected =
+ currentArchetypeSelectedForTransform === "annotation-plane";
+
+ const { sceneBoundingBox, upVector } = useFo3dContext();
+ const meshRef = useRef(null);
+ const lineRef = useRef(null);
+ const materialRef = useRef(null);
+ const transformControlsRef = useRef(null);
+ const [isHovered, setIsHovered] = useState(false);
+ const [isDragging, setIsDragging] = useState(false);
+ const [isMouseDown, setIsMouseDown] = useState(false);
+ const [dragStartPosition, setDragStartPosition] = useState<{
+ x: number;
+ y: number;
+ } | null>(null);
+
+ useEffect(() => {
+ setCurrentArchetypeSelectedForTransform(null);
+ setAnnotationPlane((prev) => ({ ...prev, enabled: false }));
+ }, [upVector]);
+
+ const planeSize = useMemo(() => {
+ if (!sceneBoundingBox) return 10;
+
+ const size = sceneBoundingBox.getSize(new THREE.Vector3());
+
+ if (upVector) {
+ // Find the two dimensions that are most orthogonal to the up vector
+ const upAbs = new THREE.Vector3(
+ Math.abs(upVector.x),
+ Math.abs(upVector.y),
+ Math.abs(upVector.z)
+ );
+
+ // Get the two largest orthogonal dimensions
+ const dimensions = [size.x, size.y, size.z];
+ const orthogonalSizes = [];
+
+ // Find dimensions where up vector component is smallest (most orthogonal)
+ for (let i = 0; i < 3; i++) {
+ if (upAbs.toArray()[i] < 0.5) {
+ // Less than 0.5 means mostly orthogonal
+ orthogonalSizes.push(Math.round(dimensions[i]));
+ }
+ }
+
+ // 2 because we want to make the plane slightly larger than the scene
+ if (orthogonalSizes.length > 0) {
+ return Math.max(...orthogonalSizes, 10) * 2;
+ }
+ }
+
+ return Math.max(size.x, size.y, size.z, 10) * 2;
+ }, [sceneBoundingBox, upVector]);
+
+ const position = useMemo(
+ () => new THREE.Vector3(...annotationPlane.position),
+ [annotationPlane]
+ );
+ const quaternion = useMemo(
+ () => new THREE.Quaternion(...annotationPlane.quaternion),
+ [annotationPlane]
+ );
+
+ // Determine which axes to show based on upVector and mode
+ const transformControlsProps = useMemo(() => {
+ // For rotate mode, always show all axes
+ if (transformMode === "rotate") {
+ return { showX: true, showY: true, showZ: true };
+ }
+
+ if (!upVector) {
+ return { showX: true, showY: true, showZ: true };
+ }
+
+ // Find which axis is most aligned with the up vector
+ const upAbs = new THREE.Vector3(
+ Math.abs(upVector.x),
+ Math.abs(upVector.y),
+ Math.abs(upVector.z)
+ );
+
+ const maxComponent = Math.max(upAbs.x, upAbs.y, upAbs.z);
+
+ // Determine which axis is the up axis
+ const isXUp = Math.abs(upVector.x) === maxComponent;
+ const isYUp = Math.abs(upVector.y) === maxComponent;
+ const isZUp = Math.abs(upVector.z) === maxComponent;
+
+ // For translate mode, only show the axis aligned with upVector
+ return {
+ showX: isXUp,
+ showY: isYUp,
+ showZ: isZUp,
+ };
+ }, [upVector, transformMode]);
+
+ // This effect syncs the showX, showY, and showZ values with the transformControlsProps values
+ useEffect(() => {
+ if (!annotationPlane.enabled) return;
+
+ setAnnotationPlane((prev) => ({
+ ...prev,
+ showX: transformControlsProps.showX,
+ showY: transformControlsProps.showY,
+ showZ: transformControlsProps.showZ,
+ }));
+ }, [transformControlsProps, annotationPlane.enabled, isSelected]);
+
+ useCursor(
+ isHovered && isSelected && !isSegmenting,
+ "pointer",
+ isSegmenting ? "crosshair" : "auto"
+ );
+
+ // Simple pulsing animation for scale and opacity for visibility
+ useFrame((state) => {
+ if (materialRef.current && meshRef.current && panelType === "main") {
+ const time = state.clock.getElapsedTime();
+
+ const baseOpacity = isSelected ? 0.2 : 0.08;
+ const pulseOpacity = Math.sin(time * 2) * 0.05;
+ materialRef.current.opacity = baseOpacity + pulseOpacity;
+
+ const baseScale = 1;
+ const pulseScale = Math.sin(time * 1.5) * 0.01;
+ meshRef.current.scale.setScalar(baseScale + pulseScale);
+ }
+ });
+
+ const handleMouseDown = useCallback((event: any) => {
+ setIsMouseDown(true);
+ setDragStartPosition({ x: event.clientX, y: event.clientY });
+ }, []);
+
+ const handleMouseMove = useCallback(
+ (event: any) => {
+ if (isMouseDown && dragStartPosition) {
+ const deltaX = Math.abs(event.clientX - dragStartPosition.x);
+ const deltaY = Math.abs(event.clientY - dragStartPosition.y);
+ const threshold = 5; // pixels
+
+ if (deltaX > threshold || deltaY > threshold) {
+ setIsDragging(true);
+ }
+ }
+ },
+ [isMouseDown, dragStartPosition]
+ );
+
+ const handleMouseUp = useCallback(() => {
+ setIsMouseDown(false);
+ setDragStartPosition(null);
+ // Reset dragging state after a short delay to allow click handlers to check it
+ setTimeout(() => setIsDragging(false), 0);
+ }, []);
+
+ const handlePlaneClick = useCallback(
+ (event: any) => {
+ if (!showTransformControls || isSegmenting) return;
+
+ event.stopPropagation();
+
+ if (!isDragging) {
+ setCurrentArchetypeSelectedForTransform((prev) =>
+ prev === "annotation-plane" ? null : "annotation-plane"
+ );
+ }
+ },
+ [showTransformControls, isDragging, isSegmenting]
+ );
+
+ const handleTransformStart = useCallback(() => {
+ setIsDragging(true);
+ }, []);
+
+ const syncAnnotationPlaneTransformation = useCallback(() => {
+ if (transformControlsRef.current && meshRef.current) {
+ const newPosition: [number, number, number] = [
+ meshRef.current.position.x,
+ meshRef.current.position.y,
+ meshRef.current.position.z,
+ ];
+
+ const newQuaternion: [number, number, number, number] = [
+ meshRef.current.quaternion.x,
+ meshRef.current.quaternion.y,
+ meshRef.current.quaternion.z,
+ meshRef.current.quaternion.w,
+ ];
+
+ setAnnotationPlane((prev) => ({
+ ...prev,
+ position: newPosition,
+ quaternion: newQuaternion,
+ }));
+ }
+ }, []);
+
+ const handleTransformEnd = useCallback(() => {
+ syncAnnotationPlaneTransformation();
+ setIsDragging(false);
+ }, [syncAnnotationPlaneTransformation]);
+
+ const handleTransformChange = useCallback(() => {
+ syncAnnotationPlaneTransformation();
+ }, [syncAnnotationPlaneTransformation]);
+
+ useEffect(() => {
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (event.key === "Escape" && isSelected) {
+ setCurrentArchetypeSelectedForTransform(null);
+ setIsDragging(false);
+ event.stopImmediatePropagation();
+ event.preventDefault();
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown);
+
+ return () => {
+ document.removeEventListener("keydown", handleKeyDown);
+ };
+ }, [isSelected]);
+
+ if (!annotationPlane.enabled) {
+ return null;
+ }
+
+ const shouldShowFullPlane =
+ panelType === "main" ||
+ (panelType === "side" && (viewType === "top" || viewType === "bottom"));
+
+ if (showTransformControls && shouldShowFullPlane) {
+ return (
+
+ setIsHovered(true)}
+ onPointerOut={() => setIsHovered(false)}
+ onPointerDown={handleMouseDown}
+ onPointerMove={handleMouseMove}
+ onPointerUp={handleMouseUp}
+ onClick={handlePlaneClick}
+ renderOrder={1000}
+ >
+
+
+
+
+ );
+ } else {
+ const halfSize = planeSize / 2;
+ let points: [number, number, number][];
+
+ if (viewType === "left" || viewType === "right") {
+ points = [
+ [0, -halfSize, 0],
+ [0, halfSize, 0],
+ ];
+ } else {
+ points = [
+ [-halfSize, 0, 0],
+ [halfSize, 0, 0],
+ ];
+ }
+
+ return (
+ setIsHovered(true)}
+ onPointerOut={() => setIsHovered(false)}
+ onPointerDown={handleMouseDown}
+ onPointerMove={handleMouseMove}
+ onPointerUp={handleMouseUp}
+ onClick={handlePlaneClick}
+ >
+
+
+ );
+ }
+};
diff --git a/app/packages/looker-3d/src/annotation/Crosshair3D.tsx b/app/packages/looker-3d/src/annotation/Crosshair3D.tsx
new file mode 100644
index 00000000000..303c56bc86e
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/Crosshair3D.tsx
@@ -0,0 +1,123 @@
+import { useTheme } from "@fiftyone/components";
+import { Html } from "@react-three/drei";
+import { useThree } from "@react-three/fiber";
+import { useMemo } from "react";
+import { useRecoilState, useRecoilValue } from "recoil";
+import styled from "styled-components";
+import * as THREE from "three";
+import { useFo3dContext } from "../fo3d/context";
+import {
+ isCurrentlyTransformingAtom,
+ sharedCursorPositionAtom,
+} from "../state";
+
+const CROSS_HAIR_SIZE = 20;
+const LINE_WIDTH = 2;
+const OPACITY = 0.7;
+
+const HtmlContainer = styled.div`
+ position: relative;
+ pointer-events: none;
+ z-index: 1000;
+`;
+
+const CrosshairBox = styled.div`
+ position: absolute;
+ left: -${CROSS_HAIR_SIZE / 2}px;
+ top: -${CROSS_HAIR_SIZE / 2}px;
+ width: ${CROSS_HAIR_SIZE}px;
+ height: ${CROSS_HAIR_SIZE}px;
+ pointer-events: none;
+`;
+
+const Horizontal = styled.div<{ $color: string }>`
+ position: absolute;
+ left: 0;
+ top: 50%;
+ width: 100%;
+ height: ${LINE_WIDTH}px;
+ background-color: ${(p) => p.$color};
+ opacity: ${OPACITY};
+ transform: translateY(-50%);
+`;
+
+const Vertical = styled.div<{ $color: string }>`
+ position: absolute;
+ left: 50%;
+ top: 0;
+ width: ${LINE_WIDTH}px;
+ height: 100%;
+ background-color: ${(p) => p.$color};
+ opacity: ${OPACITY};
+ transform: translateX(-50%);
+`;
+
+export const Crosshair3D = () => {
+ const { camera } = useThree();
+ const { sceneBoundingBox } = useFo3dContext();
+ const theme = useTheme();
+ const isCurrentlyTransforming = useRecoilValue(isCurrentlyTransformingAtom);
+
+ const worldPosition = useRecoilValue(sharedCursorPositionAtom);
+
+ const worldVector = useMemo(() => {
+ if (!worldPosition) return null;
+
+ const vector = new THREE.Vector3(...worldPosition);
+
+ if (sceneBoundingBox) {
+ vector.clamp(sceneBoundingBox.min, sceneBoundingBox.max);
+ }
+
+ return vector;
+ }, [worldPosition, sceneBoundingBox]);
+
+ // Convert 3D world position to 2D screen coordinates
+ const screenPosition = useMemo(() => {
+ if (!worldVector) return null;
+
+ return worldVector.clone().project(camera);
+ }, [worldVector, camera]);
+
+ // Calculate scale based on camera zoom (inverse)
+ const scale = useMemo(() => {
+ if (camera instanceof THREE.OrthographicCamera) {
+ return 1 / camera.zoom;
+ }
+ return 1;
+ }, [camera]);
+
+ if (isCurrentlyTransforming) {
+ return null;
+ }
+
+ if (!screenPosition) {
+ return null;
+ }
+
+ // Only render if the point is in front of the camera
+ if (screenPosition.z > 1) {
+ return null;
+ }
+
+ // Only render if the point is within the viewport bounds (NDC: -1 to 1)
+ if (
+ screenPosition.x < -1 ||
+ screenPosition.x > 1 ||
+ screenPosition.y < -1 ||
+ screenPosition.y > 1
+ ) {
+ return null;
+ }
+
+ return (
+
+
+
+
+
+
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/MultiPanelView.tsx b/app/packages/looker-3d/src/annotation/MultiPanelView.tsx
new file mode 100644
index 00000000000..c8f384777ed
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/MultiPanelView.tsx
@@ -0,0 +1,485 @@
+import { useTheme } from "@fiftyone/components";
+import { useBrowserStorage } from "@fiftyone/state";
+import { MenuItem, Select } from "@mui/material";
+import {
+ Bounds,
+ CameraControls,
+ MapControls,
+ OrthographicCamera,
+ View,
+} from "@react-three/drei";
+import { Canvas } from "@react-three/fiber";
+import CameraControlsImpl from "camera-controls";
+import {
+ useCallback,
+ useEffect,
+ useLayoutEffect,
+ useMemo,
+ useRef,
+ useState,
+} from "react";
+import { useRecoilCallback } from "recoil";
+import styled from "styled-components";
+import * as THREE from "three";
+import { Box3, Vector3 } from "three";
+import { PcdColorMapTunnel } from "../components/PcdColormapModal";
+import { StatusBarRootContainer } from "../containers";
+import { useFo3dContext } from "../fo3d/context";
+import { Fo3dSceneContent } from "../fo3d/Fo3dCanvas";
+import { FoSceneComponent } from "../fo3d/FoScene";
+import { Gizmos } from "../fo3d/Gizmos";
+import HoverMetadataHUD from "../fo3d/HoverMetadataHUD";
+import { Lights } from "../fo3d/scene-controls/lights/Lights";
+import { FoScene } from "../hooks";
+import { ThreeDLabels } from "../labels";
+import {
+ activeNodeAtom,
+ currentArchetypeSelectedForTransformAtom,
+ currentHoveredPointAtom,
+ selectedPolylineVertexAtom,
+} from "../state";
+import { StatusBar } from "../StatusBar";
+import { AnnotationPlane } from "./AnnotationPlane";
+import { Crosshair3D } from "./Crosshair3D";
+import { SegmentPolylineRenderer } from "./SegmentPolylineRenderer";
+
+const CANVAS_WRAPPER_ID = "sample3d-canvas-wrapper";
+
+const GridMain = styled.main`
+ display: grid;
+ grid-template-areas: "main top" "main bottom";
+ grid-template-columns: 2fr 1.1fr;
+ grid-template-rows: 1fr 1fr;
+ height: 100%;
+ width: 100%;
+ gap: 1px;
+`;
+
+const AbsoluteFill = styled.div`
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+`;
+
+const MainPanelContainer = styled.div`
+ grid-area: main;
+ position: relative;
+ z-index: 2;
+`;
+
+const SidePanelContainer = styled.div<{ $area: string }>`
+ grid-area: ${(p) => p.$area};
+ position: relative;
+ z-index: 200;
+`;
+
+const ViewSelectorWrapper = styled.div`
+ position: absolute;
+ top: 10px;
+ left: 10px;
+ z-index: 1000;
+`;
+
+/**
+ * Calculate camera position for different side panel views based on upVector and lookAt point
+ */
+const calculateCameraPositionForSidePanel = (
+ viewType: ViewType,
+ upVector: Vector3,
+ lookAt: Vector3,
+ sceneBoundingBox: Box3 | null
+): Vector3 => {
+ if (!sceneBoundingBox) {
+ // Fallback to default positions if no bounding box
+ const defaultPositions = {
+ Top: [0, 10, 0] as [number, number, number],
+ Bottom: [0, -10, 0] as [number, number, number],
+ Left: [-10, 0, 0] as [number, number, number],
+ Right: [10, 0, 0] as [number, number, number],
+ Back: [0, 0, -10] as [number, number, number],
+ Front: [0, 0, 10] as [number, number, number],
+ };
+ return new Vector3(...defaultPositions[viewType]);
+ }
+
+ const size = new Vector3();
+ sceneBoundingBox.getSize(size);
+ const maxSize = Math.max(size.x, size.y, size.z);
+ const distance = maxSize * 2.5;
+
+ const upDir = upVector.clone().normalize();
+ const center = lookAt.clone();
+
+ // Create orthogonal vectors for different views
+ let direction: Vector3;
+
+ switch (viewType) {
+ case "Top":
+ direction = upDir.clone();
+ break;
+ case "Bottom":
+ direction = upDir.clone().negate();
+ break;
+ case "Left":
+ // Create a vector perpendicular to up vector
+ if (Math.abs(upDir.y) > 0.9) {
+ // If up is mostly Y, use negative X axis for left
+ direction = new Vector3(-1, 0, 0);
+ } else {
+ const right = new Vector3(0, 1, 0).cross(upDir).normalize();
+ direction = right.negate();
+ }
+ break;
+ case "Right":
+ // Opposite of Left
+ if (Math.abs(upDir.y) > 0.9) {
+ direction = new Vector3(1, 0, 0);
+ } else {
+ direction = new Vector3(0, 1, 0).cross(upDir).normalize();
+ }
+ break;
+ case "Front":
+ // Create a vector perpendicular to both up and left
+ if (Math.abs(upDir.y) > 0.9) {
+ direction = new Vector3(0, 0, 1);
+ } else {
+ const left = new Vector3(0, -1, 0).cross(upDir).normalize();
+ direction = upDir.clone().cross(left).normalize();
+ }
+ break;
+ case "Back":
+ // Opposite of Front
+ if (Math.abs(upDir.y) > 0.9) {
+ direction = new Vector3(0, 0, -1);
+ } else {
+ const left = new Vector3(0, 1, 0).cross(upDir).normalize();
+ direction = upDir.clone().cross(left).normalize();
+ }
+ break;
+ default:
+ direction = upDir.clone();
+ }
+
+ return center.clone().add(direction.multiplyScalar(distance));
+};
+
+type ViewType = "Top" | "Bottom" | "Left" | "Right" | "Front" | "Back";
+
+interface MultiPanelViewState {
+ top: ViewType;
+ bottom: ViewType;
+}
+
+const defaultState: MultiPanelViewState = {
+ top: "Front",
+ bottom: "Right",
+};
+
+interface MultiPanelViewProps {
+ assetsGroupRef: React.RefObject;
+ cameraControlsRef: React.MutableRefObject;
+ cameraRef: React.MutableRefObject;
+ defaultCameraPosition: Vector3;
+ foScene: FoScene;
+ sample: any;
+}
+
+export const MultiPanelView = ({
+ assetsGroupRef,
+ cameraControlsRef,
+ cameraRef,
+ defaultCameraPosition,
+ foScene,
+ sample,
+}: MultiPanelViewProps) => {
+ const { isSceneInitialized, upVector, lookAt, sceneBoundingBox } =
+ useFo3dContext();
+ const [panelState, setPanelState] = useBrowserStorage(
+ "fo3d-multi-panel-state",
+ defaultState
+ );
+
+ const containerRef = useRef(null);
+
+ const resetActiveNode = useRecoilCallback(
+ ({ set }) =>
+ (event: MouseEvent | null) => {
+ if (event?.type === "contextmenu") return;
+ set(activeNodeAtom, null);
+ set(currentHoveredPointAtom, null);
+ set(selectedPolylineVertexAtom, null);
+ set(currentArchetypeSelectedForTransformAtom, null);
+ },
+ []
+ );
+
+ useLayoutEffect(() => {
+ const canvas = document.getElementById(CANVAS_WRAPPER_ID);
+ if (canvas) {
+ canvas.querySelector("canvas")?.setAttribute("canvas-loaded", "true");
+ }
+ }, [isSceneInitialized]);
+
+ const [autoRotate] = useBrowserStorage("fo3dAutoRotate", false);
+
+ const [pointCloudSettings] = useBrowserStorage("fo3dPointCloudSettings", {
+ enableTooltip: false,
+ rayCastingSensitivity: "medium",
+ });
+
+ const setPanelView = useCallback(
+ (
+ which: keyof Pick,
+ view: ViewType
+ ) => {
+ setPanelState((prev) => ({ ...prev, [which]: view }));
+ },
+ [setPanelState]
+ );
+
+ return (
+
+
+
+
+
+
+
+
+ setPanelView("top", view)}
+ foScene={foScene}
+ upVector={upVector}
+ lookAt={lookAt}
+ sceneBoundingBox={sceneBoundingBox}
+ isSceneInitialized={isSceneInitialized}
+ sample={sample}
+ />
+
+ setPanelView("bottom", view)}
+ foScene={foScene}
+ upVector={upVector}
+ lookAt={lookAt}
+ sceneBoundingBox={sceneBoundingBox}
+ isSceneInitialized={isSceneInitialized}
+ sample={sample}
+ />
+
+
+
+
+
+ );
+};
+
+const MainPanel = ({
+ cameraRef,
+ cameraControlsRef,
+ defaultCameraPosition,
+ autoRotate,
+ foScene,
+ upVector,
+ isSceneInitialized,
+ sample,
+ pointCloudSettings,
+ assetsGroupRef,
+}: {
+ cameraRef: React.RefObject;
+ cameraControlsRef: React.RefObject;
+ defaultCameraPosition: THREE.Vector3;
+ autoRotate: boolean;
+ foScene: any;
+ upVector: Vector3 | null;
+ isSceneInitialized: boolean;
+ sample: any;
+ pointCloudSettings: any;
+ assetsGroupRef: React.RefObject;
+}) => {
+ return (
+
+
+
+
+
+ );
+};
+
+const SidePanel = ({
+ which,
+ view,
+ setView,
+ foScene,
+ upVector,
+ lookAt,
+ sceneBoundingBox,
+ isSceneInitialized,
+ sample,
+}: {
+ which: "top" | "middle" | "bottom";
+ view: ViewType;
+ setView: (view: ViewType) => void;
+ foScene: FoScene;
+ upVector: Vector3 | null;
+ lookAt: Vector3 | null;
+ sceneBoundingBox: Box3 | null;
+ isSceneInitialized: boolean;
+ sample: any;
+}) => {
+ const position = useMemo(
+ () =>
+ upVector && lookAt
+ ? calculateCameraPositionForSidePanel(
+ view,
+ upVector,
+ lookAt,
+ sceneBoundingBox
+ )
+ : new Vector3(0, 10, 0),
+ [view, upVector, lookAt, sceneBoundingBox]
+ );
+
+ const theme = useTheme();
+
+ const cameraRef = useRef();
+
+ // We need to observe the bounds for a short period of time to ensure the camera is in the correct position
+ // But turn it off so that users can pan/zoom and work with a stable scene
+ const [observe, setObserve] = useState(true);
+
+ useEffect(() => {
+ const timer = setTimeout(() => setObserve(false), 1000);
+ return () => clearTimeout(timer);
+ }, []);
+
+ return (
+
+
+
+
+
+
+
+
+
+ {isSceneInitialized && (
+
+ )}
+
+
+
+
+
+
+
+
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/PolylinePointMarker.tsx b/app/packages/looker-3d/src/annotation/PolylinePointMarker.tsx
new file mode 100644
index 00000000000..752a50ac7d7
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/PolylinePointMarker.tsx
@@ -0,0 +1,260 @@
+import { useCursor } from "@react-three/drei";
+import { useFrame } from "@react-three/fiber";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { useRecoilState, useSetRecoilState } from "recoil";
+import { Mesh, Vector3, Matrix4 } from "three";
+import { LABEL_3D_ANNOTATION_POINT_SELECTED_FOR_TRANSFORMATION_COLOR } from "../constants";
+import { Transformable } from "../labels/shared/TransformControls";
+import {
+ currentArchetypeSelectedForTransformAtom,
+ editSegmentsModeAtom,
+ hoveredPolylineInfoAtom,
+ segmentPolylineStateAtom,
+ selectedPolylineVertexAtom,
+ tempVertexTransformsAtom,
+ transformModeAtom,
+} from "../state";
+import type { SelectedPoint } from "./types";
+
+interface PolylinePointMarkerProps {
+ position: Vector3;
+ color?: string;
+ size?: number;
+ pulsate?: boolean;
+ isDraggable?: boolean;
+ labelId: string;
+ segmentIndex: number;
+ pointIndex: number;
+ onPointMove?: (newPosition: Vector3) => void;
+}
+
+export const PolylinePointMarker = ({
+ position,
+ color = "#ffffff",
+ size = 0.05,
+ pulsate = true,
+ isDraggable = false,
+ labelId,
+ segmentIndex,
+ pointIndex,
+ onPointMove,
+}: PolylinePointMarkerProps) => {
+ const meshRef = useRef(null);
+ const transformControlsRef = useRef(null);
+ const [startMatrix, setStartMatrix] = useState(null);
+
+ const [isHovered, setIsHovered] = useState(false);
+
+ const setHoveredPolylineInfo = useSetRecoilState(hoveredPolylineInfoAtom);
+ const setTransformMode = useSetRecoilState(transformModeAtom);
+
+ const [selectedPoint, setSelectedPoint] = useRecoilState(
+ selectedPolylineVertexAtom
+ );
+ const setCurrentArchetypeSelectedForTransform = useSetRecoilState(
+ currentArchetypeSelectedForTransformAtom
+ );
+
+ const setSegmentPolylineState = useSetRecoilState(segmentPolylineStateAtom);
+ const setEditSegmentsMode = useSetRecoilState(editSegmentsModeAtom);
+
+ const isSelected =
+ selectedPoint?.labelId === labelId &&
+ selectedPoint?.segmentIndex === segmentIndex &&
+ selectedPoint?.pointIndex === pointIndex;
+
+ useCursor(isHovered && isDraggable, "grab", "auto");
+
+ const handlePointClick = useCallback(
+ (event: any) => {
+ if (!isDraggable) return;
+
+ event.stopPropagation();
+
+ const newSelectedPoint: SelectedPoint = {
+ labelId,
+ segmentIndex,
+ pointIndex,
+ };
+
+ setSelectedPoint(newSelectedPoint);
+ setCurrentArchetypeSelectedForTransform("point");
+ setTransformMode("translate");
+
+ // Deactivate other modes when selecting a point
+ setSegmentPolylineState((prev) => ({
+ ...prev,
+ isActive: false,
+ }));
+ setEditSegmentsMode(false);
+ },
+ [
+ isDraggable,
+ labelId,
+ segmentIndex,
+ pointIndex,
+ setSelectedPoint,
+ setCurrentArchetypeSelectedForTransform,
+ setTransformMode,
+ setSegmentPolylineState,
+ setEditSegmentsMode,
+ ]
+ );
+
+ const syncPointTransformationToTempStore = useCallback(() => {
+ if (groupRef.current) {
+ const worldPosition = groupRef.current.position.clone();
+ setTempVertexTransforms({
+ position: [worldPosition.x, worldPosition.y, worldPosition.z],
+ quaternion: groupRef.current.quaternion.toArray(),
+ });
+ }
+ }, []);
+
+ const handleTransformStart = useCallback(() => {
+ if (groupRef.current) {
+ // Store the start matrix for computing delta later
+ setStartMatrix(groupRef.current.matrixWorld.clone());
+ }
+ }, []);
+
+ const handleTransformEnd = useCallback(() => {
+ setTempVertexTransforms(null);
+
+ if (groupRef.current && onPointMove && startMatrix) {
+ // Compute world-space delta from start and end matrices
+ const endMatrix = groupRef.current.matrixWorld.clone();
+ const deltaMatrix = endMatrix
+ .clone()
+ .multiply(startMatrix.clone().invert());
+
+ // Extract position delta from the delta matrix
+ const deltaPosition = new Vector3();
+ deltaPosition.setFromMatrixPosition(deltaMatrix);
+
+ const newPosition = position.clone().add(deltaPosition);
+
+ groupRef.current.position.set(0, 0, 0);
+
+ onPointMove(newPosition);
+
+ // note: this is a hack:
+ // transformControls is updating in a buggy way when the point is moved
+ const prevSelectedPoint = selectedPoint;
+ setSelectedPoint(null);
+ setTimeout(() => {
+ setSelectedPoint(prevSelectedPoint);
+ }, 0);
+
+ // Clear the start matrix
+ setStartMatrix(null);
+ }
+ }, [onPointMove, selectedPoint, position, startMatrix]);
+
+ useEffect(() => {
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (event.key === "Escape" && selectedPoint) {
+ setSelectedPoint(null);
+ event.stopImmediatePropagation();
+ event.preventDefault();
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown);
+
+ return () => {
+ document.removeEventListener("keydown", handleKeyDown);
+ };
+ }, [setSelectedPoint, selectedPoint]);
+
+ // Apply distance-based scaling with min/max bounds
+ useFrame(({ clock, camera }) => {
+ const distance = camera.position.distanceTo(position);
+
+ const rawScale = distance;
+ const minScale = 2;
+ const maxScale = 10.0;
+ const screenSpaceScale = Math.max(minScale, Math.min(maxScale, rawScale));
+ let finalScale = screenSpaceScale;
+
+ if (pulsate) {
+ const t = clock.getElapsedTime();
+ const pulse = 0.8 + 0.2 * Math.sin(t * 3);
+ finalScale = screenSpaceScale * pulse;
+ }
+
+ if (meshRef.current) {
+ meshRef.current.scale.set(finalScale, finalScale, finalScale);
+ }
+
+ if (transformControlsRef.current) {
+ transformControlsRef.current.position.set(0, 0, 0);
+ groupRef.current.updateMatrixWorld();
+ transformControlsRef.current.updateMatrixWorld();
+ }
+ });
+
+ const groupRef = useRef(null);
+
+ const [tempVertexTransforms, setTempVertexTransforms] = useRecoilState(
+ tempVertexTransformsAtom(`${labelId}-${segmentIndex}-${pointIndex}`)
+ );
+
+ useEffect(() => {
+ return () => {
+ setTempVertexTransforms(null);
+ };
+ }, []);
+
+ return (
+
+
+ {
+ setIsHovered(true);
+ setHoveredPolylineInfo({
+ labelId,
+ segmentIndex,
+ pointIndex,
+ });
+ }}
+ onPointerOut={() => {
+ setIsHovered(false);
+ setHoveredPolylineInfo(null);
+ }}
+ onClick={handlePointClick}
+ >
+
+
+
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/SegmentPolylineRenderer.tsx b/app/packages/looker-3d/src/annotation/SegmentPolylineRenderer.tsx
new file mode 100644
index 00000000000..38eb9934d6b
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/SegmentPolylineRenderer.tsx
@@ -0,0 +1,573 @@
+import { objectId } from "@fiftyone/utilities";
+import { Line as LineDrei } from "@react-three/drei";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
+import * as THREE from "three";
+import { SCENE_BOUNDS_EXPANSION_FACTOR, SNAP_TOLERANCE } from "../constants";
+import { useFo3dContext } from "../fo3d/context";
+import { useEmptyCanvasInteraction } from "../hooks/use-empty-canvas-interaction";
+import {
+ annotationPlaneAtom,
+ isSegmentingPointerDownAtom,
+ isSnapToAnnotationPlaneAtom,
+ polylineEffectivePointsAtom,
+ polylinePointTransformsAtom,
+ segmentPolylineStateAtom,
+ selectedLabelForAnnotationAtom,
+ sharedCursorPositionAtom,
+ snapCloseAutomaticallyAtom,
+ tempPolylinesAtom,
+} from "../state";
+import { expandBoundingBox, getPlaneFromPositionAndQuaternion } from "../utils";
+import { PolylinePointMarker } from "./PolylinePointMarker";
+import type { PolylinePointTransform, TempPolyline } from "./types";
+import { shouldClosePolylineLoop } from "./utils/polyline-utils";
+
+interface SegmentPolylineRendererProps {
+ ignoreEffects?: boolean;
+ color?: string;
+ lineWidth?: number;
+ rubberBandColor?: string;
+ rubberBandLineWidth?: number;
+}
+
+export const SegmentPolylineRenderer = ({
+ ignoreEffects = false,
+ color = "#00ff00",
+ lineWidth = 3,
+ rubberBandColor = "#ff0000",
+ rubberBandLineWidth = 2,
+}: SegmentPolylineRendererProps) => {
+ const [segmentState, setSegmentState] = useRecoilState(
+ segmentPolylineStateAtom
+ );
+ const [tempPolylines, setTempPolylines] = useRecoilState(tempPolylinesAtom);
+ const setPolylinePointTransforms = useSetRecoilState(
+ polylinePointTransformsAtom
+ );
+ const selectedLabelForAnnotation = useRecoilValue(
+ selectedLabelForAnnotationAtom
+ );
+
+ const [tempLabelId, setTempLabelId] = useState(objectId());
+
+ useEffect(() => {
+ if (ignoreEffects) return;
+
+ const newObjectId = objectId();
+ setTempLabelId(newObjectId);
+ }, [segmentState.isActive]);
+
+ const polylineEffectivePoints = useRecoilValue(
+ polylineEffectivePointsAtom(selectedLabelForAnnotation?._id || tempLabelId)
+ );
+ const setIsActivelySegmenting = useSetRecoilState(
+ isSegmentingPointerDownAtom
+ );
+ const setSharedCursorPosition = useSetRecoilState(sharedCursorPositionAtom);
+ const annotationPlane = useRecoilValue(annotationPlaneAtom);
+ const isSnapToAnnotationPlane = useRecoilValue(isSnapToAnnotationPlaneAtom);
+ const snapCloseAutomatically = useRecoilValue(snapCloseAutomaticallyAtom);
+ const { upVector, sceneBoundingBox } = useFo3dContext();
+
+ // We want annotation margin to be larger than the scene bounds
+ const expandedBoundingBox = useMemo(() => {
+ if (!sceneBoundingBox || sceneBoundingBox.isEmpty()) {
+ return null;
+ }
+ return expandBoundingBox(sceneBoundingBox, SCENE_BOUNDS_EXPANSION_FACTOR);
+ }, [sceneBoundingBox]);
+
+ // Track last click time for double-click detection
+ const lastClickTimeRef = useRef(0);
+ const DOUBLE_CLICK_THRESHOLD_MS = 200;
+ const lastAddedVertexRef = useRef<[number, number, number] | null>(null);
+
+ // Check if current position is close to first vertex for closing
+ const shouldCloseLoop = useCallback(
+ (currentPos: THREE.Vector3): boolean => {
+ return shouldClosePolylineLoop(
+ segmentState.vertices,
+ [currentPos.x, currentPos.y, currentPos.z],
+ SNAP_TOLERANCE
+ );
+ },
+ [segmentState.vertices]
+ );
+
+ const handleClick = useCallback(
+ (worldPos: THREE.Vector3) => {
+ setIsActivelySegmenting(false);
+
+ if (!segmentState.isActive) return;
+
+ const finalPos = worldPos;
+
+ const currentTime = Date.now();
+ const isDoubleClick =
+ currentTime - lastClickTimeRef.current < DOUBLE_CLICK_THRESHOLD_MS;
+
+ lastClickTimeRef.current = currentTime;
+
+ // Check for double-click behavior
+ if (isDoubleClick && lastAddedVertexRef.current) {
+ // Remove the last added vertex from the previous click
+ setSegmentState((prev) => ({
+ ...prev,
+ vertices: prev.vertices.slice(0, -1),
+ }));
+
+ // Get the vertices without the duplicate
+ const verticesWithoutDuplicate = segmentState.vertices.slice(0, -1);
+
+ if (snapCloseAutomatically) {
+ // Close the polyline automatically
+ const tempPolyline: TempPolyline = {
+ id: `temp-polyline-${Date.now()}`,
+ vertices: verticesWithoutDuplicate,
+ isClosed: true,
+ color,
+ lineWidth,
+ };
+
+ setTempPolylines((prev) => [...prev, tempPolyline]);
+
+ setSegmentState({
+ isActive: true,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+ } else {
+ // End the segment at the current position without closing
+ const tempPolyline: TempPolyline = {
+ id: `temp-polyline-${Date.now()}`,
+ vertices: [
+ ...verticesWithoutDuplicate,
+ [finalPos.x, finalPos.y, finalPos.z],
+ ],
+ isClosed: false,
+ color,
+ lineWidth,
+ };
+
+ setTempPolylines((prev) => [...prev, tempPolyline]);
+
+ setSegmentState({
+ isActive: true,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+ }
+
+ lastAddedVertexRef.current = null;
+ return;
+ }
+
+ // Check if we should close the loop by clicking near the first vertex
+ if (shouldCloseLoop(finalPos)) {
+ const tempPolyline: TempPolyline = {
+ id: `temp-polyline-${Date.now()}`,
+ vertices: segmentState.vertices,
+ isClosed: true,
+ color,
+ lineWidth,
+ };
+
+ setTempPolylines((prev) => [...prev, tempPolyline]);
+
+ setSegmentState({
+ isActive: true,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+
+ lastAddedVertexRef.current = null;
+ return;
+ }
+
+ // Add new vertex for single click
+ const newVertex: [number, number, number] = [
+ finalPos.x,
+ finalPos.y,
+ finalPos.z,
+ ];
+ setSegmentState((prev) => ({
+ ...prev,
+ vertices: [...prev.vertices, newVertex],
+ }));
+
+ lastAddedVertexRef.current = newVertex;
+ },
+ [
+ segmentState,
+ shouldCloseLoop,
+ setSegmentState,
+ setTempPolylines,
+ snapCloseAutomatically,
+ ]
+ );
+
+ // Handle mouse move for rubber band effect
+ const handleMouseMove = useCallback(
+ (worldPos: THREE.Vector3) => {
+ const finalPos = worldPos;
+
+ // Constrain position to expanded scene bounds
+ if (expandedBoundingBox) {
+ finalPos.clamp(expandedBoundingBox.min, expandedBoundingBox.max);
+ }
+
+ setSegmentState((prev) => ({
+ ...prev,
+ currentMousePosition: [finalPos.x, finalPos.y, finalPos.z],
+ }));
+
+ setSharedCursorPosition([finalPos.x, finalPos.y, finalPos.z]);
+ },
+ [expandedBoundingBox]
+ );
+
+ // Calculate the annotation plane for raycasting
+ const raycastPlane = useMemo(() => {
+ if (annotationPlane.enabled || isSnapToAnnotationPlane) {
+ const plane = getPlaneFromPositionAndQuaternion(
+ annotationPlane.position,
+ annotationPlane.quaternion
+ );
+ // Note: createPlane negates the constant, so we negate it here to cancel that out
+ return {
+ normal: plane.normal,
+ constant: -plane.constant,
+ };
+ }
+ return {
+ normal: upVector || new THREE.Vector3(0, 0, 1),
+ constant: 0,
+ };
+ }, [annotationPlane, isSnapToAnnotationPlane, upVector]);
+
+ useEmptyCanvasInteraction({
+ onPointerUp: segmentState.isActive ? handleClick : undefined,
+ onPointerDown: segmentState.isActive
+ ? () => setIsActivelySegmenting(true)
+ : undefined,
+ onPointerMove: handleMouseMove,
+ planeNormal: raycastPlane.normal,
+ planeConstant: raycastPlane.constant,
+ });
+
+ useEffect(() => {
+ if (ignoreEffects) return;
+ if (segmentState.isActive) {
+ document.body.style.cursor = "crosshair";
+ return () => {
+ document.body.style.cursor = "default";
+ };
+ }
+ }, [segmentState.isActive]);
+
+ useEffect(() => {
+ if (ignoreEffects) return;
+
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (event.key === "Escape" && segmentState.isActive) {
+ setSegmentState({
+ isActive: false,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+
+ setIsActivelySegmenting(false);
+
+ event.stopImmediatePropagation();
+ event.preventDefault();
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown);
+ return () => document.removeEventListener("keydown", handleKeyDown);
+ }, [segmentState.isActive, setSegmentState]);
+
+ // Render completed segments
+ const completedSegments = useMemo(() => {
+ if (segmentState.vertices.length < 2) return null;
+
+ const segments = [];
+ for (let i = 0; i < segmentState.vertices.length - 1; i++) {
+ segments.push(
+
+ );
+ }
+
+ // If closed, add line from last vertex to first
+ if (segmentState.isClosed && segmentState.vertices.length > 2) {
+ segments.push(
+
+ );
+ }
+
+ return segments;
+ }, [segmentState.vertices, segmentState.isClosed, color, lineWidth]);
+
+ // Temporary polylines
+ const tempPolylineElements = useMemo(() => {
+ return tempPolylines.map((polyline) => {
+ const segments = [];
+
+ // Line segments between consecutive vertices
+ for (let i = 0; i < polyline.vertices.length - 1; i++) {
+ segments.push(
+
+ );
+ }
+
+ // If closed, add line from last vertex to first
+ if (polyline.isClosed && polyline.vertices.length > 2) {
+ segments.push(
+
+ );
+ }
+
+ return segments;
+ });
+ }, [tempPolylines]);
+
+ // Rubber band from last vertex to current mouse position
+ const rubberBand = useMemo(() => {
+ if (
+ !segmentState.isActive ||
+ segmentState.vertices.length === 0 ||
+ !segmentState.currentMousePosition
+ ) {
+ return null;
+ }
+
+ const lastVertex = segmentState.vertices[segmentState.vertices.length - 1];
+
+ return (
+
+ );
+ }, [
+ segmentState.isActive,
+ segmentState.vertices,
+ segmentState.currentMousePosition,
+ rubberBandColor,
+ rubberBandLineWidth,
+ ]);
+
+ const vertexMarkers = useMemo(() => {
+ const markers = [];
+
+ // Current segment vertices (while actively segmenting)
+ if (segmentState.vertices.length > 0) {
+ segmentState.vertices.forEach((vertex, index) => {
+ markers.push(
+
+ );
+ });
+ }
+
+ // Completed polylines vertices (for editing)
+ tempPolylines.forEach((polyline, polylineIndex) => {
+ polyline.vertices.forEach((vertex, vertexIndex) => {
+ markers.push(
+ {
+ // Update the vertex position in tempPolylines
+ setTempPolylines((prev) =>
+ prev.map((p) =>
+ p.id === polyline.id
+ ? {
+ ...p,
+ vertices: p.vertices.map((v, i) =>
+ i === vertexIndex
+ ? ([
+ newPosition.x,
+ newPosition.y,
+ newPosition.z,
+ ] as [number, number, number])
+ : v
+ ),
+ }
+ : p
+ )
+ );
+ }}
+ />
+ );
+ });
+ });
+
+ return markers.length > 0 ? markers : null;
+ }, [segmentState.vertices, tempPolylines, color, setTempPolylines]);
+
+ // Sync with polylinePointTransformsAtom
+ useEffect(() => {
+ if (ignoreEffects) return;
+
+ if (tempPolylines.length === 0) return;
+ const labelId = selectedLabelForAnnotation?._id || tempLabelId;
+
+ const newPolyline: PolylinePointTransform[] = [];
+
+ // Find the highest existing segment index so that we can start from the next one
+ const highestSegmentIndex =
+ polylineEffectivePoints.length > 0
+ ? polylineEffectivePoints.length - 1
+ : -1;
+
+ const segmentIndexOffset = highestSegmentIndex + 1;
+
+ // Each vertex gets a segmentIndex (which segment it belongs to) and pointIndex (0 or 1 for start/end of segment)
+ for (let i = 0; i < tempPolylines[0].vertices.length; i++) {
+ const vertex = tempPolylines[0].vertices[i];
+
+ // For each vertex, it can be either the start or end of a segment
+ // If it's the last vertex and the polyline is closed, it connects back to the first vertex
+ // Otherwise, it connects to the next vertex
+ if (i < tempPolylines[0].vertices.length - 1) {
+ // This vertex is the start of a segment that goes to the next vertex
+ newPolyline.push({
+ segmentIndex: segmentIndexOffset + i,
+ pointIndex: 0,
+ position: vertex,
+ });
+ // The next vertex is the end of this segment
+ newPolyline.push({
+ segmentIndex: segmentIndexOffset + i,
+ pointIndex: 1,
+ position: tempPolylines[0].vertices[i + 1],
+ });
+ } else if (
+ tempPolylines[0].isClosed &&
+ tempPolylines[0].vertices.length > 2
+ ) {
+ // Last vertex connects back to first vertex if closed
+ newPolyline.push({
+ segmentIndex: segmentIndexOffset + i,
+ pointIndex: 0,
+ position: vertex,
+ });
+ newPolyline.push({
+ segmentIndex: segmentIndexOffset + i,
+ pointIndex: 1,
+ position: tempPolylines[0].vertices[0],
+ });
+ }
+ }
+
+ setPolylinePointTransforms((prev) => {
+ const newTransforms = [...(prev[labelId] || []), ...newPolyline];
+
+ // Remove duplicates
+ const uniqueTransforms = newTransforms.filter(
+ (transform, index, self) =>
+ index ===
+ self.findIndex(
+ (t) =>
+ t.segmentIndex === transform.segmentIndex &&
+ t.pointIndex === transform.pointIndex
+ )
+ );
+
+ return {
+ ...prev,
+ [labelId]: uniqueTransforms,
+ };
+ });
+
+ // Get out of segmenting mode
+ setSegmentState({
+ isActive: false,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+
+ setIsActivelySegmenting(false);
+
+ setTempPolylines([]);
+ }, [tempPolylines, polylineEffectivePoints, tempLabelId]);
+
+ if (
+ !segmentState.isActive &&
+ segmentState.vertices.length === 0 &&
+ tempPolylines.length === 0
+ ) {
+ return null;
+ }
+
+ return (
+
+ {tempPolylineElements}
+ {completedSegments}
+ {rubberBand}
+ {vertexMarkers}
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/annotation-toolbar/AnnotationToolbar.tsx b/app/packages/looker-3d/src/annotation/annotation-toolbar/AnnotationToolbar.tsx
new file mode 100644
index 00000000000..969186af83f
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/annotation-toolbar/AnnotationToolbar.tsx
@@ -0,0 +1,254 @@
+import { DragIndicator } from "@mui/icons-material";
+import { Box, IconButton, Tooltip, Typography, styled } from "@mui/material";
+import React, { useCallback, useRef, useState } from "react";
+import { useRecoilState } from "recoil";
+import { annotationToolbarPositionAtom } from "../../state";
+import type { AnnotationToolbarProps } from "../types";
+import { useAnnotationActions } from "./useAnnotationActions";
+
+const ToolbarContainer = styled(Box)<{
+ topposition: number;
+ isdragging?: string;
+}>(({ theme, topposition, isdragging }) => {
+ const isDraggingBool = isdragging === "true";
+
+ return {
+ position: "absolute",
+ top: `${topposition}%`,
+ left: "8px",
+ transform: "translateY(-50%)",
+ display: "flex",
+ flexDirection: "column",
+ backgroundColor: theme.palette.background.paper,
+ borderRadius: "6px",
+ boxShadow: isDraggingBool
+ ? "0 4px 16px rgba(0, 0, 0, 0.2)"
+ : "0 2px 8px rgba(0, 0, 0, 0.12)",
+ border: `1px solid ${theme.palette.divider}`,
+ zIndex: 1000,
+ minWidth: "36px",
+ opacity: isDraggingBool ? 0.95 : 0.75,
+ userSelect: "none",
+ transition: isDraggingBool
+ ? "none"
+ : "opacity 0.2s ease, box-shadow 0.2s ease",
+
+ "&:hover": {
+ opacity: 0.95,
+ },
+ };
+});
+
+const DraggableHeader = styled(Box)<{ isdragging?: string }>(
+ ({ theme, isdragging }) => {
+ const isDraggingBool = isdragging === "true";
+
+ return {
+ width: "100%",
+ height: "12px",
+ borderRadius: "6px 6px 0 0",
+ cursor: isDraggingBool ? "grabbing" : "grab",
+ opacity: 0,
+ transition: "opacity 0.2s ease",
+ margin: "2px 2px 0 2px",
+ position: "relative",
+ display: "flex",
+ alignItems: "center",
+ justifyContent: "center",
+
+ ".MuiBox-root:hover &": {
+ opacity: 0.8,
+ },
+
+ // rotate drag handler by 90 degrees
+ "& > *": {
+ transform: "rotate(90deg)",
+ },
+ };
+ }
+);
+
+const ToolbarContent = styled(Box)({
+ display: "flex",
+ flexDirection: "column",
+ gap: "8px",
+ padding: "8px",
+});
+
+const ActionGroup = styled(Box)({
+ display: "flex",
+ flexDirection: "column",
+ gap: "6px",
+ alignItems: "center",
+});
+
+const GroupLabel = styled(Typography)(({ theme }) => ({
+ fontSize: "9px",
+ fontWeight: 600,
+ color: theme.palette.text.secondary,
+ textAlign: "center",
+ marginBottom: "2px",
+ textTransform: "uppercase",
+ letterSpacing: "0.3px",
+}));
+
+const ActionButton = styled(IconButton)<{
+ isactive?: string;
+ isdisabled?: string;
+}>(({ theme, isactive, isdisabled }) => {
+ const isActiveBool = isactive === "true";
+ const isDisabledBool = isdisabled === "true";
+
+ return {
+ width: "28px",
+ height: "28px",
+ cursor: isDisabledBool ? "not-allowed" : "pointer",
+ backgroundColor: isActiveBool ? theme.palette.primary.main : "transparent",
+ color: isActiveBool
+ ? theme.palette.primary.contrastText
+ : isDisabledBool
+ ? theme.palette.text.disabled
+ : theme.palette.text.primary,
+ display: "flex",
+ alignItems: "center",
+ justifyContent: "center",
+ "&:hover": {
+ backgroundColor: isActiveBool
+ ? theme.palette.primary.dark
+ : theme.palette.action.hover,
+ },
+ "&:disabled": {
+ color: theme.palette.text.disabled,
+ backgroundColor: "transparent",
+ },
+ "& .MuiSvgIcon-root": {
+ fontSize: "18px",
+ },
+ };
+});
+
+export const AnnotationToolbar = ({ className }: AnnotationToolbarProps) => {
+ const { actions } = useAnnotationActions();
+ const [position, setPosition] = useRecoilState(annotationToolbarPositionAtom);
+ const [isDragging, setIsDragging] = useState(false);
+ const [dragStart, setDragStart] = useState({ y: 0, position: 0 });
+ const containerRef = useRef(null);
+
+ const handleActionClick = useCallback((action: () => void) => {
+ action();
+ }, []);
+
+ const handleHeaderMouseDown = useCallback(
+ (e: React.MouseEvent) => {
+ e.preventDefault();
+ e.stopPropagation();
+ setIsDragging(true);
+ setDragStart({
+ y: e.clientY,
+ position: position,
+ });
+ },
+ [position]
+ );
+
+ const handleMouseMove = useCallback(
+ (e: MouseEvent) => {
+ if (!isDragging) return;
+
+ const deltaY = e.clientY - dragStart.y;
+ const containerHeight =
+ containerRef.current?.parentElement?.clientHeight || window.innerHeight;
+ const deltaPercentage = (deltaY / containerHeight) * 100;
+
+ const newPosition = Math.max(
+ 5,
+ Math.min(95, dragStart.position + deltaPercentage)
+ );
+ setPosition(newPosition);
+ },
+ [isDragging, dragStart, setPosition]
+ );
+
+ const handleMouseUp = useCallback(() => {
+ setIsDragging(false);
+ }, []);
+
+ React.useEffect(() => {
+ if (isDragging) {
+ document.addEventListener("mousemove", handleMouseMove);
+ document.addEventListener("mouseup", handleMouseUp);
+
+ return () => {
+ document.removeEventListener("mousemove", handleMouseMove);
+ document.removeEventListener("mouseup", handleMouseUp);
+ };
+ }
+ }, [isDragging, handleMouseMove, handleMouseUp]);
+
+ return (
+
+
+
+
+
+ {actions
+ .filter((group) => !group.isHidden)
+ .map((group) => (
+
+ {group.label && {group.label}}
+ {group.actions
+ .filter((action) => action.isVisible !== false)
+ .map((action) => {
+ // Render custom component if provided
+ if (action.customComponent) {
+ return {action.customComponent};
+ }
+
+ // Render regular action button if no custom component is provided
+ return (
+
+
+ {action.tooltip || action.label}
+
+ {action.shortcut && (
+
+ {action.shortcut}
+
+ )}
+
+ }
+ placement="left"
+ arrow
+ disableHoverListener={false}
+ >
+
+ !action.isDisabled &&
+ handleActionClick(action.onClick)
+ }
+ isdisabled={String(action.isDisabled)}
+ isactive={String(action.isActive)}
+ size="small"
+ >
+ {action.icon}
+
+
+ );
+ })}
+
+ ))}
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/annotation-toolbar/CoordinateInputs.tsx b/app/packages/looker-3d/src/annotation/annotation-toolbar/CoordinateInputs.tsx
new file mode 100644
index 00000000000..d8d98a87f21
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/annotation-toolbar/CoordinateInputs.tsx
@@ -0,0 +1,404 @@
+import { Box, TextField } from "@mui/material";
+import { useCallback, useEffect, useMemo, useState } from "react";
+import { useRecoilState, useRecoilValue } from "recoil";
+import type { PolyLineProps } from "../../labels/polyline";
+import {
+ annotationPlaneAtom,
+ polylinePointTransformsAtom,
+ selectedLabelForAnnotationAtom,
+ selectedPolylineVertexAtom,
+} from "../../state";
+import {
+ getVertexPosition,
+ updateDuplicateVertices,
+ applyTransformsToPolyline,
+} from "../utils/polyline-utils";
+
+interface CoordinateInputsProps {
+ className?: string;
+ hideTranslate?: boolean;
+ hideQuaternion?: boolean;
+}
+
+interface CoordinateFieldProps {
+ label: string;
+ value: string;
+ onChange: (value: string) => void;
+ onFocus: () => void;
+ onBlur: () => void;
+}
+
+const CoordinateField = ({
+ label,
+ value,
+ onChange,
+ onFocus,
+ onBlur,
+}: CoordinateFieldProps) => {
+ return (
+ onChange(e.target.value)}
+ type="number"
+ inputProps={{
+ step: "0.1",
+ style: { fontSize: "11px", padding: "4px 6px" },
+ }}
+ sx={{
+ "& .MuiInputLabel-root": { fontSize: "10px" },
+ "& .MuiOutlinedInput-root": { height: "24px" },
+ "& .MuiOutlinedInput-input": { padding: "4px 6px" },
+ }}
+ />
+ );
+};
+
+export const PlaneCoordinateInputs = ({
+ className,
+ hideTranslate = false,
+ hideQuaternion = true,
+}: CoordinateInputsProps) => {
+ const [annotationPlane, setAnnotationPlane] =
+ useRecoilState(annotationPlaneAtom);
+ const [x, setX] = useState("0");
+ const [y, setY] = useState("0");
+ const [z, setZ] = useState("0");
+ const [qx, setQx] = useState("0");
+ const [qy, setQy] = useState("0");
+ const [qz, setQz] = useState("0");
+ const [qw, setQw] = useState("1");
+
+ const [isEditing, setIsEditing] = useState(false);
+
+ useEffect(() => {
+ if (annotationPlane && !isEditing) {
+ setX(annotationPlane.position[0].toFixed(3));
+ setY(annotationPlane.position[1].toFixed(3));
+ setZ(annotationPlane.position[2].toFixed(3));
+ setQx(annotationPlane.quaternion[0].toFixed(3));
+ setQy(annotationPlane.quaternion[1].toFixed(3));
+ setQz(annotationPlane.quaternion[2].toFixed(3));
+ setQw(annotationPlane.quaternion[3].toFixed(3));
+ }
+ }, [annotationPlane, isEditing]);
+
+ const handlePositionChange = useCallback(
+ (axis: "x" | "y" | "z", value: string) => {
+ // Always update local state to allow blank values while editing
+ if (axis === "x") setX(value);
+ else if (axis === "y") setY(value);
+ else if (axis === "z") setZ(value);
+
+ const numValue = parseFloat(value);
+ if (isNaN(numValue)) return;
+
+ const newPosition: [number, number, number] = [
+ ...annotationPlane.position,
+ ];
+ if (axis === "x") newPosition[0] = numValue;
+ else if (axis === "y") newPosition[1] = numValue;
+ else if (axis === "z") newPosition[2] = numValue;
+
+ setAnnotationPlane((prev) => ({
+ ...prev,
+ position: newPosition,
+ }));
+ },
+ [annotationPlane.position]
+ );
+
+ const handleQuaternionChange = useCallback(
+ (axis: "x" | "y" | "z" | "w", value: string) => {
+ // Always update local state to allow blank values while editing
+ if (axis === "x") setQx(value);
+ else if (axis === "y") setQy(value);
+ else if (axis === "z") setQz(value);
+ else if (axis === "w") setQw(value);
+
+ const numValue = parseFloat(value);
+ if (isNaN(numValue)) return;
+
+ const newQuaternion: [number, number, number, number] = [
+ ...annotationPlane.quaternion,
+ ];
+ if (axis === "x") newQuaternion[0] = numValue;
+ else if (axis === "y") newQuaternion[1] = numValue;
+ else if (axis === "z") newQuaternion[2] = numValue;
+ else if (axis === "w") newQuaternion[3] = numValue;
+
+ setAnnotationPlane((prev) => ({
+ ...prev,
+ quaternion: newQuaternion,
+ }));
+ },
+ [annotationPlane.quaternion]
+ );
+
+ if (!annotationPlane.enabled) {
+ return null;
+ }
+
+ return (
+
+
+ {!hideTranslate && (
+ <>
+ {annotationPlane.showX && (
+ handlePositionChange("x", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ )}
+ {annotationPlane.showY && (
+ handlePositionChange("y", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ )}
+ {annotationPlane.showZ && (
+ handlePositionChange("z", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ )}
+
+ )}
+ {!hideQuaternion && (
+ <>
+ handleQuaternionChange("x", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ handleQuaternionChange("y", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ handleQuaternionChange("z", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ handleQuaternionChange("w", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+
+ )}
+
+
+ );
+};
+
+export const VertexCoordinateInputs = ({
+ className,
+ hideTranslate = false,
+}: CoordinateInputsProps) => {
+ const [selectedPoint, setSelectedPoint] = useRecoilState(
+ selectedPolylineVertexAtom
+ );
+ const selectedLabel = useRecoilValue(selectedLabelForAnnotationAtom);
+ const [polylinePointTransforms, setPolylinePointTransforms] = useRecoilState(
+ polylinePointTransformsAtom
+ );
+ const selectedPointPosition = useMemo(() => {
+ if (!selectedPoint || !selectedLabel) return null;
+
+ const polylineLabel = selectedLabel as unknown as PolyLineProps;
+ const transforms = polylinePointTransforms[selectedPoint.labelId] || [];
+
+ return getVertexPosition(
+ selectedPoint,
+ polylineLabel.points3d || [],
+ transforms
+ );
+ }, [selectedPoint, selectedLabel, polylinePointTransforms]);
+ const [x, setX] = useState("0");
+ const [y, setY] = useState("0");
+ const [z, setZ] = useState("0");
+
+ const [isEditing, setIsEditing] = useState(false);
+
+ useEffect(() => {
+ if (selectedPointPosition && !isEditing) {
+ setX(selectedPointPosition[0].toFixed(3));
+ setY(selectedPointPosition[1].toFixed(3));
+ setZ(selectedPointPosition[2].toFixed(3));
+ }
+ }, [selectedPointPosition, isEditing]);
+
+ const handleCoordinateChange = useCallback(
+ (axis: "x" | "y" | "z", value: string) => {
+ // Always update local state to allow blank values while editing
+ if (axis === "x") setX(value);
+ else if (axis === "y") setY(value);
+ else if (axis === "z") setZ(value);
+
+ const numValue = parseFloat(value);
+ if (isNaN(numValue)) return;
+
+ if (selectedPoint && selectedLabel) {
+ const { segmentIndex, pointIndex, labelId } = selectedPoint;
+
+ setPolylinePointTransforms((prev) => {
+ const currentTransforms = prev[labelId] || [];
+ const polylineLabel = selectedLabel as unknown as PolyLineProps;
+ const originalPoints = polylineLabel.points3d || [];
+
+ // Get current position (either from existing transform or original point)
+ let currentPosition: [number, number, number];
+ const existingTransformIndex = currentTransforms.findIndex(
+ (transform) =>
+ transform.segmentIndex === segmentIndex &&
+ transform.pointIndex === pointIndex
+ );
+
+ if (existingTransformIndex >= 0) {
+ currentPosition =
+ currentTransforms[existingTransformIndex].position;
+ } else if (selectedPointPosition) {
+ currentPosition = selectedPointPosition;
+ } else {
+ // Fallback to original position from label
+ if (
+ segmentIndex < originalPoints.length &&
+ pointIndex < originalPoints[segmentIndex].length
+ ) {
+ currentPosition = originalPoints[segmentIndex][pointIndex] as [
+ number,
+ number,
+ number
+ ];
+ } else {
+ currentPosition = [0, 0, 0];
+ }
+ }
+
+ const newPosition: [number, number, number] = [...currentPosition];
+ if (axis === "x") newPosition[0] = numValue;
+ else if (axis === "y") newPosition[1] = numValue;
+ else if (axis === "z") newPosition[2] = numValue;
+
+ // Compute current effective points to find all shared vertices
+ const effectivePoints3d = applyTransformsToPolyline(
+ originalPoints,
+ currentTransforms
+ );
+
+ // Use updateDuplicateVertices to handle shared vertices
+ const newTransforms = updateDuplicateVertices(
+ currentPosition,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ return { ...prev, [labelId]: newTransforms };
+ });
+
+ // Note: this is a hack because transform controls
+ // is updating in a buggy way when the point is moved
+ const prevSelectedPoint = selectedPoint;
+ setSelectedPoint(null);
+ setTimeout(() => {
+ setSelectedPoint(prevSelectedPoint);
+ }, 0);
+ }
+ },
+ [
+ selectedPoint,
+ selectedLabel,
+ selectedPointPosition,
+ setPolylinePointTransforms,
+ ]
+ );
+
+ if (!selectedPoint) {
+ return null;
+ }
+
+ return (
+
+
+ {!hideTranslate && (
+ <>
+ handleCoordinateChange("x", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ handleCoordinateChange("y", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+ handleCoordinateChange("z", value)}
+ onFocus={() => setIsEditing(true)}
+ onBlur={() => setIsEditing(false)}
+ />
+
+ )}
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/annotation/annotation-toolbar/index.ts b/app/packages/looker-3d/src/annotation/annotation-toolbar/index.ts
new file mode 100644
index 00000000000..1f5f9e7b4fd
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/annotation-toolbar/index.ts
@@ -0,0 +1,7 @@
+export { AnnotationToolbar } from "./AnnotationToolbar";
+export type {
+ AnnotationAction,
+ AnnotationActionGroup,
+ AnnotationToolbarProps,
+} from "../types";
+export { useAnnotationActions } from "./useAnnotationActions";
diff --git a/app/packages/looker-3d/src/annotation/annotation-toolbar/useAnnotationActions.tsx b/app/packages/looker-3d/src/annotation/annotation-toolbar/useAnnotationActions.tsx
new file mode 100644
index 00000000000..becfd061b94
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/annotation-toolbar/useAnnotationActions.tsx
@@ -0,0 +1,475 @@
+import {
+ Add,
+ Close,
+ Delete,
+ Edit,
+ OpenWith,
+ RotateRight,
+ Straighten,
+} from "@mui/icons-material";
+import RectangleIcon from "@mui/icons-material/Rectangle";
+import RestartAltIcon from "@mui/icons-material/RestartAlt";
+import { Typography } from "@mui/material";
+import { useCallback, useEffect, useMemo } from "react";
+import { useRecoilState, useSetRecoilState } from "recoil";
+import * as THREE from "three";
+import MagnetIcon from "../../assets/icons/magnet.svg?react";
+import { useFo3dContext } from "../../fo3d/context";
+import {
+ annotationPlaneAtom,
+ currentArchetypeSelectedForTransformAtom,
+ editSegmentsModeAtom,
+ isSnapToAnnotationPlaneAtom,
+ polylinePointTransformsAtom,
+ segmentPolylineStateAtom,
+ selectedLabelForAnnotationAtom,
+ selectedPolylineVertexAtom,
+ snapCloseAutomaticallyAtom,
+ tempPolylinesAtom,
+ transformModeAtom,
+ transformSpaceAtom,
+} from "../../state";
+import { getGridQuaternionFromUpVector } from "../../utils";
+import type {
+ AnnotationAction,
+ AnnotationActionGroup,
+ TransformMode,
+ TransformSpace,
+} from "../types";
+import { deletePolylinePoint } from "../utils/polyline-delete";
+import {
+ PlaneCoordinateInputs,
+ VertexCoordinateInputs,
+} from "./CoordinateInputs";
+
+export const useAnnotationActions = () => {
+ const [selectedLabelForAnnotation, setSelectedLabelForAnnotation] =
+ useRecoilState(selectedLabelForAnnotationAtom);
+ const [
+ currentArchetypeSelectedForTransform,
+ setCurrentArchetypeSelectedForTransform,
+ ] = useRecoilState(currentArchetypeSelectedForTransformAtom);
+ const [transformMode, setTransformMode] = useRecoilState(transformModeAtom);
+ const [transformSpace, setTransformSpace] =
+ useRecoilState(transformSpaceAtom);
+ const [selectedPoint, setSelectedPoint] = useRecoilState(
+ selectedPolylineVertexAtom
+ );
+ const [segmentPolylineState, setSegmentPolylineState] = useRecoilState(
+ segmentPolylineStateAtom
+ );
+ const [isSnapToAnnotationPlane, setIsSnapToAnnotationPlane] = useRecoilState(
+ isSnapToAnnotationPlaneAtom
+ );
+ const [snapCloseAutomatically, setSnapCloseAutomatically] = useRecoilState(
+ snapCloseAutomaticallyAtom
+ );
+ const [editSegmentsMode, setEditSegmentsMode] =
+ useRecoilState(editSegmentsModeAtom);
+ const [tempPolylines, setTempPolylines] = useRecoilState(tempPolylinesAtom);
+ const setPolylinePointTransforms = useSetRecoilState(
+ polylinePointTransformsAtom
+ );
+ const [annotationPlane, setAnnotationPlane] =
+ useRecoilState(annotationPlaneAtom);
+ const { sceneBoundingBox, upVector } = useFo3dContext();
+
+ const handleTransformModeChange = useCallback(
+ (mode: TransformMode) => {
+ setTransformMode(mode);
+ },
+ [setTransformMode]
+ );
+
+ const handleTransformSpaceChange = useCallback(
+ (space: TransformSpace) => {
+ setTransformSpace(space);
+ },
+ [setTransformSpace]
+ );
+
+ const handleStartSegmentPolyline = useCallback(() => {
+ setSegmentPolylineState({
+ isActive: true,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+ }, [setSegmentPolylineState]);
+
+ const handleCancelSegmentPolyline = useCallback(() => {
+ setSegmentPolylineState({
+ isActive: false,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+ }, [setSegmentPolylineState]);
+
+ const handleClearTempPolylines = useCallback(() => {
+ setTempPolylines([]);
+ }, [setTempPolylines]);
+
+ const handleDeleteSelectedPoint = useCallback(() => {
+ if (!selectedPoint) return;
+
+ const { labelId, segmentIndex, pointIndex } = selectedPoint;
+
+ setPolylinePointTransforms((prev) => {
+ const currentTransforms = prev[labelId] || [];
+ const result = deletePolylinePoint(
+ currentTransforms,
+ segmentIndex,
+ pointIndex
+ );
+
+ return {
+ ...prev,
+ [labelId]: result.newTransforms,
+ };
+ });
+
+ setSelectedPoint(null);
+ setCurrentArchetypeSelectedForTransform(null);
+ }, [
+ selectedPoint,
+ setPolylinePointTransforms,
+ setSelectedPoint,
+ setCurrentArchetypeSelectedForTransform,
+ ]);
+
+ const handleDeleteEntireTransform = useCallback(() => {
+ if (
+ !selectedLabelForAnnotation ||
+ selectedLabelForAnnotation._cls !== "Polyline"
+ )
+ return;
+
+ const labelId = selectedLabelForAnnotation._id;
+
+ setPolylinePointTransforms((prev) => {
+ const newTransforms = { ...prev };
+ delete newTransforms[labelId];
+ return newTransforms;
+ });
+
+ setSelectedLabelForAnnotation(null);
+ setCurrentArchetypeSelectedForTransform(null);
+ setSelectedPoint(null);
+ }, [
+ selectedLabelForAnnotation,
+ setPolylinePointTransforms,
+ setSelectedLabelForAnnotation,
+ setCurrentArchetypeSelectedForTransform,
+ setSelectedPoint,
+ ]);
+
+ const handleContextualDelete = useCallback(() => {
+ if (selectedPoint) {
+ handleDeleteSelectedPoint();
+ } else if (
+ selectedLabelForAnnotation &&
+ selectedLabelForAnnotation._cls === "Polyline"
+ ) {
+ handleDeleteEntireTransform();
+ }
+ }, [
+ selectedPoint,
+ handleDeleteSelectedPoint,
+ selectedLabelForAnnotation,
+ handleDeleteEntireTransform,
+ ]);
+
+ useEffect(() => {
+ const handler = (event: KeyboardEvent) => {
+ if (event.defaultPrevented) {
+ return;
+ }
+
+ const target = event.target as HTMLElement | null;
+ if (
+ target &&
+ (target.isContentEditable ||
+ target.tagName === "INPUT" ||
+ target.tagName === "TEXTAREA")
+ ) {
+ return;
+ }
+
+ if (event.key === "Delete" || event.key === "Backspace") {
+ handleContextualDelete();
+ }
+ };
+
+ document.addEventListener("keydown", handler);
+
+ return () => {
+ document.removeEventListener("keydown", handler);
+ };
+ }, [handleContextualDelete]);
+
+ const handleToggleAnnotationPlane = useCallback(() => {
+ if (!annotationPlane.enabled) {
+ if (sceneBoundingBox && upVector) {
+ const center = new THREE.Vector3(...annotationPlane.position);
+
+ const quaternion = getGridQuaternionFromUpVector(
+ upVector,
+ new THREE.Vector3(0, 0, 1)
+ );
+
+ setAnnotationPlane({
+ enabled: true,
+ position: [center.x, center.y, center.z],
+ quaternion: [quaternion.x, quaternion.y, quaternion.z, quaternion.w],
+ showX: false,
+ showY: false,
+ showZ: false,
+ });
+ } else {
+ // Fallback to origin with Y-up
+ setAnnotationPlane({
+ enabled: true,
+ position: [0, 0, 0],
+ quaternion: [0, 0, 0, 1],
+ showX: false,
+ showY: true,
+ showZ: false,
+ });
+ }
+ } else {
+ setAnnotationPlane((prev) => ({ ...prev, enabled: false }));
+ setCurrentArchetypeSelectedForTransform(null);
+ }
+ }, [annotationPlane.enabled, sceneBoundingBox, upVector]);
+
+ const handleToggleEditSegmentsMode = useCallback(() => {
+ setEditSegmentsMode(!editSegmentsMode);
+ // Deactivate other modes when entering edit segments mode
+ if (!editSegmentsMode) {
+ setSegmentPolylineState((prev) => ({ ...prev, isActive: false }));
+ }
+ }, [editSegmentsMode, setEditSegmentsMode, setSegmentPolylineState]);
+
+ const actions: AnnotationActionGroup[] = useMemo(() => {
+ const baseActions: AnnotationActionGroup[] = [
+ {
+ id: "edit-actions",
+ label: "Edit",
+ isHidden: selectedLabelForAnnotation === null,
+ actions: [
+ {
+ id: "exit-edit-mode",
+ label: "Deselect",
+ icon: ,
+ shortcut: "Esc",
+ tooltip: "Exit edit mode and deselect annotation",
+ isActive: false,
+ isVisible: selectedLabelForAnnotation !== null,
+ onClick: () => setSelectedLabelForAnnotation(null),
+ },
+ ],
+ },
+ {
+ id: "polyline-actions",
+ label: "Polyline",
+ actions: [
+ {
+ id: "new-segment",
+ label: "New Segment",
+ icon: ,
+ tooltip: "Add new polyline segment",
+ isActive: segmentPolylineState.isActive,
+ onClick: segmentPolylineState.isActive
+ ? handleCancelSegmentPolyline
+ : handleStartSegmentPolyline,
+ },
+ {
+ id: "edit-segments",
+ label: "Edit Segments",
+ icon: ,
+ tooltip: "Click on a segment to add a new vertex",
+ isActive: editSegmentsMode,
+ isVisible: selectedLabelForAnnotation !== null,
+ onClick: handleToggleEditSegmentsMode,
+ },
+ {
+ id: "contextual-delete",
+ label: "Delete",
+ icon: ,
+ shortcut: "Delete",
+ tooltip: selectedPoint
+ ? "Delete selected polyline point"
+ : "Delete polyline",
+ isActive: false,
+ isVisible:
+ selectedLabelForAnnotation !== null &&
+ selectedLabelForAnnotation._cls === "Polyline",
+ onClick: handleContextualDelete,
+ },
+ {
+ id: "snap-to-annotation-plane",
+ label: "Snap to Annotation Plane",
+ icon: (
+
+ ),
+ tooltip:
+ annotationPlane.enabled && !isSnapToAnnotationPlane
+ ? "Project new vertices to the annotation plane"
+ : "Project new vertices to the annotation plane whether or not it is enabled",
+ isActive: isSnapToAnnotationPlane || annotationPlane.enabled,
+ isDisabled: annotationPlane.enabled,
+ isVisible: true,
+ onClick: () => setIsSnapToAnnotationPlane(!isSnapToAnnotationPlane),
+ },
+ {
+ id: "toggle-annotation-plane",
+ label: "Annotation Plane",
+ icon: ,
+ tooltip: "Toggle annotation plane for z-drift prevention",
+ isActive: annotationPlane.enabled,
+ onClick: handleToggleAnnotationPlane,
+ },
+ {
+ id: "snap-close-automatically",
+ label: "Snap Close Automatically",
+ icon: ,
+ tooltip:
+ "When enabled, double-click closes polylines. When disabled, double-click ends segment at click location.",
+ isActive: snapCloseAutomatically,
+ onClick: () => setSnapCloseAutomatically(!snapCloseAutomatically),
+ },
+ ],
+ },
+ {
+ id: "transform-actions",
+ label: "Transform",
+ isHidden: !currentArchetypeSelectedForTransform,
+ actions: [
+ {
+ id: "translate",
+ label: "Translate",
+ icon: ,
+ tooltip: "Translate or move object",
+ isActive:
+ currentArchetypeSelectedForTransform &&
+ transformMode === "translate",
+ isDisabled: !currentArchetypeSelectedForTransform,
+ onClick: () => handleTransformModeChange("translate"),
+ },
+ {
+ id: "rotate",
+ label: "Rotate",
+ icon: ,
+ tooltip: "Rotate object",
+ isVisible: currentArchetypeSelectedForTransform === "cuboid",
+ isActive: transformMode === "rotate",
+ onClick: () => handleTransformModeChange("rotate"),
+ },
+ {
+ id: "scale",
+ label: "Scale",
+ icon: ,
+ tooltip: "Scale object",
+ isActive: transformMode === "scale",
+ isVisible: currentArchetypeSelectedForTransform === "cuboid",
+ onClick: () => handleTransformModeChange("scale"),
+ },
+ ],
+ },
+ {
+ id: "space-actions",
+ label: "Space",
+ isHidden:
+ currentArchetypeSelectedForTransform !== "cuboid" ||
+ (transformMode !== "translate" && transformMode !== "rotate"),
+ actions: [
+ {
+ id: "world-space",
+ label: "World Space",
+ icon: W,
+ tooltip: "Transform in world space",
+ isActive: transformSpace === "world",
+ isVisible:
+ currentArchetypeSelectedForTransform === "cuboid" ||
+ currentArchetypeSelectedForTransform === "annotation-plane",
+ onClick: () => handleTransformSpaceChange("world"),
+ },
+ {
+ id: "local-space",
+ label: "Local Space",
+ icon: L,
+ tooltip: "Transform in local space",
+ isActive: transformSpace === "local",
+ isVisible:
+ currentArchetypeSelectedForTransform === "cuboid" ||
+ currentArchetypeSelectedForTransform === "annotation-plane",
+ onClick: () => handleTransformSpaceChange("local"),
+ },
+ ],
+ },
+ ];
+
+ if (
+ (selectedPoint && currentArchetypeSelectedForTransform === "point") ||
+ currentArchetypeSelectedForTransform === "annotation-plane"
+ ) {
+ const coordinateInputAction: AnnotationAction = {
+ id: "coordinate-inputs-component",
+ label: "Coordinates",
+ icon: XYZ,
+ tooltip: "Edit point coordinates",
+ isActive: false,
+ isDisabled: false,
+ isVisible: true,
+ // No-op since this is a custom component
+ onClick: () => {},
+ customComponent:
+ currentArchetypeSelectedForTransform === "annotation-plane" ? (
+
+ ) : (
+
+ ),
+ };
+
+ baseActions.push({
+ id: "coordinate-inputs",
+ actions: [coordinateInputAction],
+ });
+ }
+
+ return baseActions;
+ }, [
+ transformMode,
+ currentArchetypeSelectedForTransform,
+ handleTransformModeChange,
+ transformSpace,
+ handleContextualDelete,
+ handleTransformSpaceChange,
+ selectedLabelForAnnotation,
+ selectedPoint,
+ segmentPolylineState,
+ annotationPlane,
+ handleStartSegmentPolyline,
+ handleCancelSegmentPolyline,
+ handleToggleAnnotationPlane,
+ isSnapToAnnotationPlane,
+ snapCloseAutomatically,
+ editSegmentsMode,
+ handleToggleEditSegmentsMode,
+ ]);
+
+ return {
+ actions,
+ transformMode,
+ transformSpace,
+ };
+};
diff --git a/app/packages/looker-3d/src/annotation/types.ts b/app/packages/looker-3d/src/annotation/types.ts
new file mode 100644
index 00000000000..d00c018772f
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/types.ts
@@ -0,0 +1,115 @@
+import { ReactNode } from "react";
+
+export interface AnnotationAction {
+ id: string;
+ label: string;
+ icon: ReactNode;
+ shortcut?: string;
+ tooltip?: string;
+ isActive?: boolean;
+ isDisabled?: boolean;
+ isVisible?: boolean;
+ onClick: () => void;
+ customComponent?: ReactNode;
+}
+
+export interface AnnotationActionGroup {
+ id: string;
+ label?: string;
+ isHidden?: boolean;
+ actions: AnnotationAction[];
+}
+
+export interface AnnotationToolbarProps {
+ className?: string;
+}
+
+// Hover state for specific polyline points/segments
+export interface HoveredPolylineInfo {
+ labelId: string;
+ segmentIndex: number;
+ // undefined means hovering over the segment, not a specific point
+ pointIndex?: number;
+}
+
+// Transform control state
+export type TransformMode = "translate" | "rotate" | "scale";
+export type TransformSpace = "world" | "local";
+
+export interface Spatial {
+ position: [number, number, number];
+ quaternion?: [number, number, number, number];
+}
+
+export interface SelectedPoint {
+ labelId: string;
+ segmentIndex: number;
+ pointIndex: number;
+}
+
+// Transform data for HUD display
+export interface TransformData {
+ // Delta X
+ dx?: number;
+ // Delta Y
+ dy?: number;
+ // Delta Z
+ dz?: number;
+ // Absolute world position X
+ x?: number;
+ // Absolute world position Y
+ y?: number;
+ // Absolute world position Z
+ z?: number;
+ // Dimensions X
+ dimensionX?: number;
+ // Dimensions Y
+ dimensionY?: number;
+ // Dimensions Z
+ dimensionZ?: number;
+ // Local rotation X (in degrees)
+ rotationX?: number;
+ // Local rotation Y (in degrees)
+ rotationY?: number;
+ // Local rotation Z (in degrees)
+ rotationZ?: number;
+}
+
+// Transformed label data storage
+export interface TransformedLabelData {
+ worldPosition: [number, number, number];
+ dimensions: [number, number, number];
+ localRotation: [number, number, number];
+ worldRotation: [number, number, number];
+}
+
+// Polyline point transformations - stores modified points for each label
+export interface PolylinePointTransform {
+ segmentIndex: number;
+ pointIndex: number;
+ position: [number, number, number];
+}
+
+export interface SegmentPolylineState {
+ isActive: boolean;
+ vertices: [number, number, number][];
+ currentMousePosition: [number, number, number] | null;
+ isClosed: boolean;
+}
+
+export interface TempPolyline {
+ id: string;
+ vertices: [number, number, number][];
+ isClosed: boolean;
+ color: string;
+ lineWidth: number;
+}
+
+export interface AnnotationPlaneState {
+ enabled: boolean;
+ position: [number, number, number];
+ quaternion: [number, number, number, number];
+ showX: boolean;
+ showY: boolean;
+ showZ: boolean;
+}
diff --git a/app/packages/looker-3d/src/annotation/usePolylineAnnotation.tsx b/app/packages/looker-3d/src/annotation/usePolylineAnnotation.tsx
new file mode 100644
index 00000000000..807015c2787
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/usePolylineAnnotation.tsx
@@ -0,0 +1,411 @@
+import { ThreeEvent } from "@react-three/fiber";
+import chroma from "chroma-js";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
+import { useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
+import type { Vector3Tuple } from "three";
+import * as THREE from "three";
+import {
+ editSegmentsModeAtom,
+ hoveredLabelAtom,
+ hoveredPolylineInfoAtom,
+ polylineEffectivePointsAtom,
+ polylinePointTransformsAtom,
+ tempLabelTransformsAtom,
+} from "../state";
+import { PolylinePointMarker } from "./PolylinePointMarker";
+import {
+ applyDeltaToAllPoints,
+ applyTransformsToPolyline,
+ findClickedSegment,
+ insertVertexInSegment,
+ updateDuplicateVertices,
+} from "./utils/polyline-utils";
+
+interface UsePolylineAnnotationProps {
+ label: any;
+ points3d: Vector3Tuple[][];
+ strokeAndFillColor: string;
+ isAnnotateMode: boolean;
+ isSelectedForAnnotation: boolean;
+}
+
+export const usePolylineAnnotation = ({
+ label,
+ points3d,
+ strokeAndFillColor,
+ isAnnotateMode,
+ isSelectedForAnnotation,
+}: UsePolylineAnnotationProps) => {
+ const [polylinePointTransforms, setPolylinePointTransforms] = useRecoilState(
+ polylinePointTransformsAtom
+ );
+
+ const editSegmentsMode = useRecoilValue(editSegmentsModeAtom);
+
+ const setHoveredLabel = useSetRecoilState(hoveredLabelAtom);
+ const setHoveredPolylineInfo = useSetRecoilState(hoveredPolylineInfoAtom);
+
+ const [effectivePoints3d, setPolylineEffectivePoints] = useRecoilState(
+ polylineEffectivePointsAtom(label._id)
+ );
+
+ const setTempPolylineTransforms = useSetRecoilState(
+ tempLabelTransformsAtom(label._id)
+ );
+
+ const transformControlsRef = useRef(null);
+ const contentRef = useRef(null);
+ const [startMatrix, setStartMatrix] = useState(null);
+
+ const updateEffectivePoints = useCallback(() => {
+ const labelId = label._id;
+ const transforms = polylinePointTransforms[labelId] || [];
+ const result = applyTransformsToPolyline(points3d, transforms);
+ setPolylineEffectivePoints(result);
+ }, [
+ polylinePointTransforms,
+ label._id,
+ points3d,
+ setPolylineEffectivePoints,
+ ]);
+
+ useEffect(() => {
+ updateEffectivePoints();
+ }, [updateEffectivePoints]);
+
+ const centroid = useMemo(() => {
+ if (effectivePoints3d.length === 0) return [0, 0, 0];
+
+ const allPoints = effectivePoints3d.flat();
+ if (allPoints.length === 0) return [0, 0, 0];
+
+ const sum = allPoints.reduce(
+ (acc, point) => [acc[0] + point[0], acc[1] + point[1], acc[2] + point[2]],
+ [0, 0, 0]
+ );
+
+ return [
+ sum[0] / allPoints.length,
+ sum[1] / allPoints.length,
+ sum[2] / allPoints.length,
+ ] as [number, number, number];
+ }, [effectivePoints3d]);
+
+ const pointMarkers = useMemo(() => {
+ if (!isAnnotateMode || !isSelectedForAnnotation) return null;
+
+ if (!strokeAndFillColor) return null;
+
+ // This is to have contrast for annotation,
+ // Or else the point markers would be invisible with large line widths
+ const complementaryColor = chroma(strokeAndFillColor)
+ .set("hsl.h", "+180")
+ .hex();
+
+ // Global deduplication set to prevent multiple markers for the same physical vertex
+ // This ensures that shared vertices between segments only get one draggable marker
+ const visitedPoints = new Set();
+
+ return effectivePoints3d.flatMap((segment, segmentIndex) => {
+ return segment.map((point, pointIndex) => {
+ // Note: important to use a key based only on coordinates (not segment/point indices)
+ // This allows proper deduplication of vertices that appear in multiple segments
+ const key = `${point[0]}-${point[1]}-${point[2]}`;
+
+ // Skip creating a marker if we've already seen this vertex position
+ if (visitedPoints.has(key)) {
+ return null;
+ }
+
+ // Mark this vertex position as visited to prevent duplicates
+ visitedPoints.add(key);
+
+ return (
+ {
+ setPolylinePointTransforms((prev) => {
+ const labelId = label._id;
+ const currentTransforms = prev[labelId] || [];
+
+ const newTransforms = updateDuplicateVertices(
+ point,
+ [newPosition.x, newPosition.y, newPosition.z],
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ return {
+ ...prev,
+ [labelId]: newTransforms,
+ };
+ });
+ }}
+ pulsate={false}
+ />
+ );
+ });
+ });
+ }, [
+ isAnnotateMode,
+ isSelectedForAnnotation,
+ effectivePoints3d,
+ label._id,
+ strokeAndFillColor,
+ setPolylinePointTransforms,
+ ]);
+
+ const centroidMarker = useMemo(() => {
+ if (!isAnnotateMode || !isSelectedForAnnotation) return null;
+
+ if (!strokeAndFillColor) return null;
+
+ const centroidColor = chroma(strokeAndFillColor)
+ .set("hsl.h", "+180")
+ .brighten(1.5)
+ .hex();
+
+ return (
+
+ );
+ }, [
+ isAnnotateMode,
+ isSelectedForAnnotation,
+ centroid,
+ strokeAndFillColor,
+ label._id,
+ ]);
+
+ const syncPolylineTransformationToTempStore = useCallback(() => {
+ const controls = transformControlsRef.current;
+ if (!controls) return;
+
+ const grp = contentRef.current;
+ if (!grp) return;
+
+ const worldPosition = grp.position.clone();
+
+ setTempPolylineTransforms({
+ position: [worldPosition.x, worldPosition.y, worldPosition.z],
+ quaternion: grp.quaternion.toArray(),
+ });
+ }, []);
+
+ const handleTransformStart = useCallback(() => {
+ const grp = contentRef.current;
+ if (!grp) return;
+
+ // Store the start matrix for computing delta later
+ setStartMatrix(grp.matrixWorld.clone());
+ }, []);
+
+ const handleTransformChange = useCallback(() => {
+ syncPolylineTransformationToTempStore();
+ }, [syncPolylineTransformationToTempStore]);
+
+ useEffect(() => {
+ return () => {
+ setTempPolylineTransforms(null);
+ };
+ }, [label._id]);
+
+ const handleTransformEnd = useCallback(() => {
+ const grp = contentRef.current;
+ if (!grp || !startMatrix) return;
+
+ setTempPolylineTransforms(null);
+
+ // Compute world-space delta from start and end matrices
+ const endMatrix = grp.matrixWorld.clone();
+ const deltaMatrix = endMatrix
+ .clone()
+ .multiply(startMatrix.clone().invert());
+
+ // Extract position delta from the delta matrix
+ const deltaPosition = new THREE.Vector3();
+ deltaPosition.setFromMatrixPosition(deltaMatrix);
+
+ const worldDelta = deltaPosition;
+
+ setPolylinePointTransforms((prev) => {
+ const labelId = label._id;
+ const currentTransforms = prev[labelId] || [];
+
+ const newTransforms = applyDeltaToAllPoints(
+ effectivePoints3d,
+ points3d,
+ currentTransforms,
+ [worldDelta.x, worldDelta.y, worldDelta.z]
+ );
+
+ return { ...prev, [labelId]: newTransforms };
+ });
+
+ // Reset group position to prevent double-application
+ // This is important because transform controls are applied to the group
+ // Whereas we create polylines from the effective points
+ if (contentRef.current) {
+ contentRef.current.position.set(0, 0, 0);
+ }
+
+ setStartMatrix(null);
+ }, [
+ label._id,
+ points3d,
+ effectivePoints3d,
+ setPolylinePointTransforms,
+ startMatrix,
+ ]);
+
+ const handlePointerOver = useCallback(() => {
+ if (isAnnotateMode) {
+ setHoveredLabel(label);
+ }
+ }, [isAnnotateMode, setHoveredLabel, label]);
+
+ const handlePointerOut = useCallback(() => {
+ if (isAnnotateMode) {
+ setHoveredLabel(null);
+ }
+ }, [isAnnotateMode, setHoveredLabel]);
+
+ const handleSegmentPointerOver = useCallback(
+ (segmentIndex: number) => {
+ if (isAnnotateMode) {
+ setHoveredPolylineInfo({
+ labelId: label._id,
+ segmentIndex,
+ // pointIndex is undefined when hovering over the segment
+ });
+
+ if (editSegmentsMode) {
+ document.body.style.cursor = "crosshair";
+ }
+ }
+ },
+ [isAnnotateMode, setHoveredPolylineInfo, label._id, editSegmentsMode]
+ );
+
+ const handleSegmentPointerOut = useCallback(() => {
+ if (isAnnotateMode) {
+ setHoveredPolylineInfo(null);
+ }
+
+ if (!editSegmentsMode) {
+ document.body.style.cursor = "default";
+ }
+ }, [isAnnotateMode, setHoveredPolylineInfo, editSegmentsMode]);
+
+ const handleSegmentClick = useCallback(
+ (event: ThreeEvent) => {
+ if (!editSegmentsMode || !isSelectedForAnnotation || !isAnnotateMode) {
+ return;
+ }
+
+ event.stopPropagation();
+
+ // Get the click position from the event
+ const clickPosition: Vector3Tuple = [
+ event.point.x,
+ event.point.y,
+ event.point.z,
+ ];
+
+ // Find which segment was clicked
+ const clickResult = findClickedSegment(
+ effectivePoints3d,
+ clickPosition,
+ // Distance threshold
+ 0.2
+ );
+
+ if (clickResult) {
+ const { segmentIndex, newVertexPosition } = clickResult;
+
+ // Insert the new vertex into the segment
+ setPolylinePointTransforms((prev) => {
+ const labelId = label._id;
+ const currentTransforms = prev[labelId] || [];
+
+ const newTransforms = insertVertexInSegment(
+ points3d,
+ currentTransforms,
+ segmentIndex,
+ newVertexPosition
+ );
+
+ // If newTransforms is null, it means the new vertex was too close to an existing vertex
+ // In that case, we don't update the transforms
+ if (newTransforms === null) {
+ return prev;
+ }
+
+ return {
+ ...prev,
+ [labelId]: newTransforms,
+ };
+ });
+ }
+ },
+ [
+ editSegmentsMode,
+ isSelectedForAnnotation,
+ isAnnotateMode,
+ effectivePoints3d,
+ points3d,
+ label._id,
+ setPolylinePointTransforms,
+ ]
+ );
+
+ const markers = useMemo(() => {
+ if (!isAnnotateMode || !isSelectedForAnnotation) return null;
+
+ return (
+ <>
+ {pointMarkers}
+ {centroidMarker}
+
+ );
+ }, [isAnnotateMode, isSelectedForAnnotation, pointMarkers, centroidMarker]);
+
+ return {
+ // State
+ effectivePoints3d,
+ centroid,
+ isAnnotateMode,
+ isSelectedForAnnotation,
+
+ // Refs
+ transformControlsRef,
+ contentRef,
+
+ // Markers
+ markers,
+
+ // Handlers
+ handleTransformStart,
+ handleTransformChange,
+ handleTransformEnd,
+ handlePointerOver,
+ handlePointerOut,
+ handleSegmentPointerOver,
+ handleSegmentPointerOut,
+ handleSegmentClick,
+ };
+};
diff --git a/app/packages/looker-3d/src/annotation/utils/polyline-delete.test.ts b/app/packages/looker-3d/src/annotation/utils/polyline-delete.test.ts
new file mode 100644
index 00000000000..700ea808d9b
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/utils/polyline-delete.test.ts
@@ -0,0 +1,247 @@
+import { describe, expect, it } from "vitest";
+import type { PolylinePointTransform } from "../types";
+import { deletePolylinePoint } from "./polyline-delete";
+
+describe("deletePolylinePoint", () => {
+ it("should return unchanged transforms when point doesn't exist", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: [1, 1, 1] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 1, 0);
+
+ expect(result.newTransforms).toEqual(transforms);
+ expect(result.deletedSegmentIndices).toEqual([]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+ });
+
+ it("should delete entire segment when point is only in one segment", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: [1, 1, 1] },
+ { segmentIndex: 1, pointIndex: 0, position: [2, 2, 2] },
+ { segmentIndex: 1, pointIndex: 1, position: [3, 3, 3] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 0);
+
+ expect(result.newTransforms).toEqual([
+ { segmentIndex: 1, pointIndex: 0, position: [2, 2, 2] },
+ { segmentIndex: 1, pointIndex: 1, position: [3, 3, 3] },
+ ]);
+ expect(result.deletedSegmentIndices).toEqual([0]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+ });
+
+ it("should merge segments when point is shared between two segments", () => {
+ const sharedPoint: [number, number, number] = [1, 1, 1];
+ const transforms: PolylinePointTransform[] = [
+ // Segment 0: [0,0,0] -> [1,1,1]
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: sharedPoint },
+ // Segment 1: [1,1,1] -> [2,2,2]
+ { segmentIndex: 1, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 1, pointIndex: 1, position: [2, 2, 2] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 1);
+
+ expect(result.newTransforms).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: [2, 2, 2] },
+ ]);
+ expect(result.deletedSegmentIndices).toEqual([0, 1]);
+ expect(result.mergedSegmentIndex).toBe(0);
+ });
+
+ it("should merge segments with different segment indices", () => {
+ const sharedPoint: [number, number, number] = [1, 1, 1];
+ const transforms: PolylinePointTransform[] = [
+ // Segment 2: [0,0,0] -> [1,1,1]
+ { segmentIndex: 2, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 2, pointIndex: 1, position: sharedPoint },
+ // Segment 5: [1,1,1] -> [2,2,2]
+ { segmentIndex: 5, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 5, pointIndex: 1, position: [2, 2, 2] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 2, 1);
+
+ expect(result.newTransforms).toEqual([
+ { segmentIndex: 2, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 2, pointIndex: 1, position: [2, 2, 2] },
+ ]);
+ expect(result.deletedSegmentIndices).toEqual([2, 5]);
+ expect(result.mergedSegmentIndex).toBe(2);
+ });
+
+ it("should only remove point from selected segment when point is in more than 2 segments", () => {
+ const sharedPoint: [number, number, number] = [1, 1, 1];
+ const transforms: PolylinePointTransform[] = [
+ // Segment 0: [0,0,0] -> [1,1,1]
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: sharedPoint },
+ // Segment 1: [1,1,1] -> [2,2,2]
+ { segmentIndex: 1, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 1, pointIndex: 1, position: [2, 2, 2] },
+ // Segment 2: [1,1,1] -> [3,3,3]
+ { segmentIndex: 2, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 2, pointIndex: 1, position: [3, 3, 3] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 1);
+
+ expect(result.newTransforms).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 1, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 1, pointIndex: 1, position: [2, 2, 2] },
+ { segmentIndex: 2, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 2, pointIndex: 1, position: [3, 3, 3] },
+ ]);
+ expect(result.deletedSegmentIndices).toEqual([]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+ });
+
+ it("should handle edge case where segments cannot be merged due to missing other points", () => {
+ const sharedPoint: [number, number, number] = [1, 1, 1];
+ const transforms: PolylinePointTransform[] = [
+ // Segment 0: only has the shared point (invalid segment)
+ { segmentIndex: 0, pointIndex: 1, position: sharedPoint },
+ // Segment 1: [1,1,1] -> [2,2,2]
+ { segmentIndex: 1, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 1, pointIndex: 1, position: [2, 2, 2] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 1);
+
+ // The function should detect that there are 2 segments sharing the point
+ // and try to merge them, but since segment 0 only has one point,
+ // it should fall back to just removing the point from the selected segment
+ expect(result.newTransforms).toHaveLength(3);
+ expect(result.deletedSegmentIndices).toEqual([]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+
+ // The result should still contain the shared point from segment 1
+ expect(
+ result.newTransforms.some(
+ (t) =>
+ t.segmentIndex === 1 &&
+ t.pointIndex === 0 &&
+ t.position[0] === 1 &&
+ t.position[1] === 1 &&
+ t.position[2] === 1
+ )
+ ).toBe(true);
+ });
+
+ it("should handle empty transforms array", () => {
+ const result = deletePolylinePoint([], 0, 0);
+
+ expect(result.newTransforms).toEqual([]);
+ expect(result.deletedSegmentIndices).toEqual([]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+ });
+
+ it("should handle single point segment deletion", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 0);
+
+ expect(result.newTransforms).toEqual([]);
+ expect(result.deletedSegmentIndices).toEqual([0]);
+ expect(result.mergedSegmentIndex).toBeUndefined();
+ });
+
+ it("should preserve other segments when deleting one segment", () => {
+ const transforms: PolylinePointTransform[] = [
+ // Segment 0: [0,0,0] -> [1,1,1]
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: [1, 1, 1] },
+ // Segment 1: [2,2,2] -> [3,3,3]
+ { segmentIndex: 1, pointIndex: 0, position: [2, 2, 2] },
+ { segmentIndex: 1, pointIndex: 1, position: [3, 3, 3] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 0);
+
+ expect(result.newTransforms).toEqual([
+ { segmentIndex: 1, pointIndex: 0, position: [2, 2, 2] },
+ { segmentIndex: 1, pointIndex: 1, position: [3, 3, 3] },
+ ]);
+ expect(result.deletedSegmentIndices).toEqual([0]);
+ });
+
+ it("should handle complex merging scenario with multiple segments", () => {
+ const sharedPoint: [number, number, number] = [1, 1, 1];
+ const transforms: PolylinePointTransform[] = [
+ // Segment 0: [0,0,0] -> [1,1,1]
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: sharedPoint },
+ // Segment 1: [1,1,1] -> [2,2,2]
+ { segmentIndex: 1, pointIndex: 0, position: sharedPoint },
+ { segmentIndex: 1, pointIndex: 1, position: [2, 2, 2] },
+ // Segment 2: [3,3,3] -> [4,4,4] (unrelated)
+ { segmentIndex: 2, pointIndex: 0, position: [3, 3, 3] },
+ { segmentIndex: 2, pointIndex: 1, position: [4, 4, 4] },
+ ];
+
+ const result = deletePolylinePoint(transforms, 0, 1);
+
+ // The result should contain the merged segment and the unrelated segment
+ // The order might vary, so let's check the content more flexibly
+ expect(result.newTransforms).toHaveLength(4);
+ expect(result.deletedSegmentIndices).toEqual([0, 1]);
+ expect(result.mergedSegmentIndex).toBe(0);
+
+ // Check that we have the merged segment [0,0,0] -> [2,2,2]
+ const mergedSegment = result.newTransforms.filter(
+ (t) => t.segmentIndex === 0
+ );
+ expect(mergedSegment).toHaveLength(2);
+ expect(
+ mergedSegment.some(
+ (t) =>
+ t.pointIndex === 0 &&
+ t.position[0] === 0 &&
+ t.position[1] === 0 &&
+ t.position[2] === 0
+ )
+ ).toBe(true);
+ expect(
+ mergedSegment.some(
+ (t) =>
+ t.pointIndex === 1 &&
+ t.position[0] === 2 &&
+ t.position[1] === 2 &&
+ t.position[2] === 2
+ )
+ ).toBe(true);
+
+ // Check that we have the unrelated segment [3,3,3] -> [4,4,4]
+ const unrelatedSegment = result.newTransforms.filter(
+ (t) => t.segmentIndex === 2
+ );
+ expect(unrelatedSegment).toHaveLength(2);
+ expect(
+ unrelatedSegment.some(
+ (t) =>
+ t.pointIndex === 0 &&
+ t.position[0] === 3 &&
+ t.position[1] === 3 &&
+ t.position[2] === 3
+ )
+ ).toBe(true);
+ expect(
+ unrelatedSegment.some(
+ (t) =>
+ t.pointIndex === 1 &&
+ t.position[0] === 4 &&
+ t.position[1] === 4 &&
+ t.position[2] === 4
+ )
+ ).toBe(true);
+ });
+});
diff --git a/app/packages/looker-3d/src/annotation/utils/polyline-delete.ts b/app/packages/looker-3d/src/annotation/utils/polyline-delete.ts
new file mode 100644
index 00000000000..6a33b4b3b6c
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/utils/polyline-delete.ts
@@ -0,0 +1,133 @@
+import type { PolylinePointTransform } from "../types";
+
+export interface DeletePointResult {
+ newTransforms: PolylinePointTransform[];
+ deletedSegmentIndices: number[];
+ mergedSegmentIndex?: number;
+}
+
+/**
+ * Deletes a polyline point and handles segment merging logic.
+ *
+ * @param currentTransforms - Current polyline point transforms
+ * @param segmentIndex - Index of the segment containing the point to delete
+ * @param pointIndex - Index of the point within the segment to delete
+ * @returns Object containing new transforms and metadata about the operation
+ */
+export function deletePolylinePoint(
+ currentTransforms: PolylinePointTransform[],
+ segmentIndex: number,
+ pointIndex: number
+): DeletePointResult {
+ // Find the position of the point being deleted
+ const pointToDelete = currentTransforms.find(
+ (transform) =>
+ transform.segmentIndex === segmentIndex &&
+ transform.pointIndex === pointIndex
+ );
+
+ if (!pointToDelete) {
+ // Point doesn't exist in transforms, nothing to delete
+ return {
+ newTransforms: currentTransforms,
+ deletedSegmentIndices: [],
+ };
+ }
+
+ const pointPosition = pointToDelete.position;
+
+ // Find all segments that share this point position
+ const segmentsWithThisPoint = currentTransforms.filter(
+ (transform) =>
+ transform.position[0] === pointPosition[0] &&
+ transform.position[1] === pointPosition[1] &&
+ transform.position[2] === pointPosition[2]
+ );
+
+ // Group by segment index
+ const segmentsMap = new Map();
+ segmentsWithThisPoint.forEach((transform) => {
+ if (!segmentsMap.has(transform.segmentIndex)) {
+ segmentsMap.set(transform.segmentIndex, []);
+ }
+ segmentsMap.get(transform.segmentIndex)!.push(transform);
+ });
+
+ let newTransforms = [...currentTransforms];
+ const deletedSegmentIndices: number[] = [];
+ let mergedSegmentIndex: number | undefined;
+
+ if (segmentsMap.size === 1) {
+ // Point is only in one segment, delete the entire segment
+ const segmentToDelete = segmentsMap.keys().next().value;
+ deletedSegmentIndices.push(segmentToDelete);
+ newTransforms = newTransforms.filter(
+ (transform) => transform.segmentIndex !== segmentToDelete
+ );
+ } else if (segmentsMap.size === 2) {
+ // Point is shared between two segments, merge them
+ const segmentIndices = Array.from(segmentsMap.keys());
+ const segment1 = segmentIndices[0];
+ const segment2 = segmentIndices[1];
+
+ // Get the other points from both segments
+ const segment1Points = currentTransforms.filter(
+ (transform) => transform.segmentIndex === segment1
+ );
+ const segment2Points = currentTransforms.filter(
+ (transform) => transform.segmentIndex === segment2
+ );
+
+ // Find the non-shared points
+ const segment1OtherPoint = segment1Points.find(
+ (transform) =>
+ transform.position[0] !== pointPosition[0] ||
+ transform.position[1] !== pointPosition[1] ||
+ transform.position[2] !== pointPosition[2]
+ );
+ const segment2OtherPoint = segment2Points.find(
+ (transform) =>
+ transform.position[0] !== pointPosition[0] ||
+ transform.position[1] !== pointPosition[1] ||
+ transform.position[2] !== pointPosition[2]
+ );
+
+ if (segment1OtherPoint && segment2OtherPoint) {
+ // Remove both old segments
+ deletedSegmentIndices.push(segment1, segment2);
+ newTransforms = newTransforms.filter(
+ (transform) =>
+ transform.segmentIndex !== segment1 &&
+ transform.segmentIndex !== segment2
+ );
+
+ // Create a new merged segment
+ mergedSegmentIndex = Math.min(segment1, segment2);
+ newTransforms.push({
+ segmentIndex: mergedSegmentIndex,
+ pointIndex: 0,
+ position: segment1OtherPoint.position,
+ });
+ newTransforms.push({
+ segmentIndex: mergedSegmentIndex,
+ pointIndex: 1,
+ position: segment2OtherPoint.position,
+ });
+ }
+ } else {
+ // Point is in more than 2 segments, just remove the point from the selected segment
+ newTransforms = newTransforms.filter(
+ (transform) =>
+ !(
+ transform.segmentIndex === segmentIndex &&
+ transform.pointIndex === pointIndex
+ )
+ );
+ }
+
+ return {
+ newTransforms,
+ deletedSegmentIndices,
+ mergedSegmentIndex,
+ };
+}
diff --git a/app/packages/looker-3d/src/annotation/utils/polyline-utils.test.ts b/app/packages/looker-3d/src/annotation/utils/polyline-utils.test.ts
new file mode 100644
index 00000000000..06874caa5f6
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/utils/polyline-utils.test.ts
@@ -0,0 +1,1555 @@
+import type { Vector3Tuple } from "three";
+import { describe, expect, it } from "vitest";
+import { SNAP_TOLERANCE } from "../../constants";
+import type { PolylinePointTransform } from "../types";
+import {
+ applyDeltaToAllPoints,
+ applyTransformsToPolyline,
+ calculatePolylineCentroid,
+ findClickedSegment,
+ findClosestPointOnSegment,
+ findSharedPointSegments,
+ getCurrentVertexPosition,
+ getVertexPosition,
+ insertVertexInSegment,
+ shouldClosePolylineLoop,
+ updateDuplicateVertices,
+} from "./polyline-utils";
+
+// Import the positionsEqual function for testing (it's not exported, so we'll test it indirectly)
+// We'll test it through findSharedPointSegments which uses it internally
+
+describe("getVertexPosition", () => {
+ const mockPoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ [
+ [2, 2, 2],
+ [3, 3, 3],
+ ],
+ ];
+
+ it("returns transformed position when transform exists", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ ];
+ const selectedPoint = { segmentIndex: 0, pointIndex: 0, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, mockPoints3d, transforms);
+
+ expect(result).toEqual([5, 5, 5]);
+ });
+
+ it("returns original position when no transform exists", () => {
+ const transforms: PolylinePointTransform[] = [];
+ const selectedPoint = { segmentIndex: 0, pointIndex: 0, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, mockPoints3d, transforms);
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("returns null for invalid segment index", () => {
+ const transforms: PolylinePointTransform[] = [];
+ const selectedPoint = { segmentIndex: 5, pointIndex: 0, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, mockPoints3d, transforms);
+
+ expect(result).toBeNull();
+ });
+
+ it("returns null for invalid point index", () => {
+ const transforms: PolylinePointTransform[] = [];
+ const selectedPoint = { segmentIndex: 0, pointIndex: 5, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, mockPoints3d, transforms);
+
+ expect(result).toBeNull();
+ });
+
+ it("handles empty transforms array", () => {
+ const transforms: PolylinePointTransform[] = [];
+ const selectedPoint = { segmentIndex: 0, pointIndex: 0, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, mockPoints3d, transforms);
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("handles empty points3d array", () => {
+ const transforms: PolylinePointTransform[] = [];
+ const selectedPoint = { segmentIndex: 0, pointIndex: 0, labelId: "test" };
+
+ const result = getVertexPosition(selectedPoint, [], transforms);
+
+ expect(result).toBeNull();
+ });
+});
+
+describe("calculatePolylineCentroid", () => {
+ it("calculates correct centroid for single segment", () => {
+ const points3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [2, 2, 2],
+ [4, 4, 4],
+ ],
+ ];
+
+ const result = calculatePolylineCentroid(points3d);
+
+ expect(result).toEqual([2, 2, 2]);
+ });
+
+ it("calculates correct centroid for multiple segments", () => {
+ const points3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [2, 2, 2],
+ ],
+ [
+ [4, 4, 4],
+ [6, 6, 6],
+ ],
+ ];
+
+ const result = calculatePolylineCentroid(points3d);
+
+ expect(result).toEqual([3, 3, 3]);
+ });
+
+ it("returns [0,0,0] for empty points array", () => {
+ const result = calculatePolylineCentroid([]);
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("handles single point", () => {
+ const points3d: Vector3Tuple[][] = [[[5, 10, 15]]];
+
+ const result = calculatePolylineCentroid(points3d);
+
+ expect(result).toEqual([5, 10, 15]);
+ });
+
+ it("handles points with negative coordinates", () => {
+ const points3d: Vector3Tuple[][] = [
+ [
+ [-1, -2, -3],
+ [1, 2, 3],
+ ],
+ ];
+
+ const result = calculatePolylineCentroid(points3d);
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("handles very large coordinate values", () => {
+ const points3d: Vector3Tuple[][] = [
+ [
+ [1000000, 2000000, 3000000],
+ [2000000, 4000000, 6000000],
+ ],
+ ];
+
+ const result = calculatePolylineCentroid(points3d);
+
+ expect(result).toEqual([1500000, 3000000, 4500000]);
+ });
+});
+
+describe("findSharedPointSegments", () => {
+ it("finds all segments sharing a point", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 1, position: [1, 1, 1] },
+ { segmentIndex: 1, pointIndex: 0, position: [1, 1, 1] },
+ { segmentIndex: 2, pointIndex: 0, position: [2, 2, 2] },
+ ];
+ const position: [number, number, number] = [1, 1, 1];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0, 1]);
+ });
+
+ it("returns empty array when no segments share point", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [1, 1, 1] },
+ ];
+ const position: [number, number, number] = [2, 2, 2];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([]);
+ });
+
+ it("handles floating point precision issues with epsilon tolerance", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ { segmentIndex: 1, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ ];
+ const position: [number, number, number] = [0.1, 0.2, 0.3];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0, 1]);
+ });
+
+ it("handles floating point precision with small differences within epsilon", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ {
+ segmentIndex: 1,
+ pointIndex: 0,
+ position: [0.1000001, 0.2000001, 0.3000001],
+ },
+ { segmentIndex: 2, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ ];
+ const position: [number, number, number] = [0.1, 0.2, 0.3];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0, 1, 2]);
+ });
+
+ it("rejects positions with differences larger than epsilon", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ { segmentIndex: 1, pointIndex: 0, position: [0.1, 0.2, 0.3] },
+ ];
+ const position: [number, number, number] = [0.1, 0.2, 0.4]; // Different Z coordinate
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([]);
+ });
+
+ it("handles very small epsilon differences", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [1.0, 2.0, 3.0] },
+ {
+ segmentIndex: 1,
+ pointIndex: 0,
+ position: [1.0000005, 2.0000005, 3.0000005],
+ },
+ ];
+ const position: [number, number, number] = [1.0, 2.0, 3.0];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0, 1]);
+ });
+
+ it("handles negative coordinates with epsilon tolerance", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [-1.0, -2.0, -3.0] },
+ {
+ segmentIndex: 1,
+ pointIndex: 0,
+ position: [-1.0000001, -2.0000001, -3.0000001],
+ },
+ ];
+ const position: [number, number, number] = [-1.0, -2.0, -3.0];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0, 1]);
+ });
+
+ it("handles duplicate transforms", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [1, 1, 1] },
+ { segmentIndex: 0, pointIndex: 0, position: [1, 1, 1] },
+ ];
+ const position: [number, number, number] = [1, 1, 1];
+
+ const result = findSharedPointSegments(transforms, position);
+
+ expect(result).toEqual([0]);
+ });
+});
+
+describe("shouldClosePolylineLoop", () => {
+ it("returns true when within snap tolerance of first vertex", () => {
+ const vertices: Vector3Tuple[] = [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ];
+ const currentPosition: Vector3Tuple = [0.1, 0.1, 0.1];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition, 0.5);
+
+ expect(result).toBe(true);
+ });
+
+ it("returns false when outside snap tolerance", () => {
+ const vertices: Vector3Tuple[] = [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ];
+ const currentPosition: Vector3Tuple = [5, 5, 5];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition, 0.5);
+
+ expect(result).toBe(false);
+ });
+
+ it("returns false when less than 3 vertices", () => {
+ const vertices: Vector3Tuple[] = [
+ [0, 0, 0],
+ [1, 1, 1],
+ ];
+ const currentPosition: Vector3Tuple = [0, 0, 0];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition);
+
+ expect(result).toBe(false);
+ });
+
+ it("returns false for empty vertices array", () => {
+ const vertices: Vector3Tuple[] = [];
+ const currentPosition: Vector3Tuple = [0, 0, 0];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition);
+
+ expect(result).toBe(false);
+ });
+
+ it("handles exact position match", () => {
+ const vertices: Vector3Tuple[] = [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ];
+ const currentPosition: Vector3Tuple = [0, 0, 0];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition);
+
+ expect(result).toBe(true);
+ });
+
+ it("handles edge case at exact tolerance boundary", () => {
+ const vertices: Vector3Tuple[] = [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ];
+ const currentPosition: Vector3Tuple = [SNAP_TOLERANCE, 0, 0];
+
+ const result = shouldClosePolylineLoop(vertices, currentPosition);
+
+ expect(result).toBe(false);
+ });
+});
+
+describe("getCurrentVertexPosition", () => {
+ const mockOriginalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ [
+ [2, 2, 2],
+ [3, 3, 3],
+ ],
+ ];
+
+ it("returns transformed position when available", () => {
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ ];
+
+ const result = getCurrentVertexPosition(
+ 0,
+ 0,
+ mockOriginalPoints,
+ transforms
+ );
+
+ expect(result).toEqual([5, 5, 5]);
+ });
+
+ it("falls back to original position", () => {
+ const transforms: PolylinePointTransform[] = [];
+
+ const result = getCurrentVertexPosition(
+ 0,
+ 0,
+ mockOriginalPoints,
+ transforms
+ );
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("handles missing segment in original points", () => {
+ const transforms: PolylinePointTransform[] = [];
+
+ const result = getCurrentVertexPosition(
+ 5,
+ 0,
+ mockOriginalPoints,
+ transforms
+ );
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("handles missing point in segment", () => {
+ const transforms: PolylinePointTransform[] = [];
+
+ const result = getCurrentVertexPosition(
+ 0,
+ 5,
+ mockOriginalPoints,
+ transforms
+ );
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+
+ it("returns sensible default for completely invalid indices", () => {
+ const transforms: PolylinePointTransform[] = [];
+
+ const result = getCurrentVertexPosition(
+ 5,
+ 5,
+ mockOriginalPoints,
+ transforms
+ );
+
+ expect(result).toEqual([0, 0, 0]);
+ });
+});
+
+describe("applyTransformsToPolyline", () => {
+ it("applies single transform correctly", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ ];
+
+ const result = applyTransformsToPolyline(originalPoints, transforms);
+
+ expect(result).toEqual([
+ [
+ [5, 5, 5],
+ [1, 1, 1],
+ ],
+ ]);
+ });
+
+ it("applies multiple transforms to different points", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ [
+ [2, 2, 2],
+ [3, 3, 3],
+ ],
+ ];
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 1, position: [6, 6, 6] },
+ ];
+
+ const result = applyTransformsToPolyline(originalPoints, transforms);
+
+ expect(result).toEqual([
+ [
+ [5, 5, 5],
+ [1, 1, 1],
+ ],
+ [
+ [2, 2, 2],
+ [6, 6, 6],
+ ],
+ ]);
+ });
+
+ it("handles transforms to non-existent segments (creates default)", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 2, pointIndex: 0, position: [5, 5, 5] },
+ ];
+
+ const result = applyTransformsToPolyline(originalPoints, transforms);
+
+ expect(result).toEqual([
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ undefined,
+ [
+ [5, 5, 5],
+ [0, 0, 0],
+ ],
+ ]);
+ });
+
+ it("handles empty transforms array", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const transforms: PolylinePointTransform[] = [];
+
+ const result = applyTransformsToPolyline(originalPoints, transforms);
+
+ expect(result).toEqual([
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ]);
+ });
+
+ it("preserves untransformed points", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ],
+ ];
+ const transforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ ];
+
+ const result = applyTransformsToPolyline(originalPoints, transforms);
+
+ expect(result).toEqual([
+ [
+ [0, 0, 0],
+ [5, 5, 5],
+ [2, 2, 2],
+ ],
+ ]);
+ });
+});
+
+describe("applyDeltaToAllPoints", () => {
+ it("applies delta to all points correctly", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const delta: Vector3Tuple = [2, 2, 2];
+
+ const result = applyDeltaToAllPoints(
+ effectivePoints,
+ originalPoints,
+ currentTransforms,
+ delta
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [2, 2, 2] },
+ { segmentIndex: 0, pointIndex: 1, position: [3, 3, 3] },
+ ]);
+ });
+
+ it("combines with existing transforms", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [5, 5, 5],
+ [1, 1, 1],
+ ],
+ ];
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ ];
+ const delta: Vector3Tuple = [1, 1, 1];
+
+ const result = applyDeltaToAllPoints(
+ effectivePoints,
+ originalPoints,
+ currentTransforms,
+ delta
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [6, 6, 6] },
+ { segmentIndex: 0, pointIndex: 1, position: [2, 2, 2] },
+ ]);
+ });
+
+ it("handles empty points", () => {
+ const effectivePoints: Vector3Tuple[][] = [];
+ const originalPoints: Vector3Tuple[][] = [];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const delta: Vector3Tuple = [1, 1, 1];
+
+ const result = applyDeltaToAllPoints(
+ effectivePoints,
+ originalPoints,
+ currentTransforms,
+ delta
+ );
+
+ expect(result).toEqual([]);
+ });
+
+ it("handles zero delta", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const delta: Vector3Tuple = [0, 0, 0];
+
+ const result = applyDeltaToAllPoints(
+ effectivePoints,
+ originalPoints,
+ currentTransforms,
+ delta
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [0, 0, 0] },
+ { segmentIndex: 0, pointIndex: 1, position: [1, 1, 1] },
+ ]);
+ });
+
+ it("handles negative delta values", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [5, 5, 5],
+ [3, 3, 3],
+ ],
+ ];
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [5, 5, 5],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const delta: Vector3Tuple = [-1, -1, -1];
+
+ const result = applyDeltaToAllPoints(
+ effectivePoints,
+ originalPoints,
+ currentTransforms,
+ delta
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [4, 4, 4] },
+ { segmentIndex: 0, pointIndex: 1, position: [2, 2, 2] },
+ ]);
+ });
+});
+
+describe("updateDuplicateVertices", () => {
+ it("updates all duplicate vertices across segments", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1], // This vertex appears in multiple segments
+ [2, 2, 2],
+ ],
+ [
+ [1, 1, 1], // Duplicate vertex
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("updates existing transforms for duplicate vertices", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ],
+ [
+ [1, 1, 1],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 1, position: [2, 2, 2] },
+ { segmentIndex: 1, pointIndex: 0, position: [3, 3, 3] },
+ ];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles vertices with no duplicates", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ],
+ [
+ [3, 3, 3],
+ [4, 4, 4],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles multiple segments with the same vertex", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [1, 1, 1], // First occurrence
+ [2, 2, 2],
+ ],
+ [
+ [1, 1, 1], // Second occurrence
+ [3, 3, 3],
+ ],
+ [
+ [1, 1, 1], // Third occurrence
+ [4, 4, 4],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ { segmentIndex: 2, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("preserves existing transforms for non-duplicate vertices", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ],
+ [
+ [1, 1, 1],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [10, 10, 10] },
+ { segmentIndex: 1, pointIndex: 1, position: [20, 20, 20] },
+ ];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [10, 10, 10] },
+ { segmentIndex: 1, pointIndex: 1, position: [20, 20, 20] },
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles empty effectivePoints3d array", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([]);
+ });
+
+ it("handles empty segments in effectivePoints3d", () => {
+ const movedPoint: Vector3Tuple = [1, 1, 1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [[], []];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([]);
+ });
+
+ it("handles floating point precision in vertex matching", () => {
+ const movedPoint: Vector3Tuple = [1.0, 1.0, 1.0];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1.0, 1.0, 1.0], // Exact match
+ [2, 2, 2],
+ ],
+ [
+ [1.0, 1.0, 1.0], // Exact match
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles negative coordinates", () => {
+ const movedPoint: Vector3Tuple = [-1, -1, -1];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [-1, -1, -1],
+ [2, 2, 2],
+ ],
+ [
+ [-1, -1, -1],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles zero coordinates", () => {
+ const movedPoint: Vector3Tuple = [0, 0, 0];
+ const newPosition: Vector3Tuple = [5, 5, 5];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1, 1, 1],
+ [2, 2, 2],
+ ],
+ [
+ [0, 0, 0],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 0, position: [5, 5, 5] },
+ { segmentIndex: 1, pointIndex: 0, position: [5, 5, 5] },
+ ]);
+ });
+
+ it("handles large coordinate values", () => {
+ const movedPoint: Vector3Tuple = [1000000, 2000000, 3000000];
+ const newPosition: Vector3Tuple = [5000000, 5000000, 5000000];
+ const effectivePoints3d: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [1000000, 2000000, 3000000],
+ [2, 2, 2],
+ ],
+ [
+ [1000000, 2000000, 3000000],
+ [3, 3, 3],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+
+ const result = updateDuplicateVertices(
+ movedPoint,
+ newPosition,
+ effectivePoints3d,
+ currentTransforms
+ );
+
+ expect(result).toEqual([
+ { segmentIndex: 0, pointIndex: 1, position: [5000000, 5000000, 5000000] },
+ { segmentIndex: 1, pointIndex: 0, position: [5000000, 5000000, 5000000] },
+ ]);
+ });
+});
+
+describe("findClosestPointOnSegment", () => {
+ it("finds closest point at the start of segment", () => {
+ const segmentStart: Vector3Tuple = [0, 0, 0];
+ const segmentEnd: Vector3Tuple = [10, 0, 0];
+ const clickPosition: Vector3Tuple = [-5, 0, 0];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ expect(result.closestPoint).toEqual([0, 0, 0]);
+ expect(result.t).toBe(0);
+ });
+
+ it("finds closest point at the end of segment", () => {
+ const segmentStart: Vector3Tuple = [0, 0, 0];
+ const segmentEnd: Vector3Tuple = [10, 0, 0];
+ const clickPosition: Vector3Tuple = [15, 0, 0];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ expect(result.closestPoint).toEqual([10, 0, 0]);
+ expect(result.t).toBe(1);
+ });
+
+ it("finds closest point in the middle of segment", () => {
+ const segmentStart: Vector3Tuple = [0, 0, 0];
+ const segmentEnd: Vector3Tuple = [10, 0, 0];
+ const clickPosition: Vector3Tuple = [5, 5, 0];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ expect(result.closestPoint).toEqual([5, 0, 0]);
+ expect(result.t).toBeCloseTo(0.5);
+ });
+
+ it("handles diagonal segments", () => {
+ const segmentStart: Vector3Tuple = [0, 0, 0];
+ const segmentEnd: Vector3Tuple = [10, 10, 10];
+ const clickPosition: Vector3Tuple = [5, 5, 0];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ // The closest point should be somewhere along the diagonal
+ expect(result.t).toBeGreaterThan(0);
+ expect(result.t).toBeLessThan(1);
+ });
+
+ it("handles zero-length segments", () => {
+ const segmentStart: Vector3Tuple = [5, 5, 5];
+ const segmentEnd: Vector3Tuple = [5, 5, 5];
+ const clickPosition: Vector3Tuple = [10, 10, 10];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ expect(result.closestPoint).toEqual([5, 5, 5]);
+ expect(result.t).toBe(0);
+ });
+
+ it("projects perpendicular clicks correctly", () => {
+ const segmentStart: Vector3Tuple = [0, 0, 0];
+ const segmentEnd: Vector3Tuple = [10, 0, 0];
+ const clickPosition: Vector3Tuple = [5, 100, 0];
+
+ const result = findClosestPointOnSegment(
+ segmentStart,
+ segmentEnd,
+ clickPosition
+ );
+
+ // Should project straight down to [5, 0, 0]
+ expect(result.closestPoint[0]).toBeCloseTo(5);
+ expect(result.closestPoint[1]).toBeCloseTo(0);
+ expect(result.closestPoint[2]).toBeCloseTo(0);
+ expect(result.t).toBeCloseTo(0.5);
+ });
+});
+
+describe("insertVertexInSegment", () => {
+ it("splits a simple two-point segment correctly", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const newVertexPosition: Vector3Tuple = [5, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+
+ // Should create transforms for segment 0 (A->B) and segment 1 (B->C)
+ expect(result!).toHaveLength(3);
+
+ // Segment 0: point 1 should be the new vertex
+ expect(
+ result!.find((t) => t.segmentIndex === 0 && t.pointIndex === 1)?.position
+ ).toEqual([5, 0, 0]);
+
+ // Segment 1: point 0 should be the new vertex
+ expect(
+ result!.find((t) => t.segmentIndex === 1 && t.pointIndex === 0)?.position
+ ).toEqual([5, 0, 0]);
+
+ // Segment 1: point 1 should be the original end point
+ expect(
+ result!.find((t) => t.segmentIndex === 1 && t.pointIndex === 1)?.position
+ ).toEqual([10, 0, 0]);
+ });
+
+ it("increments subsequent segment indices", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ [
+ [10, 0, 0],
+ [20, 0, 0],
+ ],
+ [
+ [20, 0, 0],
+ [30, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 1, pointIndex: 0, position: [12, 2, 0] },
+ { segmentIndex: 2, pointIndex: 1, position: [32, 2, 0] },
+ ];
+ const newVertexPosition: Vector3Tuple = [5, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+
+ // Original segment 1 should now be segment 2
+ const segmentTwoTransforms = result!.filter((t) => t.segmentIndex === 2);
+ expect(segmentTwoTransforms.length).toBeGreaterThan(0);
+ expect(
+ segmentTwoTransforms.find((t) => t.pointIndex === 0)?.position
+ ).toEqual([12, 2, 0]);
+
+ // Original segment 2 should now be segment 3
+ const segmentThreeTransforms = result!.filter((t) => t.segmentIndex === 3);
+ expect(segmentThreeTransforms.length).toBeGreaterThan(0);
+ expect(
+ segmentThreeTransforms.find((t) => t.pointIndex === 1)?.position
+ ).toEqual([32, 2, 0]);
+ });
+
+ it("preserves transforms on the target segment's first point", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 0, position: [1, 1, 1] },
+ { segmentIndex: 0, pointIndex: 1, position: [11, 1, 1] },
+ ];
+ const newVertexPosition: Vector3Tuple = [5, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+
+ // First point of segment 0 should keep its transform
+ expect(
+ result!.find((t) => t.segmentIndex === 0 && t.pointIndex === 0)?.position
+ ).toEqual([1, 1, 1]);
+ });
+
+ it("throws error for invalid segment index", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const newVertexPosition: Vector3Tuple = [5, 0, 0];
+
+ expect(() =>
+ insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 5,
+ newVertexPosition
+ )
+ ).toThrow("Invalid segment index");
+ });
+
+ it("throws error for segment with less than 2 points", () => {
+ const originalPoints: Vector3Tuple[][] = [[[0, 0, 0]]];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const newVertexPosition: Vector3Tuple = [5, 0, 0];
+
+ expect(() =>
+ insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ )
+ ).toThrow("must have at least 2 points");
+ });
+
+ it("handles multi-point segments (more than 2 points)", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [5, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const newVertexPosition: Vector3Tuple = [2, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+
+ // Segment 0 should have 2 points: [0,0,0] and [2,0,0]
+ expect(result!.filter((t) => t.segmentIndex === 0).length).toBe(1); // Only new vertex
+
+ // Segment 1 should have 3 points: [2,0,0], [5,0,0], [10,0,0]
+ const segment1Transforms = result!.filter((t) => t.segmentIndex === 1);
+ expect(segment1Transforms.length).toBe(3);
+ });
+
+ it("handles splitting middle segment in multi-segment polyline", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ [
+ [10, 0, 0],
+ [20, 0, 0],
+ ],
+ [
+ [20, 0, 0],
+ [30, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ const newVertexPosition: Vector3Tuple = [15, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 1,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+
+ // Segment 0 should remain unchanged (no transforms)
+ expect(result!.filter((t) => t.segmentIndex === 0).length).toBe(0);
+
+ // Segment 1 should have new vertex at point 1
+ expect(
+ result!.find((t) => t.segmentIndex === 1 && t.pointIndex === 1)?.position
+ ).toEqual([15, 0, 0]);
+
+ // Segment 2 should have new vertex at point 0 and original end point at point 1
+ expect(
+ result!.find((t) => t.segmentIndex === 2 && t.pointIndex === 0)?.position
+ ).toEqual([15, 0, 0]);
+ expect(
+ result!.find((t) => t.segmentIndex === 2 && t.pointIndex === 1)?.position
+ ).toEqual([20, 0, 0]);
+
+ // Original segment 2 is now at index 3, but since it had no transforms,
+ // it still won't have any (it uses original points)
+ // Verify the total transform count: segment 1 (1 transform) + segment 2 (2 transforms) = 3
+ expect(result!.length).toBe(3);
+ });
+
+ it("returns null when new vertex is too close to existing vertex (at start)", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ // New vertex is exactly at the start point
+ const newVertexPosition: Vector3Tuple = [0, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).toBeNull();
+ });
+
+ it("returns null when new vertex is too close to existing vertex (at end)", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ // New vertex is exactly at the end point
+ const newVertexPosition: Vector3Tuple = [10, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).toBeNull();
+ });
+
+ it("returns null when new vertex is within epsilon tolerance of existing vertex", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ // New vertex is very close to start point (within epsilon)
+ const newVertexPosition: Vector3Tuple = [0.0000001, 0.0000001, 0.0000001];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).toBeNull();
+ });
+
+ it("inserts vertex when new position is outside epsilon tolerance", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ // New vertex is outside epsilon tolerance (default is 1e-6)
+ const newVertexPosition: Vector3Tuple = [0.001, 0, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).not.toBeNull();
+ expect(result!.length).toBeGreaterThan(0);
+ });
+
+ it("returns null when new vertex is close to any vertex in multi-point segment", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [5, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const currentTransforms: PolylinePointTransform[] = [];
+ // New vertex is very close to the middle point
+ const newVertexPosition: Vector3Tuple = [5, 0.0000001, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).toBeNull();
+ });
+
+ it("returns null when new vertex is close to a transformed vertex", () => {
+ const originalPoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ // The end point has been transformed to [12, 2, 0]
+ const currentTransforms: PolylinePointTransform[] = [
+ { segmentIndex: 0, pointIndex: 1, position: [12, 2, 0] },
+ ];
+ // New vertex is very close to the transformed position
+ const newVertexPosition: Vector3Tuple = [12, 2.0000001, 0];
+
+ const result = insertVertexInSegment(
+ originalPoints,
+ currentTransforms,
+ 0,
+ newVertexPosition
+ );
+
+ expect(result).toBeNull();
+ });
+});
+
+describe("findClickedSegment", () => {
+ it("finds segment when click is close to it", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const clickPosition: Vector3Tuple = [5, 0.05, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition, 0.1);
+
+ expect(result).not.toBeNull();
+ expect(result?.segmentIndex).toBe(0);
+ expect(result?.newVertexPosition[0]).toBeCloseTo(5);
+ expect(result?.newVertexPosition[1]).toBeCloseTo(0);
+ });
+
+ it("returns null when click is too far from any segment", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const clickPosition: Vector3Tuple = [5, 10, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition, 0.1);
+
+ expect(result).toBeNull();
+ });
+
+ it("finds closest segment in multi-segment polyline", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ [
+ [10, 0, 0],
+ [20, 10, 0],
+ ],
+ [
+ [20, 10, 0],
+ [30, 10, 0],
+ ],
+ ];
+ const clickPosition: Vector3Tuple = [25, 10.05, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition, 0.2);
+
+ expect(result).not.toBeNull();
+ expect(result?.segmentIndex).toBe(2);
+ });
+
+ it("respects custom maxDistance parameter", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const clickPosition: Vector3Tuple = [5, 0.5, 0];
+
+ // With default tolerance (0.1), should not find
+ const result1 = findClickedSegment(effectivePoints, clickPosition, 0.1);
+ expect(result1).toBeNull();
+
+ // With larger tolerance (1.0), should find
+ const result2 = findClickedSegment(effectivePoints, clickPosition, 1.0);
+ expect(result2).not.toBeNull();
+ expect(result2?.segmentIndex).toBe(0);
+ });
+
+ it("handles segments with multiple points", () => {
+ const effectivePoints: Vector3Tuple[][] = [
+ [
+ [0, 0, 0],
+ [5, 0, 0],
+ [10, 0, 0],
+ ],
+ ];
+ const clickPosition: Vector3Tuple = [7, 0.05, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition, 0.2);
+
+ expect(result).not.toBeNull();
+ expect(result?.segmentIndex).toBe(0);
+ // Click is between [5,0,0] and [10,0,0]
+ expect(result?.newVertexPosition[0]).toBeCloseTo(7);
+ });
+
+ it("returns null for empty segments", () => {
+ const effectivePoints: Vector3Tuple[][] = [];
+ const clickPosition: Vector3Tuple = [5, 0, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition);
+
+ expect(result).toBeNull();
+ });
+
+ it("handles segments with only one point", () => {
+ const effectivePoints: Vector3Tuple[][] = [[[0, 0, 0]]];
+ const clickPosition: Vector3Tuple = [0, 0, 0];
+
+ const result = findClickedSegment(effectivePoints, clickPosition);
+
+ // Single point segments should be skipped
+ expect(result).toBeNull();
+ });
+});
diff --git a/app/packages/looker-3d/src/annotation/utils/polyline-utils.ts b/app/packages/looker-3d/src/annotation/utils/polyline-utils.ts
new file mode 100644
index 00000000000..619ccddb29a
--- /dev/null
+++ b/app/packages/looker-3d/src/annotation/utils/polyline-utils.ts
@@ -0,0 +1,536 @@
+import * as THREE from "three";
+import type { Vector3Tuple } from "three";
+import { SNAP_TOLERANCE } from "../../constants";
+import type { PolylinePointTransform } from "../types";
+
+// Epsilon tolerance for floating-point position comparisons
+const EPS = 1e-6;
+
+/**
+ * Compares two 3D positions using epsilon tolerance for floating-point precision.
+ *
+ * @param p1 - First position
+ * @param p2 - Second position
+ * @param epsilon - Tolerance for comparison (default: EPS)
+ * @returns True if positions are equal within tolerance
+ */
+function positionsEqual(
+ p1: [number, number, number],
+ p2: [number, number, number],
+ epsilon: number = EPS
+): boolean {
+ return (
+ Math.abs(p1[0] - p2[0]) <= epsilon &&
+ Math.abs(p1[1] - p2[1]) <= epsilon &&
+ Math.abs(p1[2] - p2[2]) <= epsilon
+ );
+}
+
+/**
+ * Gets the current position of a vertex, considering any applied transforms.
+ *
+ * @param selectedPoint - The point to look up with segmentIndex, pointIndex, and labelId
+ * @param polylinePoints3d - Original polyline points array
+ * @param transforms - Array of applied transforms
+ * @returns The current position of the vertex or null if not found
+ */
+export function getVertexPosition(
+ selectedPoint: { segmentIndex: number; pointIndex: number; labelId: string },
+ polylinePoints3d: Vector3Tuple[][],
+ transforms: PolylinePointTransform[]
+): [number, number, number] | null {
+ if (!selectedPoint || !polylinePoints3d) return null;
+
+ const { segmentIndex, pointIndex } = selectedPoint;
+
+ // Check if segment exists
+ if (segmentIndex >= polylinePoints3d.length) return null;
+
+ // Check if point exists in segment
+ if (pointIndex >= polylinePoints3d[segmentIndex].length) return null;
+
+ // Look for existing transform
+ const transform = transforms.find(
+ (t) => t.segmentIndex === segmentIndex && t.pointIndex === pointIndex
+ );
+
+ // Return transformed position if exists, otherwise return original position
+ if (transform) {
+ return transform.position;
+ }
+
+ return polylinePoints3d[segmentIndex][pointIndex];
+}
+
+/**
+ * Calculates the centroid (center of mass) of a polyline from its points.
+ *
+ * @param points3d - Array of polyline segments, each containing 3D points
+ * @returns The centroid as [x, y, z] coordinates
+ */
+export function calculatePolylineCentroid(
+ points3d: Vector3Tuple[][]
+): [number, number, number] {
+ if (!points3d || points3d.length === 0) return [0, 0, 0];
+
+ // Flatten all points from all segments
+ const allPoints = points3d.flat();
+
+ if (allPoints.length === 0) return [0, 0, 0];
+
+ // Calculate sum of all coordinates
+ const sum = allPoints.reduce(
+ (acc, point) => [acc[0] + point[0], acc[1] + point[1], acc[2] + point[2]],
+ [0, 0, 0]
+ );
+
+ // Return average coordinates
+ return [
+ sum[0] / allPoints.length,
+ sum[1] / allPoints.length,
+ sum[2] / allPoints.length,
+ ];
+}
+
+/**
+ * Finds all segments that share a specific point position.
+ *
+ * @param transforms - Array of point transforms
+ * @param position - The position to search for
+ * @returns Array of segment indices that contain this position
+ */
+export function findSharedPointSegments(
+ transforms: PolylinePointTransform[],
+ position: [number, number, number]
+): number[] {
+ const segmentsWithThisPoint = transforms.filter((transform) =>
+ positionsEqual(transform.position, position)
+ );
+
+ // Get unique segment indices
+ const segmentIndices = new Set(
+ segmentsWithThisPoint.map((transform) => transform.segmentIndex)
+ );
+
+ return Array.from(segmentIndices);
+}
+
+/**
+ * Determines if a polyline should be closed based on proximity to the first vertex.
+ *
+ * @param vertices - Array of current vertices in the polyline
+ * @param currentPosition - The current mouse/cursor position
+ * @param snapTolerance - Distance threshold for snapping (default: SNAP_TOLERANCE)
+ * @returns True if the current position is close enough to the first vertex to close the loop
+ */
+export function shouldClosePolylineLoop(
+ vertices: Vector3Tuple[],
+ currentPosition: Vector3Tuple,
+ snapTolerance: number = SNAP_TOLERANCE
+): boolean {
+ if (vertices.length < 3) return false;
+
+ const firstVertex = new THREE.Vector3(...vertices[0]);
+ const currentPos = new THREE.Vector3(...currentPosition);
+
+ return currentPos.distanceTo(firstVertex) < snapTolerance;
+}
+
+/**
+ * Gets the current effective position of a vertex, considering transforms.
+ *
+ * @param segmentIndex - Index of the segment
+ * @param pointIndex - Index of the point within the segment
+ * @param originalPoints - Original polyline points
+ * @param transforms - Applied transforms
+ * @returns The current effective position of the vertex
+ */
+export function getCurrentVertexPosition(
+ segmentIndex: number,
+ pointIndex: number,
+ originalPoints: Vector3Tuple[][],
+ transforms: PolylinePointTransform[]
+): [number, number, number] {
+ // Look for existing transform
+ const transform = transforms.find(
+ (t) => t.segmentIndex === segmentIndex && t.pointIndex === pointIndex
+ );
+
+ if (transform) {
+ return transform.position;
+ }
+
+ // Fall back to original position
+ if (
+ segmentIndex < originalPoints.length &&
+ pointIndex < originalPoints[segmentIndex].length
+ ) {
+ return originalPoints[segmentIndex][pointIndex];
+ }
+
+ // Return default position for invalid indices
+ return [0, 0, 0];
+}
+
+/**
+ * Applies transforms to polyline points, returning the effective points.
+ *
+ * @param originalPoints - Original polyline points array
+ * @param transforms - Array of transforms to apply
+ * @returns Array of effective points after applying transforms
+ */
+export function applyTransformsToPolyline(
+ originalPoints: Vector3Tuple[][],
+ transforms: PolylinePointTransform[]
+): Vector3Tuple[][] {
+ // Start with a copy of original points
+ const result = originalPoints.map((segment) => [...segment]);
+
+ // Apply each transform
+ transforms.forEach((transform) => {
+ const { segmentIndex, pointIndex, position } = transform;
+
+ // Ensure segment exists
+ if (result[segmentIndex] === undefined) {
+ result[segmentIndex] = [
+ [0, 0, 0],
+ [0, 0, 0],
+ ];
+ }
+
+ // Apply transform if point exists
+ if (pointIndex < result[segmentIndex].length) {
+ result[segmentIndex][pointIndex] = position;
+ }
+ });
+
+ return result;
+}
+
+/**
+ * Applies a delta transformation to all points in a polyline.
+ *
+ * @param effectivePoints - Current effective points
+ * @param originalPoints - Original polyline points
+ * @param currentTransforms - Existing transforms
+ * @param delta - The delta to apply to all points
+ * @returns New transforms array with delta applied to all points
+ */
+export function applyDeltaToAllPoints(
+ effectivePoints: Vector3Tuple[][],
+ originalPoints: Vector3Tuple[][],
+ currentTransforms: PolylinePointTransform[],
+ delta: Vector3Tuple
+): PolylinePointTransform[] {
+ const newTransforms: PolylinePointTransform[] = [];
+
+ // Helper to get current effective point for (segmentIndex, pointIndex)
+ const getCurrent = (
+ segmentIndex: number,
+ pointIndex: number
+ ): [number, number, number] => {
+ const t = currentTransforms.find(
+ (x) => x.segmentIndex === segmentIndex && x.pointIndex === pointIndex
+ );
+ return (t?.position ?? originalPoints[segmentIndex][pointIndex]) as [
+ number,
+ number,
+ number
+ ];
+ };
+
+ // Apply delta to all points
+ effectivePoints.forEach((segment, segmentIndex) => {
+ segment.forEach((_, pointIndex) => {
+ const base = getCurrent(segmentIndex, pointIndex);
+ const p = new THREE.Vector3(...base).add(new THREE.Vector3(...delta));
+ newTransforms.push({
+ segmentIndex,
+ pointIndex,
+ position: [p.x, p.y, p.z],
+ });
+ });
+ });
+
+ return newTransforms;
+}
+
+/**
+ * Updates all duplicate vertices across segments when a vertex is moved.
+ * This prevents tearing by ensuring all segments sharing a vertex move together.
+ *
+ * @param movedPoint - The original point that was moved
+ * @param newPosition - The new position to apply to all duplicates
+ * @param effectivePoints3d - Current effective points array
+ * @param currentTransforms - Existing transforms
+ * @returns Updated transforms array with new position applied to all duplicate vertices
+ */
+export function updateDuplicateVertices(
+ movedPoint: Vector3Tuple,
+ newPosition: Vector3Tuple,
+ effectivePoints3d: Vector3Tuple[][],
+ currentTransforms: PolylinePointTransform[]
+): PolylinePointTransform[] {
+ // Find all occurrences of this vertex across all segments
+ const duplicates: { segmentIndex: number; pointIndex: number }[] = [];
+
+ effectivePoints3d.forEach((segmentPoints, segIdx) => {
+ segmentPoints.forEach((candidatePoint, ptIdx) => {
+ if (positionsEqual(candidatePoint, movedPoint)) {
+ duplicates.push({
+ segmentIndex: segIdx,
+ pointIndex: ptIdx,
+ });
+ }
+ });
+ });
+
+ let newTransforms = [...currentTransforms];
+
+ // Apply the new position to all duplicate vertices
+ duplicates.forEach(({ segmentIndex: dupSeg, pointIndex: dupPt }) => {
+ // Check if this vertex already has a transform
+ const existingTransformIndex = newTransforms.findIndex(
+ (transform) =>
+ transform.segmentIndex === dupSeg && transform.pointIndex === dupPt
+ );
+
+ // Create the new transform with the updated position
+ const newTransform = {
+ segmentIndex: dupSeg,
+ pointIndex: dupPt,
+ position: newPosition,
+ };
+
+ // Update existing transform or add new one
+ if (existingTransformIndex >= 0) {
+ newTransforms[existingTransformIndex] = newTransform;
+ } else {
+ newTransforms.push(newTransform);
+ }
+ });
+
+ return newTransforms;
+}
+
+/**
+ * Finds the closest point on a line segment to a given click position.
+ *
+ * @param segmentStart - Start point of the segment
+ * @param segmentEnd - End point of the segment
+ * @param clickPosition - The position where the user clicked
+ * @returns The closest point on the segment and the parametric t value (0 to 1)
+ */
+export function findClosestPointOnSegment(
+ segmentStart: Vector3Tuple,
+ segmentEnd: Vector3Tuple,
+ clickPosition: Vector3Tuple
+): { closestPoint: Vector3Tuple; t: number } {
+ const start = new THREE.Vector3(...segmentStart);
+ const end = new THREE.Vector3(...segmentEnd);
+ const click = new THREE.Vector3(...clickPosition);
+
+ const segmentVector = new THREE.Vector3().subVectors(end, start);
+ const segmentLength = segmentVector.length();
+
+ // If segment has zero length, return the start point
+ if (segmentLength < EPS) {
+ return { closestPoint: segmentStart, t: 0 };
+ }
+
+ const clickVector = new THREE.Vector3().subVectors(click, start);
+
+ // Project click onto segment vector
+ const t = clickVector.dot(segmentVector) / (segmentLength * segmentLength);
+
+ // Clamp t to [0, 1] to stay on the segment
+ const clampedT = Math.max(0, Math.min(1, t));
+
+ const closestPoint = new THREE.Vector3()
+ .copy(start)
+ .add(segmentVector.multiplyScalar(clampedT));
+
+ return {
+ closestPoint: [closestPoint.x, closestPoint.y, closestPoint.z],
+ t: clampedT,
+ };
+}
+
+/**
+ * Inserts a new vertex into a polyline segment, splitting it into two segments.
+ * All subsequent segments have their indices incremented.
+ *
+ * @param originalPoints - Original polyline points array
+ * @param currentTransforms - Current transforms applied to the polyline
+ * @param targetSegmentIndex - Index of the segment to split
+ * @param newVertexPosition - Position of the new vertex to insert
+ * @returns New transforms array with the inserted vertex and renumbered segments, or null if the new vertex is too close to existing vertices
+ */
+export function insertVertexInSegment(
+ originalPoints: Vector3Tuple[][],
+ currentTransforms: PolylinePointTransform[],
+ targetSegmentIndex: number,
+ newVertexPosition: Vector3Tuple
+): PolylinePointTransform[] | null {
+ // Validate segment index
+ if (targetSegmentIndex < 0 || targetSegmentIndex >= originalPoints.length) {
+ throw new Error(`Invalid segment index: ${targetSegmentIndex}`);
+ }
+
+ const targetSegment = originalPoints[targetSegmentIndex];
+ if (targetSegment.length < 2) {
+ throw new Error(
+ `Segment ${targetSegmentIndex} must have at least 2 points`
+ );
+ }
+
+ // Get effective points for the target segment
+ const effectiveSegment = targetSegment.map((point, pointIndex) => {
+ const transform = currentTransforms.find(
+ (t) =>
+ t.segmentIndex === targetSegmentIndex && t.pointIndex === pointIndex
+ );
+ return transform ? transform.position : point;
+ });
+
+ // Check if the new vertex is too close to any existing vertex in the segment
+ // If so, we should not insert it to avoid redundant vertices
+ for (const existingVertex of effectiveSegment) {
+ if (positionsEqual(newVertexPosition, existingVertex)) {
+ // New vertex is too close to an existing vertex, skip insertion
+ return null;
+ }
+ }
+
+ // Step 1: Increment all segment indices >= targetSegmentIndex + 1
+ const transformsWithUpdatedIndices = currentTransforms.map((transform) => {
+ if (transform.segmentIndex > targetSegmentIndex) {
+ return {
+ ...transform,
+ segmentIndex: transform.segmentIndex + 1,
+ };
+ }
+ return transform;
+ });
+
+ // Step 2: Split the target segment
+ // Assume we are inserting a new vertex B between points A and C in segment N
+ // Original segment N: A -> C becomes:
+ // New segment N: A -> B (new vertex)
+ // New segment N+1: B (new vertex) -> C
+
+ // For segment N (keeping first point, adding new vertex as second point)
+ const segmentNTransforms: PolylinePointTransform[] = [
+ // Keep the first point of original segment as is (point 0)
+ ...transformsWithUpdatedIndices.filter(
+ (t) => t.segmentIndex === targetSegmentIndex && t.pointIndex === 0
+ ),
+ // Add new vertex as point 1 of segment N
+ {
+ segmentIndex: targetSegmentIndex,
+ pointIndex: 1,
+ position: newVertexPosition,
+ },
+ ];
+
+ // For segment N+1 (new vertex as first point, rest of original segment as subsequent points)
+ const segmentNPlus1Transforms: PolylinePointTransform[] = [
+ // New vertex as point 0 of segment N+1
+ {
+ segmentIndex: targetSegmentIndex + 1,
+ pointIndex: 0,
+ position: newVertexPosition,
+ },
+ ];
+
+ // Add the rest of the original segment's points (from point 1 onwards) to segment N+1
+ // but shifted: original point 1 becomes point 1, point 2 becomes point 2, etc.
+ for (let i = 1; i < effectiveSegment.length; i++) {
+ const originalTransform = transformsWithUpdatedIndices.find(
+ (t) => t.segmentIndex === targetSegmentIndex && t.pointIndex === i
+ );
+
+ // If there was a transform for this point, keep it but with adjusted indices
+ if (originalTransform) {
+ segmentNPlus1Transforms.push({
+ segmentIndex: targetSegmentIndex + 1,
+ pointIndex: i,
+ position: originalTransform.position,
+ });
+ } else {
+ // Otherwise, create a new transform from the effective point
+ segmentNPlus1Transforms.push({
+ segmentIndex: targetSegmentIndex + 1,
+ pointIndex: i,
+ position: effectiveSegment[i],
+ });
+ }
+ }
+
+ // Step 3: Combine all transforms
+ // - Remove old transforms for the target segment
+ // - Add new transforms for segments N and N+1
+ const finalTransforms = [
+ ...transformsWithUpdatedIndices.filter(
+ (t) => t.segmentIndex !== targetSegmentIndex
+ ),
+ ...segmentNTransforms,
+ ...segmentNPlus1Transforms,
+ ];
+
+ return finalTransforms;
+}
+
+/**
+ * Finds which segment was clicked based on the click position and effective points.
+ *
+ * @param effectivePoints - Current effective points of the polyline
+ * @param clickPosition - Position where user clicked
+ * @param maxDistance - Maximum distance to consider a click on a segment (default: 0.1)
+ * @returns Segment index and the new vertex position, or null if no segment was close enough
+ */
+export function findClickedSegment(
+ effectivePoints: Vector3Tuple[][],
+ clickPosition: Vector3Tuple,
+ maxDistance: number = 0.1
+): { segmentIndex: number; newVertexPosition: Vector3Tuple } | null {
+ let closestSegmentIndex = -1;
+ let closestDistance = Infinity;
+ let closestPoint: Vector3Tuple | null = null;
+
+ effectivePoints.forEach((segment, segmentIndex) => {
+ if (segment.length < 2) return;
+
+ // Check each line in the segment (between consecutive points)
+ for (let i = 0; i < segment.length - 1; i++) {
+ const start = segment[i];
+ const end = segment[i + 1];
+
+ const { closestPoint: pointOnSegment } = findClosestPointOnSegment(
+ start,
+ end,
+ clickPosition
+ );
+
+ const distance = new THREE.Vector3(...pointOnSegment).distanceTo(
+ new THREE.Vector3(...clickPosition)
+ );
+
+ if (distance < closestDistance) {
+ closestDistance = distance;
+ closestSegmentIndex = segmentIndex;
+ closestPoint = pointOnSegment;
+ }
+ }
+ });
+
+ // Check if the closest distance is within the threshold
+ if (closestDistance <= maxDistance && closestPoint !== null) {
+ return {
+ segmentIndex: closestSegmentIndex,
+ newVertexPosition: closestPoint,
+ };
+ }
+
+ return null;
+}
diff --git a/app/packages/looker-3d/src/assets/icons/magnet.svg b/app/packages/looker-3d/src/assets/icons/magnet.svg
new file mode 100644
index 00000000000..034d6681887
--- /dev/null
+++ b/app/packages/looker-3d/src/assets/icons/magnet.svg
@@ -0,0 +1,38 @@
+
+
+
+
+
\ No newline at end of file
diff --git a/app/packages/looker-3d/src/fo3d/components/HoveredPointMarker.tsx b/app/packages/looker-3d/src/components/HoveredPointMarker.tsx
similarity index 100%
rename from app/packages/looker-3d/src/fo3d/components/HoveredPointMarker.tsx
rename to app/packages/looker-3d/src/components/HoveredPointMarker.tsx
diff --git a/app/packages/looker-3d/src/constants.ts b/app/packages/looker-3d/src/constants.ts
index 306d34e5fe1..fdb75bc0f16 100644
--- a/app/packages/looker-3d/src/constants.ts
+++ b/app/packages/looker-3d/src/constants.ts
@@ -1,5 +1,5 @@
import { ColorscaleInput } from "@fiftyone/looker/src/state";
-import { Vector3 } from "three";
+import { Box3, Vector3 } from "three";
import type { ShadeBy } from "./types";
export const ACTION_GRID = "grid";
@@ -23,6 +23,14 @@ export const SHADE_BY_NONE = "none";
export const DEFAULT_CAMERA_POSITION = () => new Vector3(0, 5, -5);
+// Default bounding box when scene bounds cannot be determined
+export const DEFAULT_BOUNDING_BOX = new Box3(
+ // min
+ new Vector3(-5, -5, -5),
+ // max
+ new Vector3(5, 5, 5)
+);
+
export const ACTIONS = [
{ label: "Color By", value: ACTION_SHADE_BY },
{ label: "Set Point Size", value: ACTION_SET_POINT_SIZE },
@@ -82,6 +90,9 @@ export const LABEL_3D_HOVERED_AND_SELECTED_COLOR = "#de7e5d";
export const LABEL_3D_HOVERED_COLOR = "#f2f0ef";
export const LABEL_3D_INSTANCE_HOVERED_COLOR = "#ffffff";
export const LABEL_3D_SIMILAR_SELECTED_COLOR = "#ffa500";
+export const LABEL_3D_SELECTED_FOR_ANNOTATION_COLOR = "orangered";
+export const LABEL_3D_ANNOTATION_POINT_SELECTED_FOR_TRANSFORMATION_COLOR =
+ "#ff0000";
// ray casting for points
export const RAY_CASTING_SENSITIVITY = {
@@ -97,3 +108,7 @@ export const RAY_CASTING_SENSITIVITY = {
// Useful for sparse clouds, large points, or user-friendly selection modes.
low: 0.25,
};
+
+export const SNAP_TOLERANCE = 0.5;
+
+export const SCENE_BOUNDS_EXPANSION_FACTOR = 5;
diff --git a/app/packages/looker-3d/src/fo3d/Fo3dCanvas.tsx b/app/packages/looker-3d/src/fo3d/Fo3dCanvas.tsx
new file mode 100644
index 00000000000..238575dab5f
--- /dev/null
+++ b/app/packages/looker-3d/src/fo3d/Fo3dCanvas.tsx
@@ -0,0 +1,167 @@
+import * as fos from "@fiftyone/state";
+import {
+ AdaptiveDpr,
+ AdaptiveEvents,
+ Bvh,
+ CameraControls,
+ OrbitControls,
+ PerspectiveCamera as PerspectiveCameraDrei,
+} from "@react-three/drei";
+import { useAtomValue } from "jotai";
+import * as THREE from "three";
+import { Vector3 } from "three";
+import { SpinningCube } from "../SpinningCube";
+import { StatusTunnel } from "../StatusBar";
+import { AnnotationPlane } from "../annotation/AnnotationPlane";
+import { SegmentPolylineRenderer } from "../annotation/SegmentPolylineRenderer";
+import { ThreeDLabels } from "../labels";
+import { FoSceneComponent } from "./FoScene";
+import { Gizmos } from "./Gizmos";
+import { SceneControls } from "./scene-controls/SceneControls";
+
+interface Fo3dSceneContentProps {
+ /**
+ * Camera position
+ */
+ cameraPosition: Vector3;
+ /**
+ * Up vector for the camera
+ */
+ upVector: Vector3 | null;
+ /**
+ * Camera field of view
+ */
+ fov?: number;
+ /**
+ * Camera near plane
+ */
+ near?: number;
+ /**
+ * Camera far plane
+ */
+ far?: number;
+ /**
+ * Camera aspect ratio
+ */
+ aspect?: number;
+ /**
+ * Camera zoom (for orthographic)
+ */
+ zoom?: number;
+ /**
+ * Whether auto-rotate is enabled
+ */
+ autoRotate: boolean;
+ /**
+ * Reference to camera controls
+ */
+ cameraControlsRef: React.RefObject;
+ /**
+ * The 3D scene to render
+ */
+ foScene: any;
+ /**
+ * Whether the scene is initialized
+ */
+ isSceneInitialized: boolean;
+ /**
+ * Sample data for labels
+ */
+ sample: any;
+ /**
+ * Point cloud settings
+ */
+ pointCloudSettings: any;
+ /**
+ * Reference to the assets group
+ */
+ assetsGroupRef: React.RefObject;
+ /**
+ * Reference to camera
+ */
+ cameraRef?: React.RefObject;
+ /**
+ * Whether or not to render gizmo helper.
+ */
+ isGizmoHelperVisible?: boolean;
+}
+
+export const Fo3dSceneContent = ({
+ cameraPosition,
+ upVector,
+ fov = 50,
+ near = 0.1,
+ far = 2500,
+ aspect = 1,
+ zoom = 100,
+ autoRotate,
+ cameraControlsRef,
+ foScene,
+ isSceneInitialized,
+ isGizmoHelperVisible,
+ sample,
+ pointCloudSettings,
+ assetsGroupRef,
+ cameraRef,
+}: Fo3dSceneContentProps) => {
+ const mode = useAtomValue(fos.modalMode);
+
+ return (
+ <>
+
+
+
+
+ }
+ position={cameraPosition}
+ up={upVector ?? [0, 1, 0]}
+ fov={foScene?.cameraProps.fov ?? fov}
+ near={foScene?.cameraProps.near ?? near}
+ far={foScene?.cameraProps.far ?? far}
+ aspect={foScene?.cameraProps.aspect ?? aspect}
+ onUpdate={(cam) => cam.updateProjectionMatrix()}
+ />
+
+ {!autoRotate ? (
+
+ ) : (
+
+ )}
+
+
+
+
+ {!isSceneInitialized && }
+
+
+
+
+
+
+ {isSceneInitialized && (
+
+ )}
+
+ {mode === "annotate" && }
+
+ );
+};
+
+const AnnotationControls = () => {
+ return (
+ <>
+
+
+
+ );
+};
diff --git a/app/packages/looker-3d/src/fo3d/Gizmos.tsx b/app/packages/looker-3d/src/fo3d/Gizmos.tsx
index 75bb486ced8..ff8cc1f7118 100644
--- a/app/packages/looker-3d/src/fo3d/Gizmos.tsx
+++ b/app/packages/looker-3d/src/fo3d/Gizmos.tsx
@@ -80,7 +80,13 @@ const FoAxesHelper = ({
return <>{axisComponents};
};
-export const Gizmos = () => {
+export const Gizmos = ({
+ isGizmoHelperVisible,
+ isGridVisible,
+}: {
+ isGizmoHelperVisible: boolean;
+ isGridVisible: boolean;
+}) => {
const { upVector, sceneBoundingBox } = useFo3dContext();
const isGridOn = useRecoilValue(isGridOnAtom);
@@ -146,7 +152,7 @@ export const Gizmos = () => {
return (
<>
- {isGridOn && (
+ {isGridOn && isGridVisible && (
<>
{
)}
-
-
-
+ {isGizmoHelperVisible && (
+
+
+
+ )}
);
};
diff --git a/app/packages/looker-3d/src/fo3d/MediaTypeFo3d.tsx b/app/packages/looker-3d/src/fo3d/MediaTypeFo3d.tsx
index 1540beecd1a..4c75109c69a 100644
--- a/app/packages/looker-3d/src/fo3d/MediaTypeFo3d.tsx
+++ b/app/packages/looker-3d/src/fo3d/MediaTypeFo3d.tsx
@@ -1,17 +1,11 @@
import { LoadingDots } from "@fiftyone/components";
import { usePluginSettings } from "@fiftyone/plugins";
import * as fos from "@fiftyone/state";
-import { useBrowserStorage } from "@fiftyone/state";
-import {
- AdaptiveDpr,
- AdaptiveEvents,
- Bvh,
- CameraControls,
- OrbitControls,
- PerspectiveCamera as PerspectiveCameraDrei,
-} from "@react-three/drei";
+import { isInMultiPanelViewAtom, useBrowserStorage } from "@fiftyone/state";
+import { CameraControls } from "@react-three/drei";
import { Canvas } from "@react-three/fiber";
import CameraControlsImpl from "camera-controls";
+import { useAtomValue } from "jotai";
import {
useCallback,
useEffect,
@@ -20,37 +14,42 @@ import {
useRef,
useState,
} from "react";
-import { useRecoilCallback, useRecoilValue } from "recoil";
+import { useRecoilCallback, useRecoilValue, useSetRecoilState } from "recoil";
+import styled from "styled-components";
import * as THREE from "three";
import { Vector3 } from "three";
-import { CAMERA_POSITION_KEY } from "../Environment";
-import { SpinningCube } from "../SpinningCube";
-import { StatusBar, StatusTunnel } from "../StatusBar";
+import { StatusBar } from "../StatusBar";
+import { MultiPanelView } from "../annotation/MultiPanelView";
+import { AnnotationToolbar } from "../annotation/annotation-toolbar/AnnotationToolbar";
import { PcdColorMapTunnel } from "../components/PcdColormapModal";
import {
+ DEFAULT_BOUNDING_BOX,
DEFAULT_CAMERA_POSITION,
- RAY_CASTING_SENSITIVITY,
SET_EGO_VIEW_EVENT,
SET_TOP_VIEW_EVENT,
} from "../constants";
import { StatusBarRootContainer } from "../containers";
import { useFo3d, useHotkey, useTrackStatus } from "../hooks";
import { useFo3dBounds } from "../hooks/use-bounds";
-import { ThreeDLabels } from "../labels";
import type { Looker3dSettings } from "../settings";
import {
activeNodeAtom,
+ annotationPlaneAtom,
cameraPositionAtom,
+ clearTransformStateSelector,
currentHoveredPointAtom,
+ isActivelySegmentingSelector,
+ isCurrentlyTransformingAtom,
isFo3dBackgroundOnAtom,
+ isSegmentingPointerDownAtom,
+ selectedPolylineVertexAtom,
} from "../state";
import { HoverMetadata } from "../types";
-import { FoSceneComponent } from "./FoScene";
-import { Gizmos } from "./Gizmos";
+import { Fo3dSceneContent } from "./Fo3dCanvas";
import HoverMetadataHUD from "./HoverMetadataHUD";
import { Fo3dSceneContext } from "./context";
-import { SceneControls } from "./scene-controls/SceneControls";
import {
+ getCameraPositionKey,
getFo3dRoot,
getMediaPathForFo3dSample,
getOrthonormalAxis,
@@ -58,6 +57,13 @@ import {
const CANVAS_WRAPPER_ID = "sample3d-canvas-wrapper";
+const MainContainer = styled.main`
+ display: flex;
+ flex-direction: column;
+ height: 100%;
+ width: 100%;
+`;
+
const calculateCameraPositionForUpVector = (
center: Vector3,
size: Vector3,
@@ -119,6 +125,9 @@ const calculateCameraPositionForUpVector = (
export const MediaTypeFo3dComponent = () => {
const sample = useRecoilValue(fos.fo3dSample);
const mediaField = useRecoilValue(fos.selectedMediaField(true));
+ const is2DSampleViewerVisible = useRecoilValue(fos.groupMediaIsMainVisible);
+ const isGroup = useRecoilValue(fos.isGroup);
+ const setIsInMultiPanelView = useSetRecoilState(isInMultiPanelViewAtom);
const settings = usePluginSettings("3d");
@@ -127,6 +136,8 @@ export const MediaTypeFo3dComponent = () => {
[mediaField, sample]
);
+ const mode = useAtomValue(fos.modalMode);
+
const mediaUrl = useMemo(() => fos.getSampleSrc(mediaPath), [mediaPath]);
const fo3dRoot = useMemo(() => getFo3dRoot(sample.sample.filepath), [sample]);
@@ -209,6 +220,9 @@ export const MediaTypeFo3dComponent = () => {
}
);
+ // todo: reconcile with lookAt from foScene, too
+ const [lookAt, setLookAt] = useState(null);
+
useEffect(() => {
if (!foScene || upVector) {
return;
@@ -217,8 +231,13 @@ export const MediaTypeFo3dComponent = () => {
setUpVectorVal(getDefaultUpVector());
}, [foScene, upVector, getDefaultUpVector]);
+ const containerRef = useRef(null);
const cameraRef = useRef();
const cameraControlsRef = useRef();
+ const datasetName = useRecoilValue(fos.datasetName);
+ const isActivelySegmenting = useRecoilValue(isActivelySegmentingSelector);
+ const isSegmentingPointerDown = useRecoilValue(isSegmentingPointerDownAtom);
+ const isCurrentlyTransforming = useRecoilValue(isCurrentlyTransformingAtom);
const keyState = useRef({
shiftRight: false,
@@ -228,6 +247,17 @@ export const MediaTypeFo3dComponent = () => {
});
const updateCameraControlsConfig = useCallback(() => {
+ if (!cameraControlsRef.current) return;
+
+ // Disable camera controls when transforming
+ if (isSegmentingPointerDown || isCurrentlyTransforming) {
+ cameraControlsRef.current.enabled = false;
+ return;
+ }
+
+ // Re-enable camera controls when not transforming
+ cameraControlsRef.current.enabled = true;
+
if (keyState.current.shiftRight || keyState.current.shiftLeft) {
cameraControlsRef.current.mouseButtons.left =
CameraControlsImpl.ACTION.TRUCK;
@@ -238,7 +268,14 @@ export const MediaTypeFo3dComponent = () => {
cameraControlsRef.current.mouseButtons.left =
CameraControlsImpl.ACTION.ROTATE;
}
- }, [keyState]);
+ }, [keyState, isCurrentlyTransforming, isSegmentingPointerDown]);
+
+ /**
+ * This effect updates the camera controls config when the transforming state changes
+ */
+ useEffect(() => {
+ updateCameraControlsConfig();
+ }, [updateCameraControlsConfig]);
fos.useEventHandler(document, "keydown", (e: KeyboardEvent) => {
if (e.code === "ShiftRight") keyState.current.shiftRight = true;
@@ -257,18 +294,32 @@ export const MediaTypeFo3dComponent = () => {
});
const assetsGroupRef = useRef();
- const sceneBoundingBox = useFo3dBounds(assetsGroupRef);
+
+ const {
+ boundingBox: sceneBoundingBox,
+ recomputeBounds,
+ isComputing: isComputingSceneBoundingBox,
+ } = useFo3dBounds(assetsGroupRef);
+
+ const effectiveSceneBoundingBox = sceneBoundingBox || DEFAULT_BOUNDING_BOX;
+
+ useEffect(() => {
+ if (sceneBoundingBox && !lookAt) {
+ const center = effectiveSceneBoundingBox.getCenter(new Vector3());
+ setLookAt(center);
+ }
+ }, [sceneBoundingBox, lookAt, effectiveSceneBoundingBox]);
const topCameraPosition = useMemo(() => {
if (
!sceneBoundingBox ||
- Math.abs(sceneBoundingBox.max.x) === Number.POSITIVE_INFINITY
+ Math.abs(effectiveSceneBoundingBox.max.x) === Number.POSITIVE_INFINITY
) {
return DEFAULT_CAMERA_POSITION();
}
- const center = sceneBoundingBox.getCenter(new Vector3());
- const size = sceneBoundingBox.getSize(new Vector3());
+ const center = effectiveSceneBoundingBox.getCenter(new Vector3());
+ const size = effectiveSceneBoundingBox.getSize(new Vector3());
return calculateCameraPositionForUpVector(
center,
@@ -277,16 +328,17 @@ export const MediaTypeFo3dComponent = () => {
2.5,
"top"
);
- }, [sceneBoundingBox, upVector]);
+ }, [sceneBoundingBox, upVector, effectiveSceneBoundingBox]);
const overriddenCameraPosition = useRecoilValue(cameraPositionAtom);
const lastSavedCameraPosition = useMemo(() => {
- const lastSavedCameraPosition =
- window?.localStorage.getItem(CAMERA_POSITION_KEY);
+ const lastSavedCameraPosition = window?.localStorage.getItem(
+ getCameraPositionKey(datasetName)
+ );
return lastSavedCameraPosition ? JSON.parse(lastSavedCameraPosition) : null;
- }, []);
+ }, [datasetName]);
const getDefaultCameraPosition = useCallback(
(ignoreLastSavedCameraPosition = false) => {
@@ -344,9 +396,9 @@ export const MediaTypeFo3dComponent = () => {
if (
sceneBoundingBox &&
- Math.abs(sceneBoundingBox.max.x) !== Number.POSITIVE_INFINITY
+ Math.abs(effectiveSceneBoundingBox.max.x) !== Number.POSITIVE_INFINITY
) {
- const size = sceneBoundingBox.getSize(new Vector3());
+ const size = effectiveSceneBoundingBox.getSize(new Vector3());
return calculateCameraPositionForUpVector(
new Vector3(0, 0, 0),
@@ -365,6 +417,7 @@ export const MediaTypeFo3dComponent = () => {
isParsingFo3d,
foScene,
sceneBoundingBox,
+ effectiveSceneBoundingBox,
upVector,
lastSavedCameraPosition,
]
@@ -377,12 +430,24 @@ export const MediaTypeFo3dComponent = () => {
const resetActiveNode = useRecoilCallback(
({ set }) =>
- () => {
+ (event: MouseEvent | null) => {
+ // Don't handle right click since that might mean we're panning the camera
+ if (event?.type === "contextmenu") {
+ return;
+ }
+
+ if (isActivelySegmenting) {
+ return;
+ }
+
set(activeNodeAtom, null);
set(currentHoveredPointAtom, null);
+ set(clearTransformStateSelector, null);
+ set(selectedPolylineVertexAtom, null);
+ set(isCurrentlyTransformingAtom, false);
setAutoRotate(false);
},
- []
+ [isActivelySegmenting]
);
useLayoutEffect(() => {
@@ -394,7 +459,7 @@ export const MediaTypeFo3dComponent = () => {
}, [isSceneInitialized]);
useEffect(() => {
- resetActiveNode();
+ resetActiveNode(null);
}, [isSceneInitialized, resetActiveNode]);
const onChangeView = useCallback(
@@ -441,7 +506,7 @@ export const MediaTypeFo3dComponent = () => {
];
// for top view, we have look at at center of bounding box
- const center = sceneBoundingBox.getCenter(new Vector3());
+ const center = effectiveSceneBoundingBox.getCenter(new Vector3());
newLookAt = [center.x, center.y, center.z] as const;
}
@@ -458,6 +523,7 @@ export const MediaTypeFo3dComponent = () => {
},
[
sceneBoundingBox,
+ effectiveSceneBoundingBox,
topCameraPosition,
getDefaultCameraPosition,
setSceneInitialized,
@@ -608,16 +674,21 @@ export const MediaTypeFo3dComponent = () => {
}
);
- // this effect runs after the scene is initialized
- // and sets the appropriate lookAt and camera position
+ // this effect sets the appropriate lookAt and camera position
+ // and marks the scene as initialized
useEffect(() => {
- if (!cameraControlsRef.current || !cameraRef.current) {
+ if (
+ !cameraControlsRef.current ||
+ !cameraRef.current ||
+ isComputingSceneBoundingBox
+ ) {
return;
}
// restore camera position and target from localStorage if it exists
- const lastSavedCameraState =
- window?.localStorage.getItem(CAMERA_POSITION_KEY);
+ const lastSavedCameraState = window?.localStorage.getItem(
+ getCameraPositionKey(datasetName)
+ );
let restored = false;
if (lastSavedCameraState) {
try {
@@ -655,14 +726,21 @@ export const MediaTypeFo3dComponent = () => {
setSceneInitialized(true);
return;
} else {
- onChangeView("pov", {
+ onChangeView("top", {
useAnimation: false,
ignoreLastSavedCameraPosition: false,
isFirstTime: true,
});
+ setSceneInitialized(true);
}
}
- }, [foScene, onChangeView, cameraControlsRef, cameraRef]);
+ }, [
+ foScene,
+ onChangeView,
+ cameraControlsRef,
+ cameraRef,
+ isComputingSceneBoundingBox,
+ ]);
useTrackStatus();
@@ -687,6 +765,26 @@ export const MediaTypeFo3dComponent = () => {
null
);
+ const isAnnotationPlaneEnabled = useRecoilValue(annotationPlaneAtom).enabled;
+
+ const shouldRenderMultiPanelView = useMemo(
+ () =>
+ mode === "annotate" &&
+ !(isGroup && is2DSampleViewerVisible) &&
+ isSceneInitialized,
+ [mode, isGroup, is2DSampleViewerVisible, isSceneInitialized]
+ );
+
+ useEffect(() => {
+ if (shouldRenderMultiPanelView) {
+ recomputeBounds();
+ }
+ }, [shouldRenderMultiPanelView, isAnnotationPlaneEnabled, recomputeBounds]);
+
+ useEffect(() => {
+ setIsInMultiPanelView(shouldRenderMultiPanelView);
+ }, [shouldRenderMultiPanelView]);
+
if (isParsingFo3d) {
return ;
}
@@ -698,8 +796,11 @@ export const MediaTypeFo3dComponent = () => {
numPrimaryAssets,
upVector,
setUpVector,
+ isComputingSceneBoundingBox,
fo3dRoot,
- sceneBoundingBox,
+ sceneBoundingBox: effectiveSceneBoundingBox,
+ lookAt,
+ setLookAt,
autoRotate,
setAutoRotate,
pointCloudSettings,
@@ -709,55 +810,50 @@ export const MediaTypeFo3dComponent = () => {
pluginSettings: settings,
}}
>
-
-
-
-
-
-
+ ) : (
+
+
+
+
+
+
+
+
+ )}
+ {mode === "annotate" && }
);
};
diff --git a/app/packages/looker-3d/src/fo3d/context.tsx b/app/packages/looker-3d/src/fo3d/context.tsx
index 3e213dd3f6a..03a54423582 100644
--- a/app/packages/looker-3d/src/fo3d/context.tsx
+++ b/app/packages/looker-3d/src/fo3d/context.tsx
@@ -13,7 +13,10 @@ interface Fo3dContextT {
numPrimaryAssets: number;
upVector: Vector3 | null;
setUpVector: (upVector: Vector3) => void;
+ isComputingSceneBoundingBox: boolean;
sceneBoundingBox: Box3 | null;
+ lookAt: Vector3 | null;
+ setLookAt: (lookAt: Vector3) => void;
pluginSettings: Looker3dSettings | null;
fo3dRoot: string | null;
autoRotate: boolean;
@@ -29,7 +32,10 @@ const defaultContext: Fo3dContextT = {
numPrimaryAssets: 0,
upVector: null,
setUpVector: () => {},
+ isComputingSceneBoundingBox: false,
sceneBoundingBox: null,
+ lookAt: null,
+ setLookAt: () => {},
pluginSettings: null,
fo3dRoot: null,
autoRotate: false,
diff --git a/app/packages/looker-3d/src/fo3d/mesh/Fbx.tsx b/app/packages/looker-3d/src/fo3d/mesh/Fbx.tsx
index 8fb8d90dd5f..a45b3d15284 100644
--- a/app/packages/looker-3d/src/fo3d/mesh/Fbx.tsx
+++ b/app/packages/looker-3d/src/fo3d/mesh/Fbx.tsx
@@ -1,15 +1,16 @@
-import { getSampleSrc } from "@fiftyone/state";
-import { useLoader } from "@react-three/fiber";
+import { getSampleSrc, isInMultiPanelViewAtom } from "@fiftyone/state";
import { useMemo } from "react";
+import { useRecoilValue } from "recoil";
import { AnimationMixer, type Quaternion, type Vector3 } from "three";
import { FBXLoader } from "three-stdlib";
import type { FbxAsset } from "../../hooks";
import { useAnimationSelect } from "../../hooks/use-animation-select";
+import { useFoLoader } from "../../hooks/use-fo-loaders";
import { useMeshMaterialControls } from "../../hooks/use-mesh-material-controls";
import { usePercolateMaterial } from "../../hooks/use-set-scene-transparency";
import { useFo3dContext } from "../context";
import { getBasePathForTextures, getResolvedUrlForFo3dAsset } from "../utils";
-import { useFoLoader } from "../../hooks/use-fo-loaders";
+import { SkeletonUtils } from "three-stdlib";
export const Fbx = ({
name,
@@ -27,6 +28,7 @@ export const Fbx = ({
children: React.ReactNode;
}) => {
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const fbxUrl = useMemo(
() =>
@@ -42,10 +44,20 @@ export const Fbx = ({
const { material } = useMeshMaterialControls(name, defaultMaterial, true);
- const fbx = useFoLoader(FBXLoader, fbxUrl, (loader) => {
+ const fbx_ = useFoLoader(FBXLoader, fbxUrl, (loader) => {
loader.setResourcePath(resourcePath);
});
+ // Deep clone mesh when in multipanel view to avoid React Three Fiber caching issues
+ // todo: optimize this with instanced mesh
+ const fbx = useMemo(() => {
+ if (isInMultiPanelView && fbx_) {
+ // Use SkeletonUtils otherwise skeletons might de-bind
+ return SkeletonUtils.clone(fbx_);
+ }
+ return fbx_;
+ }, [fbx_, isInMultiPanelView]);
+
usePercolateMaterial(fbx, material);
const animationClips = useMemo(() => {
diff --git a/app/packages/looker-3d/src/fo3d/mesh/Gltf.tsx b/app/packages/looker-3d/src/fo3d/mesh/Gltf.tsx
index 8386f873e46..332563bf3e6 100644
--- a/app/packages/looker-3d/src/fo3d/mesh/Gltf.tsx
+++ b/app/packages/looker-3d/src/fo3d/mesh/Gltf.tsx
@@ -1,7 +1,9 @@
-import { getSampleSrc } from "@fiftyone/state";
+import { getSampleSrc, isInMultiPanelViewAtom } from "@fiftyone/state";
import { useGLTF } from "@react-three/drei";
-import { useEffect, useMemo, useRef } from "react";
-import { AnimationMixer, Mesh, type Quaternion, type Vector3 } from "three";
+import { useMemo, useRef } from "react";
+import { useRecoilValue } from "recoil";
+import { AnimationMixer, type Quaternion, type Vector3 } from "three";
+import { SkeletonUtils } from "three-stdlib";
import type { GltfAsset } from "../../hooks";
import { useAnimationSelect } from "../../hooks/use-animation-select";
import { useMeshMaterialControls } from "../../hooks/use-mesh-material-controls";
@@ -25,6 +27,7 @@ export const Gltf = ({
children: React.ReactNode;
}) => {
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const gltfUrl = useMemo(
() =>
@@ -40,9 +43,24 @@ export const Gltf = ({
const { material } = useMeshMaterialControls(name, defaultMaterial, true);
- const { scene, animations } = useGLTF(gltfUrl, true, undefined, (loader) => {
- loader.setResourcePath(resourcePath);
- });
+ const { scene: scene_, animations } = useGLTF(
+ gltfUrl,
+ true,
+ undefined,
+ (loader) => {
+ loader.setResourcePath(resourcePath);
+ }
+ );
+
+ // Deep clone scene when in multipanel view to avoid React Three Fiber caching issues
+ // todo: optimize this with instanced mesh
+ const scene = useMemo(() => {
+ if (isInMultiPanelView && scene_) {
+ // Use SkeletonUtils otherwise skeletons might de-bind
+ return SkeletonUtils.clone(scene_);
+ }
+ return scene_;
+ }, [scene_, isInMultiPanelView]);
usePercolateMaterial(scene, material);
diff --git a/app/packages/looker-3d/src/fo3d/mesh/Obj.tsx b/app/packages/looker-3d/src/fo3d/mesh/Obj.tsx
index 2373973b4fd..1ed5c84b999 100644
--- a/app/packages/looker-3d/src/fo3d/mesh/Obj.tsx
+++ b/app/packages/looker-3d/src/fo3d/mesh/Obj.tsx
@@ -1,6 +1,6 @@
-import { getSampleSrc } from "@fiftyone/state";
-import { useLoader } from "@react-three/fiber";
+import { getSampleSrc, isInMultiPanelViewAtom } from "@fiftyone/state";
import { useEffect, useMemo } from "react";
+import { useRecoilValue } from "recoil";
import {
Mesh,
MeshStandardMaterial,
@@ -10,11 +10,11 @@ import {
import { MTLLoader } from "three/examples/jsm/loaders/MTLLoader";
import { OBJLoader } from "three/examples/jsm/loaders/OBJLoader";
import type { ObjAsset } from "../../hooks";
+import { useFoLoader } from "../../hooks/use-fo-loaders";
import { useMeshMaterialControls } from "../../hooks/use-mesh-material-controls";
import { getColorFromPoolBasedOnHash } from "../../utils";
import { useFo3dContext } from "../context";
import { getBasePathForTextures, getResolvedUrlForFo3dAsset } from "../utils";
-import { useFoLoader } from "../../hooks/use-fo-loaders";
const ObjMeshDefaultMaterial = ({
name,
@@ -28,6 +28,7 @@ const ObjMeshDefaultMaterial = ({
const { objPath, preTransformedObjPath } = obj;
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const objUrl = useMemo(
() =>
@@ -36,7 +37,16 @@ const ObjMeshDefaultMaterial = ({
[objPath, preTransformedObjPath, fo3dRoot]
);
- const mesh = useFoLoader(OBJLoader, objUrl);
+ const mesh_ = useFoLoader(OBJLoader, objUrl);
+
+ // Deep clone mesh when in multipanel view to avoid React Three Fiber caching issues
+ // todo: optimize this with instanced mesh
+ const mesh = useMemo(() => {
+ if (isInMultiPanelView && mesh_) {
+ return mesh_.clone(true);
+ }
+ return mesh_;
+ }, [mesh_, isInMultiPanelView]);
const { material } = useMeshMaterialControls(name, obj.defaultMaterial);
@@ -60,10 +70,15 @@ const ObjMeshDefaultMaterial = ({
onLoad?.();
}, [mesh, objUrl, material, onLoad]);
+ if (!mesh) {
+ return null;
+ }
+
return ;
};
const ObjMeshWithCustomMaterial = ({
+ name,
obj,
onLoad,
}: {
@@ -75,6 +90,7 @@ const ObjMeshWithCustomMaterial = ({
obj;
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const objUrl = useMemo(
() =>
@@ -100,19 +116,32 @@ const ObjMeshWithCustomMaterial = ({
loader.setResourcePath(resourcePath);
}
});
- const mesh = useFoLoader(OBJLoader, objUrl, (loader) => {
+ const mesh_ = useFoLoader(OBJLoader, objUrl, (loader) => {
if (mtlUrl) {
materials.preload();
loader.setMaterials(materials);
}
});
+ // Deep clone mesh when in multipanel view to avoid React Three Fiber caching issues
+ // todo: optimize this with instanced mesh
+ const mesh = useMemo(() => {
+ if (isInMultiPanelView && mesh_) {
+ return mesh_.clone(true);
+ }
+ return mesh_;
+ }, [mesh_, isInMultiPanelView]);
+
useEffect(() => {
if (mesh) {
onLoad?.();
}
}, [mesh, onLoad]);
+ if (!mesh) {
+ return null;
+ }
+
return ;
};
diff --git a/app/packages/looker-3d/src/fo3d/mesh/Ply.tsx b/app/packages/looker-3d/src/fo3d/mesh/Ply.tsx
index 323071505ed..b36206ef51a 100644
--- a/app/packages/looker-3d/src/fo3d/mesh/Ply.tsx
+++ b/app/packages/looker-3d/src/fo3d/mesh/Ply.tsx
@@ -1,5 +1,6 @@
-import { getSampleSrc } from "@fiftyone/state";
+import { getSampleSrc, isInMultiPanelViewAtom } from "@fiftyone/state";
import { useEffect, useMemo, useRef, useState } from "react";
+import { useRecoilValue } from "recoil";
import {
type BufferGeometry,
Mesh,
@@ -8,6 +9,7 @@ import {
Vector3,
} from "three";
import { PLYLoader } from "three/examples/jsm/loaders/PLYLoader";
+import { HoveredPointMarker } from "../../components/HoveredPointMarker";
import type {
FoMeshBasicMaterialProps,
FoMeshMaterial,
@@ -17,7 +19,6 @@ import type {
import { useFoLoader } from "../../hooks/use-fo-loaders";
import { useMeshMaterialControls } from "../../hooks/use-mesh-material-controls";
import { usePointCloudHover } from "../../hooks/use-point-cloud-hover";
-import { HoveredPointMarker } from "../components/HoveredPointMarker";
import { useFo3dContext } from "../context";
import { usePcdMaterial } from "../point-cloud/use-pcd-material";
import { getBasePathForTextures, getResolvedUrlForFo3dAsset } from "../utils";
@@ -172,6 +173,7 @@ export const Ply = ({
children,
}: PlyProps) => {
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const plyUrl = useMemo(
() =>
@@ -185,10 +187,18 @@ export const Ply = ({
[fo3dRoot, plyUrl]
);
- const geometry = useFoLoader(PLYLoader, plyUrl, (loader) => {
+ const geometry_ = useFoLoader(PLYLoader, plyUrl, (loader) => {
loader.resourcePath = resourcePath;
});
+ // Clone geometry when in multipanel view to avoid React Three Fiber caching issues
+ const geometry = useMemo(() => {
+ if (isInMultiPanelView && geometry_) {
+ return geometry_.clone();
+ }
+ return geometry_;
+ }, [geometry_, isInMultiPanelView]);
+
const [isUsingVertexColors, setIsUsingVertexColors] = useState(false);
const [isGeometryResolved, setIsGeometryResolved] = useState(false);
diff --git a/app/packages/looker-3d/src/fo3d/mesh/Stl.tsx b/app/packages/looker-3d/src/fo3d/mesh/Stl.tsx
index 10b6e6dde47..fb88ca68ef4 100644
--- a/app/packages/looker-3d/src/fo3d/mesh/Stl.tsx
+++ b/app/packages/looker-3d/src/fo3d/mesh/Stl.tsx
@@ -1,13 +1,13 @@
-import { getSampleSrc } from "@fiftyone/state";
-import { useLoader } from "@react-three/fiber";
+import { getSampleSrc, isInMultiPanelViewAtom } from "@fiftyone/state";
import { useEffect, useMemo, useState } from "react";
+import { useRecoilValue } from "recoil";
import { Mesh, type Quaternion, type Vector3 } from "three";
import { STLLoader } from "three/examples/jsm/loaders/STLLoader";
import type { StlAsset } from "../../hooks";
+import { useFoLoader } from "../../hooks/use-fo-loaders";
import { useMeshMaterialControls } from "../../hooks/use-mesh-material-controls";
import { useFo3dContext } from "../context";
import { getResolvedUrlForFo3dAsset } from "../utils";
-import { useFoLoader } from "../../hooks/use-fo-loaders";
/**
* Renders a single STL mesh.
@@ -31,6 +31,7 @@ export const Stl = ({
children?: React.ReactNode;
}) => {
const { fo3dRoot } = useFo3dContext();
+ const isInMultiPanelView = useRecoilValue(isInMultiPanelViewAtom);
const stlUrl = useMemo(
() =>
@@ -39,7 +40,16 @@ export const Stl = ({
[stlPath, preTransformedStlPath, fo3dRoot]
);
- const points = useFoLoader(STLLoader, stlUrl);
+ const points_ = useFoLoader(STLLoader, stlUrl);
+
+ // Clone points when in multipanel view to avoid React Three Fiber caching issues
+ const points = useMemo(() => {
+ if (isInMultiPanelView && points_) {
+ return points_.clone();
+ }
+ return points_;
+ }, [points_, isInMultiPanelView]);
+
const [mesh, setMesh] = useState(null);
const { material } = useMeshMaterialControls(name, defaultMaterial);
diff --git a/app/packages/looker-3d/src/fo3d/point-cloud/Pcd.tsx b/app/packages/looker-3d/src/fo3d/point-cloud/Pcd.tsx
index cad9bdfc1e5..97aca448a3f 100644
--- a/app/packages/looker-3d/src/fo3d/point-cloud/Pcd.tsx
+++ b/app/packages/looker-3d/src/fo3d/point-cloud/Pcd.tsx
@@ -9,7 +9,7 @@ import type { PcdAsset } from "../../hooks";
import { useFoLoader } from "../../hooks/use-fo-loaders";
import { usePointCloudHover } from "../../hooks/use-point-cloud-hover";
import { DynamicPCDLoader } from "../../loaders/dynamic-pcd-loader";
-import { HoveredPointMarker } from "../components/HoveredPointMarker";
+import { HoveredPointMarker } from "../../components/HoveredPointMarker";
import { useFo3dContext } from "../context";
import { getResolvedUrlForFo3dAsset } from "../utils";
import { usePcdMaterial } from "./use-pcd-material";
diff --git a/app/packages/looker-3d/src/fo3d/point-cloud/use-pcd-material.tsx b/app/packages/looker-3d/src/fo3d/point-cloud/use-pcd-material.tsx
index 61eeee0a953..71dc8fec044 100644
--- a/app/packages/looker-3d/src/fo3d/point-cloud/use-pcd-material.tsx
+++ b/app/packages/looker-3d/src/fo3d/point-cloud/use-pcd-material.tsx
@@ -1,6 +1,7 @@
import { useCallback, useMemo } from "react";
import type { BufferGeometry, Quaternion } from "three";
import {
+ DEFAULT_BOUNDING_BOX,
SHADE_BY_CUSTOM,
SHADE_BY_HEIGHT,
SHADE_BY_INTENSITY,
@@ -64,7 +65,7 @@ export const usePcdMaterial = (
) => {
const { upVector, pluginSettings } = useFo3dContext();
- const pcdBoundingBox = useFo3dBounds(
+ const { boundingBox: pcdBoundingBox } = useFo3dBounds(
pcdContainerRef,
useCallback(() => {
return !!geometry;
@@ -72,17 +73,18 @@ export const usePcdMaterial = (
);
const minMaxCoordinates = useMemo(() => {
- if (!pcdBoundingBox) {
- return null;
- }
+ const effectiveBoundingBox = pcdBoundingBox || DEFAULT_BOUNDING_BOX;
// note: we should deprecate minZ in the plugin settings, since it doesn't account for non-z up vectors
- if (pluginSettings?.pointCloud?.minZ) {
- return [pluginSettings.pointCloud.minZ, pcdBoundingBox.max.z];
+ if (
+ pluginSettings?.pointCloud?.minZ !== null &&
+ pluginSettings?.pointCloud?.minZ !== undefined
+ ) {
+ return [pluginSettings.pointCloud.minZ, effectiveBoundingBox.max.z];
}
- const min = pcdBoundingBox.min.dot(upVector);
- const max = pcdBoundingBox.max.dot(upVector);
+ const min = effectiveBoundingBox.min.dot(upVector);
+ const max = effectiveBoundingBox.max.dot(upVector);
return [min, max] as const;
}, [upVector, pcdBoundingBox, pluginSettings]);
diff --git a/app/packages/looker-3d/src/fo3d/scene-controls/SceneControls.tsx b/app/packages/looker-3d/src/fo3d/scene-controls/SceneControls.tsx
index c76dada9de2..f179f6e6049 100644
--- a/app/packages/looker-3d/src/fo3d/scene-controls/SceneControls.tsx
+++ b/app/packages/looker-3d/src/fo3d/scene-controls/SceneControls.tsx
@@ -1,13 +1,15 @@
+import * as fos from "@fiftyone/state";
import { CameraControls } from "@react-three/drei";
import { useFrame } from "@react-three/fiber";
import { folder, useControls } from "leva";
import { useMemo, useRef } from "react";
+import { useRecoilState, useRecoilValue } from "recoil";
import { Vector3 } from "three";
import { PANEL_ORDER_SCENE_CONTROLS } from "../../constants";
-import { CAMERA_POSITION_KEY } from "../../Environment";
import { FoScene } from "../../hooks";
+import { avoidZFightingAtom } from "../../state";
import { useFo3dContext } from "../context";
-import { getOrthonormalAxis } from "../utils";
+import { getCameraPositionKey, getOrthonormalAxis } from "../utils";
import { Lights } from "./lights/Lights";
export const SceneControls = ({
@@ -24,8 +26,13 @@ export const SceneControls = ({
setAutoRotate,
pointCloudSettings,
setPointCloudSettings,
+ isComputingSceneBoundingBox,
} = useFo3dContext();
+ const datasetName = useRecoilValue(fos.datasetName);
+ const [avoidZFighting, setAvoidZFighting] =
+ useRecoilState(avoidZFightingAtom);
+
const dirFromUpVector = useMemo(
() => getOrthonormalAxis(upVector),
[upVector]
@@ -37,6 +44,8 @@ export const SceneControls = ({
const CAMERA_UPDATE_INTERVAL = 250;
useFrame((state) => {
+ if (isComputingSceneBoundingBox) return;
+
const now = Date.now();
const cameraControls = cameraControlsRef?.current;
if (
@@ -49,7 +58,7 @@ export const SceneControls = ({
target: cameraControls.getTarget(new Vector3()).toArray(),
};
window?.localStorage.setItem(
- CAMERA_POSITION_KEY,
+ getCameraPositionKey(datasetName),
JSON.stringify(cameraState)
);
lastCameraUpdateRef.current = now;
@@ -80,6 +89,13 @@ export const SceneControls = ({
setAutoRotate(value);
},
},
+ "Avoid Z fighting": {
+ value: avoidZFighting,
+ label: "Avoid Z fighting",
+ onChange: (value) => {
+ setAvoidZFighting(value);
+ },
+ },
},
{ collapsed: true, order: PANEL_ORDER_SCENE_CONTROLS }
),
diff --git a/app/packages/looker-3d/src/fo3d/utils.ts b/app/packages/looker-3d/src/fo3d/utils.ts
index 877019a5c08..e3bc82f284a 100644
--- a/app/packages/looker-3d/src/fo3d/utils.ts
+++ b/app/packages/looker-3d/src/fo3d/utils.ts
@@ -12,6 +12,7 @@ import {
Vector3,
type Vector3Tuple,
} from "three";
+import * as paths from "../../../utilities/src/paths";
import type {
FoMeshBasicMaterialProps,
FoMeshLambertMaterialProps,
@@ -19,7 +20,9 @@ import type {
FoScene,
FoSceneNode,
} from "../hooks";
-import * as paths from "../../../utilities/src/paths";
+
+export const getCameraPositionKey = (datasetName?: string) =>
+ `${datasetName ?? "fiftyone"}-fo3d-camera-position`;
export const getAssetUrlForSceneNode = (node: FoSceneNode): string => {
if (!node.asset) return null;
@@ -150,11 +153,13 @@ export const getResolvedUrlForFo3dAsset = (
};
export const getThreeMaterialFromFo3dMaterial = (
- foMtl: Record
+ foMtl: Record,
+ avoidZFighting: boolean = true
) => {
const { _type, ...props } = foMtl;
props["transparent"] = (props.opacity as number) < 1;
props["side"] = DoubleSide;
+ props["depthWrite"] = !avoidZFighting;
if (foMtl._type === "MeshBasicMaterial") {
return new MeshBasicMaterial(props as FoMeshBasicMaterialProps);
diff --git a/app/packages/looker-3d/src/hooks/use-3d-label-color.ts b/app/packages/looker-3d/src/hooks/use-3d-label-color.ts
index 3c084dbc5da..4b3ad700931 100644
--- a/app/packages/looker-3d/src/hooks/use-3d-label-color.ts
+++ b/app/packages/looker-3d/src/hooks/use-3d-label-color.ts
@@ -3,6 +3,7 @@ import {
LABEL_3D_HOVERED_AND_SELECTED_COLOR,
LABEL_3D_HOVERED_COLOR,
LABEL_3D_INSTANCE_HOVERED_COLOR,
+ LABEL_3D_SELECTED_FOR_ANNOTATION_COLOR,
LABEL_3D_SIMILAR_SELECTED_COLOR,
} from "../constants";
@@ -11,20 +12,30 @@ export const use3dLabelColor = ({
isHovered,
isSimilarLabelHovered,
defaultColor,
+ isSelectedForAnnotation,
}: {
isSelected: boolean;
isHovered: boolean;
isSimilarLabelHovered: boolean;
defaultColor: string;
+ isSelectedForAnnotation?: boolean;
}) => {
return useMemo(() => {
const isAnyHovered = isHovered || isSimilarLabelHovered;
+ if (isSelectedForAnnotation) return LABEL_3D_SELECTED_FOR_ANNOTATION_COLOR;
+
if (isAnyHovered && isSelected) return LABEL_3D_HOVERED_AND_SELECTED_COLOR;
if (isSelected) return LABEL_3D_SIMILAR_SELECTED_COLOR;
if (isSimilarLabelHovered) return LABEL_3D_INSTANCE_HOVERED_COLOR;
if (isHovered) return LABEL_3D_HOVERED_COLOR;
return defaultColor;
- }, [isSelected, isHovered, isSimilarLabelHovered, defaultColor]);
+ }, [
+ isSelected,
+ isHovered,
+ isSimilarLabelHovered,
+ defaultColor,
+ isSelectedForAnnotation,
+ ]);
};
diff --git a/app/packages/looker-3d/src/hooks/use-bounds.test.ts b/app/packages/looker-3d/src/hooks/use-bounds.test.ts
index 220cca69c50..a43005a7df4 100644
--- a/app/packages/looker-3d/src/hooks/use-bounds.test.ts
+++ b/app/packages/looker-3d/src/hooks/use-bounds.test.ts
@@ -8,6 +8,7 @@ vi.useFakeTimers();
vi.mock("three", () => {
return {
Box3: vi.fn(),
+ Vector3: vi.fn(),
};
});
@@ -17,18 +18,19 @@ describe("useFo3dBounds", () => {
vi.resetAllMocks();
});
- it("does not set bounding box when objectRef.current is null", () => {
+ it("returns null when objectRef.current is null", () => {
const objectRef = { current: null } as React.RefObject;
const { result } = renderHook(() => useFo3dBounds(objectRef));
- expect(result.current).toBeNull();
+ expect(result.current.boundingBox).toBeNull();
act(() => {
vi.advanceTimersByTime(500);
});
- expect(result.current).toBeNull();
+ // The hook should return null when objectRef.current is null
+ expect(result.current.boundingBox).toBeNull();
});
it("sets bounding box when bounding box stabilizes", () => {
@@ -66,7 +68,7 @@ describe("useFo3dBounds", () => {
const { result } = renderHook(() => useFo3dBounds(objectRef));
- expect(result.current).toBeNull();
+ expect(result.current.boundingBox).toBeNull();
act(() => {
for (let i = 0; i < 10; i++) {
@@ -74,9 +76,35 @@ describe("useFo3dBounds", () => {
}
});
- expect(result.current).not.toBeNull();
- expect(result.current.min).toEqual(boxes[1].min);
- expect(result.current.max).toEqual(boxes[1].max);
+ expect(result.current.boundingBox).not.toBeNull();
+ expect(result.current.boundingBox.min).toEqual(boxes[1].min);
+ expect(result.current.boundingBox.max).toEqual(boxes[1].max);
+ });
+
+ it("returns null when bounds are incomputable (non-finite box)", () => {
+ const objectRef = { current: {} } as React.RefObject;
+
+ // Mock Box3 to return a box with non-finite values
+ const MockBox3 = vi.fn().mockImplementation(() => {
+ return {
+ min: { x: Infinity, y: 0, z: 0, equals: vi.fn(() => true) },
+ max: { x: 1, y: 1, z: 1, equals: vi.fn(() => true) },
+ setFromObject: vi.fn().mockReturnThis(),
+ };
+ });
+
+ (Box3 as unknown as Mock).mockImplementation(MockBox3);
+
+ const { result } = renderHook(() => useFo3dBounds(objectRef));
+
+ expect(result.current.boundingBox).toBeNull();
+
+ act(() => {
+ vi.advanceTimersByTime(500);
+ });
+
+ // The hook should return null when bounds are incomputable
+ expect(result.current.boundingBox).toBeNull();
});
it("does not proceed if predicate returns false", () => {
@@ -85,13 +113,13 @@ describe("useFo3dBounds", () => {
const { result } = renderHook(() => useFo3dBounds(objectRef, predicate));
- expect(result.current).toBeNull();
+ expect(result.current.boundingBox).toBeNull();
expect(predicate).toHaveBeenCalled();
act(() => {
vi.advanceTimersByTime(500);
});
- expect(result.current).toBeNull();
+ expect(result.current.boundingBox).toBeNull();
});
});
diff --git a/app/packages/looker-3d/src/hooks/use-bounds.ts b/app/packages/looker-3d/src/hooks/use-bounds.ts
index c38fbb9b8a2..dc93a143cc8 100644
--- a/app/packages/looker-3d/src/hooks/use-bounds.ts
+++ b/app/packages/looker-3d/src/hooks/use-bounds.ts
@@ -1,8 +1,24 @@
-import { useLayoutEffect, useRef, useState } from "react";
+import { useCallback, useLayoutEffect, useRef, useState } from "react";
import { Box3, type Group } from "three";
const BOUNDING_BOX_POLLING_INTERVAL = 50;
const UNCHANGED_COUNT_THRESHOLD = 6;
+const MAX_BOUNDING_BOX_RETRIES = 1000;
+
+/**
+ * Checks if a bounding box has all finite values in its min and max components.
+ * Returns true if all components are finite, false otherwise.
+ */
+const isFiniteBox = (box: Box3): boolean => {
+ return (
+ Number.isFinite(box.min.x) &&
+ Number.isFinite(box.min.y) &&
+ Number.isFinite(box.min.z) &&
+ Number.isFinite(box.max.x) &&
+ Number.isFinite(box.max.y) &&
+ Number.isFinite(box.max.z)
+ );
+};
/**
* Calculates the bounding box of the object with the given ref.
@@ -10,19 +26,42 @@ const UNCHANGED_COUNT_THRESHOLD = 6;
* @param objectRef - Ref to the object
* @param predicate - Optional predicate to check before calculating the bounding box.
* IMPORTANT: Make sure this predicate is memoized using useCallback
- * @returns Bounding box of the object
+ * @returns Object containing the bounding box, a function to recompute it, and a flag indicating if computation is in progress
*/
export const useFo3dBounds = (
objectRef: React.RefObject,
predicate?: () => boolean
) => {
- const [sceneBoundingBox, setSceneBoundingBox] = useState(null);
+ const [boundingBox, setBoundingBox] = useState(null);
+ const [isComputing, setIsComputing] = useState(false);
const unchangedCount = useRef(0);
const previousBox = useRef(null);
+ const retryCount = useRef(0);
const timeOutIdRef = useRef(null);
+ const recomputeBounds = useCallback(() => {
+ setIsComputing(true);
+
+ if (!objectRef.current) {
+ setBoundingBox(null);
+ setIsComputing(false);
+ return;
+ }
+
+ const box = new Box3().setFromObject(objectRef.current);
+
+ if (!isFiniteBox(box)) {
+ setBoundingBox(null);
+ setIsComputing(false);
+ return;
+ }
+
+ setBoundingBox(box);
+ setIsComputing(false);
+ }, [objectRef]);
+
useLayoutEffect(() => {
if (predicate && !predicate()) {
return;
@@ -38,7 +77,18 @@ export const useFo3dBounds = (
const getBoundingBox = () => {
if (!isMounted) return;
+ setIsComputing(true);
+
if (!objectRef.current) {
+ retryCount.current += 1;
+ if (retryCount.current >= MAX_BOUNDING_BOX_RETRIES) {
+ retryCount.current = 0;
+ unchangedCount.current = 0;
+ previousBox.current = null;
+ setBoundingBox(null);
+ setIsComputing(false);
+ return;
+ }
timeOutIdRef.current = window.setTimeout(
getBoundingBox,
BOUNDING_BOX_POLLING_INTERVAL
@@ -48,7 +98,16 @@ export const useFo3dBounds = (
const box = new Box3().setFromObject(objectRef.current);
- if (Math.abs(box.max?.x) === Number.POSITIVE_INFINITY) {
+ if (!isFiniteBox(box)) {
+ retryCount.current += 1;
+ if (retryCount.current >= MAX_BOUNDING_BOX_RETRIES) {
+ retryCount.current = 0;
+ unchangedCount.current = 0;
+ previousBox.current = null;
+ setBoundingBox(null);
+ setIsComputing(false);
+ return;
+ }
timeOutIdRef.current = window.setTimeout(
getBoundingBox,
BOUNDING_BOX_POLLING_INTERVAL
@@ -65,7 +124,9 @@ export const useFo3dBounds = (
previousBox.current = box;
if (unchangedCount.current >= UNCHANGED_COUNT_THRESHOLD) {
- setSceneBoundingBox(box);
+ retryCount.current = 0;
+ setBoundingBox(box);
+ setIsComputing(false);
} else {
timeOutIdRef.current = window.setTimeout(
getBoundingBox,
@@ -74,7 +135,7 @@ export const useFo3dBounds = (
}
};
- // this is a hack, yet to find a better way than polling to know when the scene is done loading
+ // this is a hack, yet to find a better way than polling to know when the asset is done loading
// callbacks in loaders are not reliable
timeOutIdRef.current = window.setTimeout(
getBoundingBox,
@@ -84,6 +145,8 @@ export const useFo3dBounds = (
// cleanup function to prevent memory leaks
return () => {
isMounted = false;
+ retryCount.current = 0;
+ setIsComputing(false);
if (timeOutIdRef.current) {
window.clearTimeout(timeOutIdRef.current);
@@ -91,5 +154,5 @@ export const useFo3dBounds = (
};
}, [objectRef, predicate]);
- return sceneBoundingBox;
+ return { boundingBox, recomputeBounds, isComputing };
};
diff --git a/app/packages/looker-3d/src/hooks/use-empty-canvas-interaction.ts b/app/packages/looker-3d/src/hooks/use-empty-canvas-interaction.ts
new file mode 100644
index 00000000000..010de0901a4
--- /dev/null
+++ b/app/packages/looker-3d/src/hooks/use-empty-canvas-interaction.ts
@@ -0,0 +1,76 @@
+import { useThree } from "@react-three/fiber";
+import { useEffect, useMemo, useRef } from "react";
+import * as THREE from "three";
+import {
+ createPlane,
+ getPlaneIntersection,
+ isButtonMatch,
+ toNDC,
+} from "../utils";
+
+type Options = {
+ onPointerUp?: (pt: THREE.Vector3, ev: PointerEvent) => void;
+ onPointerDown?: () => void;
+ onPointerMove?: (pt: THREE.Vector3, ev: PointerEvent) => void;
+ planeNormal?: THREE.Vector3;
+ planeConstant?: number;
+ button?: number;
+};
+
+export function useEmptyCanvasInteraction({
+ onPointerUp,
+ onPointerDown,
+ onPointerMove,
+ planeNormal = new THREE.Vector3(0, 1, 0),
+ planeConstant = 0,
+ button = 0,
+}: Options = {}) {
+ // Refs to avoid re-rendering
+ const onPointerUpRef = useRef(onPointerUp);
+ const onPointerDownRef = useRef(onPointerDown);
+ const onPointerMoveRef = useRef(onPointerMove);
+
+ onPointerUpRef.current = onPointerUp;
+ onPointerDownRef.current = onPointerDown;
+ onPointerMoveRef.current = onPointerMove;
+
+ const { gl, camera, scene, raycaster, events } = useThree();
+
+ const plane = useMemo(
+ () => createPlane(planeNormal, planeConstant),
+ [planeNormal, planeConstant]
+ );
+
+ useEffect(() => {
+ const el = (events.connected ?? gl.domElement) as HTMLCanvasElement;
+
+ const handlePointerMove = (ev: PointerEvent) => {
+ if (!onPointerMoveRef.current) return;
+ const ndc = toNDC(ev, el);
+ const pt = getPlaneIntersection(raycaster, camera, ndc, plane);
+ if (pt) onPointerMoveRef.current(pt, ev);
+ };
+
+ const handleDown = (ev: PointerEvent) => {
+ if (!isButtonMatch(ev, button)) return;
+ onPointerDownRef.current?.();
+ };
+
+ const handleUp = (ev: PointerEvent) => {
+ if (!isButtonMatch(ev, button)) return;
+ const ndc = toNDC(ev, el);
+ const pt = getPlaneIntersection(raycaster, camera, ndc, plane);
+ if (pt) onPointerUpRef.current?.(pt, ev);
+ };
+
+ el.addEventListener("pointermove", handlePointerMove, { passive: true });
+ el.addEventListener("pointerdown", handleDown, { passive: true });
+ el.addEventListener("pointerup", handleUp, { passive: true });
+
+ return () => {
+ el.removeEventListener("pointermove", handlePointerMove);
+ el.removeEventListener("pointerdown", handleDown);
+ el.removeEventListener("pointerup", handleUp);
+ };
+ }, [gl, camera, scene, raycaster, events, plane, button]);
+}
diff --git a/app/packages/looker-3d/src/hooks/use-mesh-material-controls.ts b/app/packages/looker-3d/src/hooks/use-mesh-material-controls.ts
index dfc5466a6fa..a28f5c1c136 100644
--- a/app/packages/looker-3d/src/hooks/use-mesh-material-controls.ts
+++ b/app/packages/looker-3d/src/hooks/use-mesh-material-controls.ts
@@ -1,7 +1,9 @@
import { folder, useControls } from "leva";
import { useMemo, useState } from "react";
+import { useRecoilValue } from "recoil";
import { PANEL_ORDER_PCD_CONTROLS } from "../constants";
import { getThreeMaterialFromFo3dMaterial } from "../fo3d/utils";
+import { avoidZFightingAtom } from "../state";
import type { FoMeshMaterial } from "./use-fo3d";
export const useMeshMaterialControls = (
@@ -9,6 +11,7 @@ export const useMeshMaterialControls = (
foMeshMaterial: FoMeshMaterial,
omitColorControls = false
) => {
+ const avoidZFighting = useRecoilValue(avoidZFightingAtom);
const [opacity, setOpacity] = useState(foMeshMaterial.opacity);
const [renderAsWireframe, setRenderAsWireframe] = useState(
foMeshMaterial.wireframe
@@ -132,16 +135,19 @@ export const useMeshMaterialControls = (
);
const material = useMemo(() => {
- return getThreeMaterialFromFo3dMaterial({
- ...foMeshMaterial,
- opacity,
- wireframe: renderAsWireframe,
- color,
- metalness,
- roughness,
- emissiveColor,
- emissiveIntensity,
- });
+ return getThreeMaterialFromFo3dMaterial(
+ {
+ ...foMeshMaterial,
+ opacity,
+ wireframe: renderAsWireframe,
+ color,
+ metalness,
+ roughness,
+ emissiveColor,
+ emissiveIntensity,
+ },
+ avoidZFighting
+ );
}, [
foMeshMaterial,
opacity,
@@ -151,6 +157,7 @@ export const useMeshMaterialControls = (
roughness,
emissiveColor,
emissiveIntensity,
+ avoidZFighting,
]);
return {
diff --git a/app/packages/looker-3d/src/labels/cuboid.tsx b/app/packages/looker-3d/src/labels/cuboid.tsx
index 9b8f467a4b5..b2792862a48 100644
--- a/app/packages/looker-3d/src/labels/cuboid.tsx
+++ b/app/packages/looker-3d/src/labels/cuboid.tsx
@@ -1,21 +1,19 @@
-import { useCursor } from "@react-three/drei";
import { extend } from "@react-three/fiber";
-import { useMemo, useState } from "react";
-import { useRecoilValue } from "recoil";
+import { useEffect, useMemo } from "react";
import * as THREE from "three";
import { LineMaterial } from "three/examples/jsm/lines/LineMaterial";
import { LineSegments2 } from "three/examples/jsm/lines/LineSegments2";
import { LineSegmentsGeometry } from "three/examples/jsm/lines/LineSegmentsGeometry";
-import { use3dLabelColor } from "../hooks/use-3d-label-color";
-import { useSimilarLabels3d } from "../hooks/use-similar-labels-3d";
-import { cuboidLabelLineWidthAtom } from "../state";
import type { OverlayProps } from "./shared";
+import { useEventHandlers, useHoverState, useLabelColor } from "./shared/hooks";
+
extend({ LineSegments2, LineMaterial, LineSegmentsGeometry });
export interface CuboidProps extends OverlayProps {
location: THREE.Vector3Tuple;
dimensions: THREE.Vector3Tuple;
itemRotation: THREE.Vector3Tuple;
+ lineWidth?: number;
}
export const Cuboid = ({
@@ -24,6 +22,7 @@ export const Cuboid = ({
opacity,
rotation,
location,
+ lineWidth,
selected,
onClick,
tooltip,
@@ -31,7 +30,6 @@ export const Cuboid = ({
color,
useLegacyCoordinates,
}: CuboidProps) => {
- const lineWidth = useRecoilValue(cuboidLabelLineWidthAtom);
const geo = useMemo(
() => dimensions && new THREE.BoxGeometry(...dimensions),
[dimensions]
@@ -58,8 +56,18 @@ export const Cuboid = ({
[resolvedRotation, itemRotationVec]
);
- const [isCuboidHovered, setIsCuboidHovered] = useState(false);
- useCursor(isCuboidHovered);
+ const { isHovered, setIsHovered } = useHoverState();
+ const { onPointerOver, onPointerOut, restEventHandlers } = useEventHandlers(
+ tooltip,
+ label
+ );
+
+ const { strokeAndFillColor, isSimilarLabelHovered } = useLabelColor(
+ { selected, color },
+ isHovered,
+ label,
+ false
+ );
const edgesGeo = useMemo(() => new THREE.EdgesGeometry(geo), [geo]);
const geometry = useMemo(
@@ -70,20 +78,11 @@ export const Cuboid = ({
[edgesGeo]
);
- const isSimilarLabelHovered = useSimilarLabels3d(label);
-
- const strokeAndFillColor = use3dLabelColor({
- isSelected: selected,
- isHovered: isCuboidHovered,
- isSimilarLabelHovered,
- defaultColor: color,
- });
-
const material = useMemo(
() =>
new LineMaterial({
opacity: opacity,
- transparent: false,
+ transparent: opacity < 0.2,
color: strokeAndFillColor,
linewidth: lineWidth,
}),
@@ -91,17 +90,21 @@ export const Cuboid = ({
selected,
lineWidth,
opacity,
- isCuboidHovered,
+ isHovered,
isSimilarLabelHovered,
strokeAndFillColor,
]
);
- const { onPointerOver, onPointerOut, ...restEventHandlers } = useMemo(() => {
- return {
- ...tooltip.getMeshProps(label),
+ // Cleanup
+ useEffect(() => {
+ return () => {
+ geo.dispose();
+ edgesGeo.dispose();
+ geometry.dispose();
+ material.dispose();
};
- }, [tooltip, label]);
+ }, [geo, edgesGeo, geometry, material]);
if (!location || !dimensions) return null;
@@ -114,20 +117,27 @@ export const Cuboid = ({
*/
return (
-
-
-
-
+ <>
+ {/* Outline */}
+ {/* @ts-ignore */}
+
+
+ {/* Clickable volume */}
{
- setIsCuboidHovered(true);
+ setIsHovered(true);
onPointerOver();
}}
onPointerOut={() => {
- setIsCuboidHovered(false);
+ setIsHovered(false);
onPointerOut();
}}
{...restEventHandlers}
@@ -139,6 +149,6 @@ export const Cuboid = ({
color={strokeAndFillColor}
/>
-
+
);
};
diff --git a/app/packages/looker-3d/src/labels/index.tsx b/app/packages/looker-3d/src/labels/index.tsx
index e3347d753f9..75ada4ba7a1 100644
--- a/app/packages/looker-3d/src/labels/index.tsx
+++ b/app/packages/looker-3d/src/labels/index.tsx
@@ -1,6 +1,6 @@
+import { activeSchemas } from "@fiftyone/core/src/components/Modal/Sidebar/Annotate/state";
import {
FO_LABEL_TOGGLED_EVENT,
- LabelData,
LabelToggledEvent,
Sample,
selectiveRenderingEventBus,
@@ -14,14 +14,27 @@ import * as fos from "@fiftyone/state";
import { fieldSchema } from "@fiftyone/state";
import { useOnShiftClickLabel } from "@fiftyone/state/src/hooks/useOnShiftClickLabel";
import { ThreeEvent } from "@react-three/fiber";
+import { useAtomValue } from "jotai";
import { folder, useControls } from "leva";
import { get as _get } from "lodash";
import { useCallback, useEffect, useMemo } from "react";
-import { useRecoilState, useRecoilValue } from "recoil";
+import { useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
+import * as THREE from "three";
+import type { PolylinePointTransform } from "../annotation/types";
import { PANEL_ORDER_LABELS } from "../constants";
import { usePathFilter } from "../hooks";
import { type Looker3dSettings, defaultPluginSettings } from "../settings";
-import { cuboidLabelLineWidthAtom, polylineLabelLineWidthAtom } from "../state";
+import {
+ cuboidLabelLineWidthAtom,
+ currentArchetypeSelectedForTransformAtom,
+ editSegmentsModeAtom,
+ polylineLabelLineWidthAtom,
+ polylinePointTransformsAtom,
+ segmentPolylineStateAtom,
+ selectedLabelForAnnotationAtom,
+ transformModeAtom,
+} from "../state";
+import { TransformArchetype } from "../types";
import { toEulerFromDegreesArray } from "../utils";
import { Cuboid, type CuboidProps } from "./cuboid";
import { type OverlayLabel, load3dOverlays } from "./loader";
@@ -29,12 +42,19 @@ import { type PolyLineProps, Polyline } from "./polyline";
export interface ThreeDLabelsProps {
sampleMap: { [sliceOrFilename: string]: Sample } | fos.Sample[];
+ globalOpacity?: number;
}
-export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
+export const ThreeDLabels = ({
+ sampleMap,
+ globalOpacity,
+}: ThreeDLabelsProps) => {
+ const mode = useAtomValue(fos.modalMode);
const schema = useRecoilValue(fieldSchema({ space: fos.State.SPACE.SAMPLE }));
+ const annotationSchemas = useAtomValue(activeSchemas);
const { coloring, selectedLabelTags, customizeColorSetting, labelTagColors } =
useRecoilValue(fos.lookerOptions({ withFilter: true, modal: true }));
+ const isSegmenting = useRecoilValue(segmentPolylineStateAtom).isActive;
const settings = fop.usePluginSettings(
"3d",
@@ -49,11 +69,21 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
const [polylineWidth, setPolylineWidth] = useRecoilState(
polylineLabelLineWidthAtom
);
+ const polylinePointTransforms = useRecoilValue(polylinePointTransformsAtom);
const selectedLabels = useRecoilValue(fos.selectedLabelMap);
const tooltip = fos.useTooltip();
- const labelAlpha = colorScheme.opacity;
+ const labelAlpha = globalOpacity ?? colorScheme.opacity;
+
+ const [selectedLabelForAnnotation, setSelectedLabelForAnnotation] =
+ useRecoilState(selectedLabelForAnnotationAtom);
+ const setEditSegmentsMode = useSetRecoilState(editSegmentsModeAtom);
+
+ const [transformMode, setTransformMode] = useRecoilState(transformModeAtom);
+ const setCurrentArchetypeSelectedForTransform = useSetRecoilState(
+ currentArchetypeSelectedForTransformAtom
+ );
- const constLabelLevaControls = {
+ const labelLevaControls = {
cuboidLineWidget: {
value: cuboidLineWidth,
min: 0,
@@ -76,9 +106,11 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
},
};
- const [labelConfig] = useControls(
+ const currentSampleId = useRecoilValue(fos.currentSampleId);
+
+ useControls(
() => ({
- Labels: folder(constLabelLevaControls, {
+ Labels: folder(labelLevaControls, {
order: PANEL_ORDER_LABELS,
collapsed: true,
}),
@@ -86,8 +118,40 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
[setCuboidLineWidth, setPolylineWidth]
);
+ const selectLabelForAnnotation = useCallback(
+ (label: OverlayLabel) => {
+ setSelectedLabelForAnnotation(label);
+
+ // We only support translate for polylines for now
+ if (
+ label._cls === "Polyline" &&
+ (transformMode === "rotate" || transformMode === "scale")
+ ) {
+ setTransformMode("translate");
+ }
+ },
+ [setSelectedLabelForAnnotation, transformMode, setTransformMode]
+ );
+
const handleSelect = useCallback(
- (label: OverlayLabel, e: ThreeEvent) => {
+ (
+ label: OverlayLabel,
+ archetype: TransformArchetype,
+ e: ThreeEvent
+ ) => {
+ if (isSegmenting) return;
+
+ if (mode === "annotate") {
+ if (archetype === "cuboid") {
+ return;
+ }
+
+ selectLabelForAnnotation(label);
+ setCurrentArchetypeSelectedForTransform(archetype);
+
+ return;
+ }
+
onSelectLabel({
detail: {
id: label._id,
@@ -98,9 +162,31 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
},
});
},
- [onSelectLabel]
+ [onSelectLabel, mode, selectLabelForAnnotation, isSegmenting]
);
+ useEffect(() => {
+ const handler = (event: KeyboardEvent) => {
+ if (
+ event.key === "Escape" &&
+ mode === "annotate" &&
+ selectedLabelForAnnotation
+ ) {
+ setSelectedLabelForAnnotation(null);
+ setEditSegmentsMode(false);
+
+ event.stopImmediatePropagation();
+ event.preventDefault();
+ }
+ };
+
+ document.addEventListener("keydown", handler);
+
+ return () => {
+ document.removeEventListener("keydown", handler);
+ };
+ }, [setSelectedLabelForAnnotation, mode, selectedLabelForAnnotation]);
+
const [overlayRotation, itemRotation] = useMemo(
() => [
toEulerFromDegreesArray(_get(settings, "overlay.rotation", [0, 0, 0])),
@@ -111,13 +197,6 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
[settings]
);
- const canonicalSampleId = useMemo(() => {
- const samples = Array.isArray(sampleMap)
- ? sampleMap
- : Object.values(sampleMap);
- return samples[0].id ?? samples[0].sample?._id;
- }, [sampleMap]);
-
const rawOverlays = useMemo(
() =>
load3dOverlays(sampleMap, selectedLabels, [], schema)
@@ -136,7 +215,18 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
return { ...l, color, id: l._id };
})
- .filter((l) => pathFilter(l.path, l)),
+ .filter((l) => {
+ if (!pathFilter(l.path, l)) {
+ return false;
+ }
+
+ // In annotate mode, only show fields that exist in annotation schemas
+ if (mode === "annotate") {
+ return annotationSchemas && l.path in annotationSchemas;
+ }
+
+ return true;
+ }),
[
coloring,
pathFilter,
@@ -146,6 +236,8 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
selectedLabelTags,
labelTagColors,
customizeColorSetting,
+ mode,
+ annotationSchemas,
]
);
const [cuboidOverlays, polylineOverlays] = useMemo(() => {
@@ -161,11 +253,12 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
newCuboidOverlays.push(
handleSelect(overlay, e)}
+ {...(overlay as CuboidProps)}
+ onClick={(e) => handleSelect(overlay, "cuboid", e)}
label={overlay}
tooltip={tooltip}
useLegacyCoordinates={settings.useLegacyCoordinates}
@@ -173,21 +266,100 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
);
} else if (
overlay._cls === "Polyline" &&
- (overlay as unknown as PolyLineProps).points3d
+ (overlay as PolyLineProps).points3d
) {
newPolylineOverlays.push(
handleSelect(overlay, e)}
+ onClick={(e) => handleSelect(overlay, "polyline", e)}
tooltip={tooltip}
/>
);
}
}
+
+ // Check for any label ids in polylinePointTransformsAtom that are not in newPolylineOverlays
+ // and create new polyline overlays for them
+ const existingPolylineIds = new Set(
+ rawOverlays
+ .filter((overlay) => overlay._cls === "Polyline")
+ .map((overlay) => overlay._id)
+ );
+
+ for (const [labelId, transforms] of Object.entries(
+ polylinePointTransforms
+ )) {
+ if (transforms.length === 0 || existingPolylineIds.has(labelId)) {
+ continue;
+ }
+
+ // Convert PolylinePointTransform[] to points3d format
+ const points3d: THREE.Vector3Tuple[][] = [];
+
+ // Group transforms by segmentIndex
+ const segmentsMap = new Map();
+ transforms.forEach((transform) => {
+ const { segmentIndex } = transform;
+ if (!segmentsMap.has(segmentIndex)) {
+ segmentsMap.set(segmentIndex, []);
+ }
+ segmentsMap.get(segmentIndex)!.push(transform);
+ });
+
+ // Create segments from grouped transforms
+ segmentsMap.forEach((segmentTransforms) => {
+ // Sort by pointIndex to maintain order
+ segmentTransforms.sort((a, b) => a.pointIndex - b.pointIndex);
+
+ // Create segment points
+ const segmentPoints: THREE.Vector3Tuple[] = [];
+ segmentTransforms.forEach((transform) => {
+ segmentPoints[transform.pointIndex] = transform.position;
+ });
+
+ // Only add segment if it has at least 2 points
+ if (segmentPoints.length >= 2) {
+ points3d.push(segmentPoints);
+ }
+ });
+
+ // Only create overlay if we have valid segments
+ if (points3d.length > 0) {
+ const overlayLabel = {
+ _id: labelId,
+ _cls: "Polyline",
+ type: "Polyline",
+ // todo: THIS HAS TO CHANGE TO A REAL PATH
+ path: `polyline-${labelId.substring(0, 5)}`,
+ selected: false,
+ sampleId: currentSampleId,
+ tags: [],
+ points3d,
+ };
+
+ newPolylineOverlays.push(
+ handleSelect(overlayLabel, "polyline", e)}
+ tooltip={tooltip}
+ />
+ );
+ }
+ }
+
return [newCuboidOverlays, newPolylineOverlays];
}, [
rawOverlays,
@@ -197,6 +369,10 @@ export const ThreeDLabels = ({ sampleMap }: ThreeDLabelsProps) => {
handleSelect,
tooltip,
settings,
+ transformMode,
+ polylinePointTransforms,
+ polylineWidth,
+ currentSampleId,
]);
const getOnShiftClickLabelCallback = useOnShiftClickLabel();
diff --git a/app/packages/looker-3d/src/labels/line.tsx b/app/packages/looker-3d/src/labels/line.tsx
deleted file mode 100644
index c33e94f251b..00000000000
--- a/app/packages/looker-3d/src/labels/line.tsx
+++ /dev/null
@@ -1,46 +0,0 @@
-import { extend } from "@react-three/fiber";
-import React from "react";
-import { useRecoilValue } from "recoil";
-import * as THREE from "three";
-import { Line2 } from "three/examples/jsm/lines/Line2";
-import { LineGeometry } from "three/examples/jsm/lines/LineGeometry";
-import { LineMaterial } from "three/examples/jsm/lines/LineMaterial";
-import { polylineLabelLineWidthAtom } from "../state";
-import type { OverlayProps } from "./shared";
-
-extend({ Line2, LineMaterial, LineGeometry });
-
-interface LineProps extends Omit {
- points: THREE.Vector3Tuple[];
-}
-
-export const Line = ({ rotation, points, color, opacity }: LineProps) => {
- const lineWidth = useRecoilValue(polylineLabelLineWidthAtom);
- const geo = React.useMemo(() => {
- const g = new THREE.BufferGeometry().setFromPoints(
- points.map((p) => new THREE.Vector3(...p))
- );
- g.rotateX(rotation[0]);
- g.rotateY(rotation[1]);
- g.rotateZ(rotation[2]);
- return g;
- }, [points, rotation]);
-
- const geometry = React.useMemo(() => {
- return new LineGeometry().fromLine(new THREE.Line(geo));
- }, [geo]);
-
- const material = React.useMemo(() => {
- return new LineMaterial({
- color: color,
- linewidth: lineWidth,
- opacity: opacity,
- });
- }, [color, lineWidth, opacity]);
-
- return (
-
-
-
- );
-};
diff --git a/app/packages/looker-3d/src/labels/polyline.tsx b/app/packages/looker-3d/src/labels/polyline.tsx
index 0ee2267f955..cd988f64be1 100644
--- a/app/packages/looker-3d/src/labels/polyline.tsx
+++ b/app/packages/looker-3d/src/labels/polyline.tsx
@@ -1,16 +1,25 @@
-import { useCursor } from "@react-three/drei";
-import { useMemo, useState } from "react";
+import * as fos from "@fiftyone/state";
+import { Line as LineDrei } from "@react-three/drei";
+import { useAtomValue } from "jotai";
+import { useEffect, useMemo, useRef } from "react";
+import { useRecoilValue } from "recoil";
import * as THREE from "three";
-import { use3dLabelColor } from "../hooks/use-3d-label-color";
-import { useSimilarLabels3d } from "../hooks/use-similar-labels-3d";
-import { Line } from "./line";
+import { usePolylineAnnotation } from "../annotation/usePolylineAnnotation";
+import {
+ selectedLabelForAnnotationAtom,
+ tempLabelTransformsAtom,
+} from "../state";
import { createFilledPolygonMeshes } from "./polygon-fill-utils";
import type { OverlayProps } from "./shared";
+import { useEventHandlers, useHoverState, useLabelColor } from "./shared/hooks";
+import { Transformable } from "./shared/TransformControls";
export interface PolyLineProps extends OverlayProps {
+ // Array of line segments, where each segment is an array of 3D points
points3d: THREE.Vector3Tuple[][];
filled: boolean;
- // we ignore closed for now
+ lineWidth?: number;
+ // We ignore closed for now
closed?: boolean;
}
@@ -21,52 +30,103 @@ export const Polyline = ({
points3d,
color,
selected,
+ lineWidth,
onClick,
tooltip,
label,
}: PolyLineProps) => {
- const { onPointerOver, onPointerOut, ...restEventHandlers } = useMemo(() => {
- return { ...tooltip.getMeshProps(label) };
- }, [tooltip, label]);
-
- const [isPolylineHovered, setIsPolylineHovered] = useState(false);
- const isSimilarLabelHovered = useSimilarLabels3d(label);
- useCursor(isPolylineHovered);
-
- const strokeAndFillColor = use3dLabelColor({
- isSelected: selected,
- isHovered: isPolylineHovered,
- isSimilarLabelHovered,
- defaultColor: color,
+ const meshesRef = useRef([]);
+
+ const { isHovered, setIsHovered } = useHoverState();
+ const { onPointerOver, onPointerOut, restEventHandlers } = useEventHandlers(
+ tooltip,
+ label
+ );
+
+ const isAnnotateMode = useAtomValue(fos.modalMode) === "annotate";
+ const isSelectedForAnnotation =
+ useRecoilValue(selectedLabelForAnnotationAtom)?._id === label._id;
+
+ const { strokeAndFillColor } = useLabelColor(
+ { selected, color },
+ isHovered,
+ label,
+ isSelectedForAnnotation
+ );
+
+ const {
+ effectivePoints3d,
+ centroid,
+ transformControlsRef,
+ contentRef,
+ markers,
+ handleTransformStart,
+ handleTransformChange,
+ handleTransformEnd,
+ handlePointerOver: handleAnnotationPointerOver,
+ handlePointerOut: handleAnnotationPointerOut,
+ handleSegmentPointerOver,
+ handleSegmentPointerOut,
+ handleSegmentClick,
+ } = usePolylineAnnotation({
+ label,
+ points3d,
+ strokeAndFillColor,
+ isAnnotateMode,
+ isSelectedForAnnotation,
});
const lines = useMemo(
() =>
- points3d.map((pts, i) => (
- (
+ handleSegmentPointerOver(i)}
+ onPointerOut={handleSegmentPointerOut}
+ onClick={handleSegmentClick}
/>
)),
- [points3d, rotation, opacity, strokeAndFillColor, label]
+ [
+ effectivePoints3d,
+ strokeAndFillColor,
+ lineWidth,
+ rotation,
+ label._id,
+ handleSegmentPointerOver,
+ handleSegmentPointerOut,
+ handleSegmentClick,
+ ]
);
- const filledMeshes = useMemo(() => {
+ const material = useMemo(() => {
if (!filled) return null;
- const material = new THREE.MeshBasicMaterial({
+ return new THREE.MeshBasicMaterial({
color: strokeAndFillColor,
opacity,
transparent: true,
side: THREE.DoubleSide,
depthWrite: false,
});
+ }, [filled, strokeAndFillColor, opacity]);
- const meshes = createFilledPolygonMeshes(points3d, material);
+ const filledMeshes = useMemo(() => {
+ if (!filled || !material) return null;
+
+ // Dispose previous meshes
+ meshesRef.current.forEach((mesh) => {
+ if (mesh.geometry) mesh.geometry.dispose();
+ });
+
+ const meshes = createFilledPolygonMeshes(effectivePoints3d, material);
+ meshesRef.current = meshes || [];
if (!meshes) return null;
@@ -77,42 +137,69 @@ export const Polyline = ({
rotation={rotation as unknown as THREE.Euler}
/>
));
- }, [filled, points3d, rotation, strokeAndFillColor, opacity, label._id]);
+ }, [filled, effectivePoints3d, rotation, material, label._id]);
- if (filled && filledMeshes) {
- return (
- {
- setIsPolylineHovered(true);
- onPointerOver();
- }}
- onPointerOut={() => {
- setIsPolylineHovered(false);
- onPointerOut();
- }}
- onClick={onClick}
- {...restEventHandlers}
- >
- {filledMeshes}
- {lines}
-
- );
- }
+ useEffect(() => {
+ return () => {
+ meshesRef.current.forEach((mesh) => {
+ if (mesh.geometry) {
+ mesh.geometry.dispose();
+ }
+ });
+ };
+ }, []);
+
+ useEffect(() => {
+ return () => {
+ if (material) {
+ material.dispose();
+ }
+ };
+ }, [material]);
+
+ const tempTransforms = useRecoilValue(tempLabelTransformsAtom(label._id));
+
+ const content = (
+ <>
+ {filled && filledMeshes}
+ {lines}
+
+ );
return (
- {
- setIsPolylineHovered(true);
- onPointerOver();
- }}
- onPointerOut={() => {
- setIsPolylineHovered(false);
- onPointerOut();
- }}
- onClick={onClick}
- {...restEventHandlers}
+
- {lines}
-
+
+ {markers}
+ {
+ setIsHovered(true);
+ handleAnnotationPointerOver();
+ onPointerOver();
+ }}
+ onPointerOut={() => {
+ setIsHovered(false);
+ handleAnnotationPointerOut();
+ onPointerOut();
+ }}
+ onClick={onClick}
+ {...restEventHandlers}
+ >
+ {content}
+
+
+
);
};
diff --git a/app/packages/looker-3d/src/labels/shared.tsx b/app/packages/looker-3d/src/labels/shared.tsx
index 984912229dc..5103cd686f3 100644
--- a/app/packages/looker-3d/src/labels/shared.tsx
+++ b/app/packages/looker-3d/src/labels/shared.tsx
@@ -1,16 +1,5 @@
-import type { useTooltip } from "@fiftyone/state";
-import { ThreeEvent } from "@react-three/fiber";
-import type { Vector3Tuple } from "three";
-import type { OverlayLabel } from "./loader";
-
-export interface OverlayProps {
- rotation: Vector3Tuple;
- opacity: number;
- tooltip: ReturnType;
- label: OverlayLabel;
- color: string;
- onClick: (e: ThreeEvent) => void;
+import type { BaseOverlayProps } from "../types";
+export interface OverlayProps extends BaseOverlayProps {
useLegacyCoordinates?: boolean;
- selected?: boolean;
}
diff --git a/app/packages/looker-3d/src/labels/shared/TransformControls.tsx b/app/packages/looker-3d/src/labels/shared/TransformControls.tsx
new file mode 100644
index 00000000000..b55927c930c
--- /dev/null
+++ b/app/packages/looker-3d/src/labels/shared/TransformControls.tsx
@@ -0,0 +1,115 @@
+import * as fos from "@fiftyone/state";
+import { TransformControls } from "@react-three/drei";
+import { useAtomValue } from "jotai";
+import { useCallback, useEffect, useRef } from "react";
+import { useRecoilState, useRecoilValue } from "recoil";
+import * as THREE from "three";
+import {
+ currentArchetypeSelectedForTransformAtom,
+ isCurrentlyTransformingAtom,
+ transformModeAtom,
+ transformSpaceAtom,
+} from "../../state";
+import type { TransformArchetype, TransformProps } from "../../types";
+
+type TransformableProps = {
+ archetype: TransformArchetype;
+ explicitObjectRef?: React.RefObject;
+ transformControlsPosition?: THREE.Vector3Tuple;
+ children: React.ReactNode;
+} & Pick<
+ TransformProps,
+ | "isSelectedForTransform"
+ | "onTransformStart"
+ | "onTransformEnd"
+ | "onTransformChange"
+ | "transformControlsRef"
+ | "translationSnap"
+ | "rotationSnap"
+ | "scaleSnap"
+ | "showX"
+ | "showY"
+ | "showZ"
+>;
+
+/**
+ * Shared component for rendering transform controls.
+ */
+export const Transformable = ({
+ archetype,
+ children,
+ explicitObjectRef,
+ isSelectedForTransform,
+ onTransformStart,
+ onTransformEnd,
+ onTransformChange,
+ transformControlsRef,
+ transformControlsPosition = [0, 0, 0],
+ ...transformControlsProps
+}: TransformableProps) => {
+ const groupRef = useRef(null);
+
+ const modalMode = useAtomValue(fos.modalMode);
+ const transformMode = useRecoilValue(transformModeAtom);
+ const transformSpace = useRecoilValue(transformSpaceAtom);
+ const currentArchetypeSelectedForTransform = useRecoilValue(
+ currentArchetypeSelectedForTransformAtom
+ );
+ const [isCurrentlyTransforming, setIsCurrentlyTransforming] = useRecoilState(
+ isCurrentlyTransformingAtom
+ );
+ const isAnnotateMode = modalMode === "annotate";
+
+ const onTransformStartDecorated = useCallback(() => {
+ setIsCurrentlyTransforming(true);
+ onTransformStart?.();
+ }, [onTransformStart, archetype]);
+
+ const onTransformEndDecorated = useCallback(() => {
+ setIsCurrentlyTransforming(false);
+ onTransformEnd?.();
+ }, [onTransformEnd, archetype]);
+
+ const onObjectChangeDecorated = useCallback(() => {
+ onTransformChange?.();
+ }, [onTransformChange]);
+
+ useEffect(() => {
+ const handleKeyDown = (event: KeyboardEvent) => {
+ if (event.key === "Escape" && isCurrentlyTransforming) {
+ setIsCurrentlyTransforming(false);
+ event.stopImmediatePropagation();
+ event.preventDefault();
+ }
+ };
+
+ document.addEventListener("keydown", handleKeyDown);
+
+ return () => {
+ document.removeEventListener("keydown", handleKeyDown);
+ };
+ }, [archetype, isCurrentlyTransforming]);
+
+ return (
+ <>
+ {explicitObjectRef ? children : {children}}
+
+ {isAnnotateMode &&
+ isSelectedForTransform &&
+ currentArchetypeSelectedForTransform === archetype && (
+
+
+
+ )}
+
+ );
+};
diff --git a/app/packages/looker-3d/src/labels/shared/hooks.ts b/app/packages/looker-3d/src/labels/shared/hooks.ts
new file mode 100644
index 00000000000..ec372825e50
--- /dev/null
+++ b/app/packages/looker-3d/src/labels/shared/hooks.ts
@@ -0,0 +1,69 @@
+import { useCursor } from "@react-three/drei";
+import { useMemo, useState } from "react";
+import { useRecoilValue } from "recoil";
+import { use3dLabelColor } from "../../hooks/use-3d-label-color";
+import { useSimilarLabels3d } from "../../hooks/use-similar-labels-3d";
+import { editSegmentsModeAtom, segmentPolylineStateAtom } from "../../state";
+import type { BaseOverlayProps, EventHandlers, HoverState } from "../../types";
+
+/**
+ * Custom hook for managing hover state and cursor behavior
+ */
+export const useHoverState = (): HoverState => {
+ const isSegmenting = useRecoilValue(segmentPolylineStateAtom).isActive;
+ const [isHovered, setIsHovered] = useState(false);
+ const isEditSegmentsMode = useRecoilValue(editSegmentsModeAtom);
+
+ useCursor(
+ isHovered,
+ isSegmenting || isEditSegmentsMode ? "crosshair" : "pointer",
+ isSegmenting ? "crosshair" : "auto"
+ );
+
+ return {
+ isHovered,
+ setIsHovered,
+ };
+};
+
+/**
+ * Custom hook for managing event handlers and tooltip integration
+ */
+export const useEventHandlers = (tooltip: any, label: any): EventHandlers => {
+ const { onPointerOver, onPointerOut, ...restEventHandlers } = useMemo(() => {
+ return {
+ ...tooltip.getMeshProps(label),
+ };
+ }, [tooltip, label]);
+
+ return {
+ onPointerOver,
+ onPointerOut,
+ restEventHandlers,
+ };
+};
+
+/**
+ * Custom hook for managing color logic with selection, hover, and similarity states
+ */
+export const useLabelColor = (
+ props: Pick,
+ isHovered: boolean,
+ label: any,
+ isSelectedForAnnotation?: boolean
+) => {
+ const isSimilarLabelHovered = useSimilarLabels3d(label);
+
+ const strokeAndFillColor = use3dLabelColor({
+ isSelected: props.selected,
+ isHovered,
+ isSimilarLabelHovered,
+ defaultColor: props.color,
+ isSelectedForAnnotation,
+ });
+
+ return {
+ strokeAndFillColor,
+ isSimilarLabelHovered,
+ };
+};
diff --git a/app/packages/looker-3d/src/labels/shared/index.ts b/app/packages/looker-3d/src/labels/shared/index.ts
new file mode 100644
index 00000000000..cf04505f1d7
--- /dev/null
+++ b/app/packages/looker-3d/src/labels/shared/index.ts
@@ -0,0 +1,2 @@
+export * from "./hooks";
+export * from "./TransformControls";
diff --git a/app/packages/looker-3d/src/renderables/pcd/index.tsx b/app/packages/looker-3d/src/renderables/pcd/index.tsx
index b3dd80d2d50..7fe795ea7f2 100644
--- a/app/packages/looker-3d/src/renderables/pcd/index.tsx
+++ b/app/packages/looker-3d/src/renderables/pcd/index.tsx
@@ -90,7 +90,7 @@ export const PointCloudMesh = ({
return null;
}
- if (minZ) {
+ if (minZ !== null && minZ !== undefined) {
return minZ;
}
diff --git a/app/packages/looker-3d/src/state.ts b/app/packages/looker-3d/src/state.ts
index da7c1e5f5a5..1aa22768384 100644
--- a/app/packages/looker-3d/src/state.ts
+++ b/app/packages/looker-3d/src/state.ts
@@ -6,10 +6,23 @@ import {
nullableModalSampleId,
} from "@fiftyone/state";
import { atom, atomFamily, selector } from "recoil";
-import { Vector3 } from "three";
+import { Vector3, Vector3Tuple } from "three";
+import type {
+ AnnotationPlaneState,
+ HoveredPolylineInfo,
+ PolylinePointTransform,
+ SegmentPolylineState,
+ SelectedPoint,
+ TempPolyline,
+ TransformedLabelData,
+ TransformMode,
+ TransformSpace,
+} from "./annotation/types";
import { SHADE_BY_HEIGHT } from "./constants";
import type { FoSceneNode } from "./hooks";
+import { OverlayLabel } from "./labels/loader";
import type { Actions, AssetLoadingLog, ShadeBy } from "./types";
+import { TransformArchetype } from "./types";
const fo3dAssetsParseStatusLog = atomFamily({
key: "fo3d-assetsParseStatusLogs",
@@ -42,7 +55,9 @@ export const cameraPositionAtom = atom<[number, number, number] | null>({
export const shadeByAtom = atom({
key: "fo3d-shadeBy",
default: SHADE_BY_HEIGHT,
- effects: [getBrowserStorageEffectForKey("shadeBy")],
+ effects: [
+ getBrowserStorageEffectForKey("shadeBy", { prependDatasetNameInKey: true }),
+ ],
});
export const customColorMapAtom = atom<{ [slice: string]: string } | null>({
@@ -51,6 +66,7 @@ export const customColorMapAtom = atom<{ [slice: string]: string } | null>({
effects: [
getBrowserStorageEffectForKey("customColorMap", {
useJsonSerialization: true,
+ prependDatasetNameInKey: true,
}),
],
});
@@ -81,7 +97,11 @@ export const currentActionAtom = atom({
export const currentPointSizeAtom = atom({
key: "fo3d-pointSize",
default: "2",
- effects: [getBrowserStorageEffectForKey("pointSize")],
+ effects: [
+ getBrowserStorageEffectForKey("pointSize", {
+ prependDatasetNameInKey: true,
+ }),
+ ],
});
export const pointSizeRangeAtom = atom({
@@ -114,6 +134,16 @@ export const isLevaConfigPanelOnAtom = atom({
default: false,
});
+export const annotationToolbarPositionAtom = atom({
+ key: "fo3d-annotationToolbarPosition",
+ default: 50,
+ effects: [
+ getBrowserStorageEffectForKey("fo3d-annotationToolbarPosition", {
+ valueClass: "number",
+ }),
+ ],
+});
+
export const gridCellSizeAtom = atom({
key: "fo3d-gridCellSize",
default: 1,
@@ -161,6 +191,7 @@ export const isFo3dBackgroundOnAtom = atom({
effects: [
getBrowserStorageEffectForKey("fo3d-isBackgroundON", {
valueClass: "boolean",
+ prependDatasetNameInKey: true,
}),
],
});
@@ -180,6 +211,17 @@ export const currentHoveredPointAtom = atom({
default: null,
});
+// Hover state for labels in annotate mode
+export const hoveredLabelAtom = atom({
+ key: "fo3d-hoveredLabel",
+ default: null,
+});
+
+export const hoveredPolylineInfoAtom = atom({
+ key: "fo3d-hoveredPolylineInfo",
+ default: null,
+});
+
export const cuboidLabelLineWidthAtom = atom({
key: "fo3d-cuboidLabelLineWidth",
default: 3,
@@ -199,3 +241,179 @@ export const polylineLabelLineWidthAtom = atom({
}),
],
});
+
+export const selectedLabelForAnnotationAtom = atom({
+ key: "fo3d-selectedLabelForAnnotation",
+ default: null,
+});
+
+export const transformModeAtom = atom({
+ key: "fo3d-transformMode",
+ default: "translate",
+});
+
+export const transformSpaceAtom = atom({
+ key: "fo3d-transformSpace",
+ default: "world",
+});
+
+export const currentArchetypeSelectedForTransformAtom =
+ atom({
+ key: "fo3d-currentArchetypeSelectedForTransformAtom",
+ default: null,
+ });
+
+// True if ANY entity is being actively transformed
+export const isCurrentlyTransformingAtom = atom({
+ key: "fo3d-isCurrentlyTransformingAtom",
+ default: false,
+});
+
+// todo: we sync this with backend
+export const transformedLabelsAtom = atom>(
+ {
+ key: "fo3d-transformedLabels",
+ default: {},
+ }
+);
+
+export const selectedPolylineVertexAtom = atom({
+ key: "fo3d-selectedPolylineVertexAtom",
+ default: null,
+});
+
+export const polylinePointTransformsAtom = atom<
+ Record
+>({
+ key: "fo3d-polylinePointTransforms",
+ default: {},
+});
+
+export const polylineEffectivePointsAtom = atomFamily(
+ {
+ key: "fo3d-polylineEffectivePoints",
+ default: [],
+ }
+);
+
+export const segmentPolylineStateAtom = atom({
+ key: "fo3d-segmentPolylineState",
+ default: {
+ isActive: false,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ },
+});
+
+export const isActivelySegmentingSelector = selector({
+ key: "fo3d-isActivelySegmentingSelector",
+ get: ({ get }) => {
+ return get(segmentPolylineStateAtom).isActive;
+ },
+});
+
+export const isSegmentingPointerDownAtom = atom({
+ key: "fo3d-isSegmentingPointerDownAtom",
+ default: false,
+});
+
+export const tempPolylinesAtom = atom({
+ key: "fo3d-tempPolylines",
+ default: [],
+});
+
+export const snapCloseAutomaticallyAtom = atom({
+ key: "fo3d-snapCloseAutomatically",
+ default: false,
+});
+
+export const editSegmentsModeAtom = atom({
+ key: "fo3d-editSegmentsMode",
+ default: false,
+});
+
+export const sharedCursorPositionAtom = atom<[number, number, number] | null>({
+ key: "fo3d-sharedCursorPosition",
+ default: null,
+});
+
+export const tempLabelTransformsAtom = atomFamily<
+ {
+ position: [number, number, number];
+ quaternion: [number, number, number, number];
+ } | null,
+ string
+>({
+ key: "fo3d-tempLabelTransforms",
+ default: null,
+});
+
+export const tempVertexTransformsAtom = atomFamily<
+ {
+ position: [number, number, number];
+ quaternion: [number, number, number, number];
+ } | null,
+ string
+>({
+ key: "fo3d-tempVertexTransforms",
+ default: null,
+});
+
+export const annotationPlaneAtom = atom({
+ key: "fo3d-annotationPlane",
+ default: {
+ enabled: false,
+ position: [0, 0, 0],
+ quaternion: [0, 0, 0, 1],
+ showX: true,
+ showY: true,
+ showZ: true,
+ },
+ effects: [
+ getBrowserStorageEffectForKey("fo3d-annotationPlane", {
+ useJsonSerialization: true,
+ sessionStorage: true,
+ }),
+ ],
+});
+
+export const isSnapToAnnotationPlaneAtom = atom({
+ key: "fo3d-isSnapToAnnotationPlane",
+ default: true,
+ effects: [
+ getBrowserStorageEffectForKey("fo3d-isSnapToAnnotationPlane", {
+ valueClass: "boolean",
+ }),
+ ],
+});
+
+export const avoidZFightingAtom = atom({
+ key: "fo3d-avoidZFighting",
+ default: true,
+ effects: [
+ getBrowserStorageEffectForKey("fo3d-avoidZFighting", {
+ valueClass: "boolean",
+ }),
+ ],
+});
+
+// Selector to clear all transform state
+export const clearTransformStateSelector = selector({
+ key: "fo3d-clearTransformState",
+ get: () => null,
+ set: ({ set }) => {
+ set(transformModeAtom, "translate");
+ set(transformSpaceAtom, "world");
+ set(selectedPolylineVertexAtom, null);
+ set(currentArchetypeSelectedForTransformAtom, null);
+ set(isCurrentlyTransformingAtom, false);
+ // Note: We don't clear polylinePointTransforms here as it should persist
+ set(segmentPolylineStateAtom, {
+ isActive: false,
+ vertices: [],
+ currentMousePosition: null,
+ isClosed: false,
+ });
+ },
+});
diff --git a/app/packages/looker-3d/src/types.ts b/app/packages/looker-3d/src/types.ts
index 13556a30320..010dfb44187 100644
--- a/app/packages/looker-3d/src/types.ts
+++ b/app/packages/looker-3d/src/types.ts
@@ -1,3 +1,6 @@
+import { TransformControlsProps } from "@react-three/drei";
+import type { RefObject } from "react";
+import * as THREE from "three";
import type {
ACTION_SET_PCDS,
ACTION_SET_POINT_SIZE,
@@ -10,6 +13,7 @@ import type {
SHADE_BY_NONE,
SHADE_BY_RGB,
} from "./constants";
+import { OverlayLabel } from "./labels/loader";
export type Actions =
| typeof ACTION_SHADE_BY
@@ -42,3 +46,38 @@ export type AssetLoadingLog = {
message: string;
status: "info" | "success" | "error";
};
+
+export interface BaseOverlayProps {
+ opacity: number;
+ rotation: THREE.Vector3Tuple;
+ selected: boolean;
+ onClick: (e: any) => void;
+ tooltip: any;
+ label: OverlayLabel;
+ color: string;
+}
+
+export interface TransformProps extends TransformControlsProps {
+ isSelectedForTransform?: boolean;
+ onTransformStart?: () => void;
+ onTransformEnd?: () => void;
+ onTransformChange?: () => void;
+ transformControlsRef?: RefObject;
+}
+
+export interface HoverState {
+ isHovered: boolean;
+ setIsHovered: (hovered: boolean) => void;
+}
+
+export interface EventHandlers {
+ onPointerOver: () => void;
+ onPointerOut: () => void;
+ restEventHandlers: Record;
+}
+
+export type TransformArchetype =
+ | "point"
+ | "cuboid"
+ | "polyline"
+ | "annotation-plane";
diff --git a/app/packages/looker-3d/src/utils.test.ts b/app/packages/looker-3d/src/utils.test.ts
index 2d994cea6a2..32a8cd83848 100644
--- a/app/packages/looker-3d/src/utils.test.ts
+++ b/app/packages/looker-3d/src/utils.test.ts
@@ -1,13 +1,24 @@
-import { BufferAttribute, Quaternion, Vector3 } from "three";
-import { describe, expect, it } from "vitest";
+import {
+ BufferAttribute,
+ PerspectiveCamera,
+ Plane,
+ Quaternion,
+ Raycaster,
+ Vector3,
+} from "three";
+import { describe, expect, it, vi } from "vitest";
import { COLOR_POOL } from "./constants";
import {
computeMinMaxForColorBufferAttribute,
computeMinMaxForScalarBufferAttribute,
+ createPlane,
deg2rad,
getColorFromPoolBasedOnHash,
getGridQuaternionFromUpVector,
+ getPlaneFromPositionAndQuaternion,
+ getPlaneIntersection,
toEulerFromDegreesArray,
+ toNDC,
} from "./utils";
describe("deg2rad", () => {
@@ -83,14 +94,311 @@ describe("getColorFromPoolBasedOnHash", () => {
});
describe("getGridQuaternionFromUpVector", () => {
- it("returns a quaternion for up vector", () => {
+ it("returns identity quaternion when up vector matches target normal", () => {
const up = new Vector3(0, 1, 0);
const result = getGridQuaternionFromUpVector(up);
- console.log(result);
expect(result).toBeInstanceOf(Quaternion);
- expect(result.w).toBe(1);
+ expect(result.w).toBeCloseTo(1);
expect(result.x).toBeCloseTo(0);
expect(result.y).toBeCloseTo(0);
expect(result.z).toBeCloseTo(0);
});
+
+ it("handles antiparallel up vector (opposite direction)", () => {
+ const up = new Vector3(0, -1, 0); // Opposite to target normal (0, 1, 0)
+ const result = getGridQuaternionFromUpVector(up);
+ expect(result).toBeInstanceOf(Quaternion);
+ // Should not be zero quaternion (0, 0, 0, 0) - at least one component should be non-zero
+ const hasNonZeroComponent =
+ result.w !== 0 || result.x !== 0 || result.y !== 0 || result.z !== 0;
+ expect(hasNonZeroComponent).toBe(true);
+ // Verify it's a valid quaternion (magnitude should be 1)
+ const magnitude = Math.sqrt(
+ result.w * result.w +
+ result.x * result.x +
+ result.y * result.y +
+ result.z * result.z
+ );
+ expect(magnitude).toBeCloseTo(1);
+ const targetNormal = new Vector3(0, 1, 0);
+ const rotated = up.clone().applyQuaternion(result);
+ expect(rotated.x).toBeCloseTo(targetNormal.x);
+ expect(rotated.y).toBeCloseTo(targetNormal.y);
+ expect(rotated.z).toBeCloseTo(targetNormal.z);
+ });
+
+ it("handles arbitrary up vector", () => {
+ const up = new Vector3(1, 0, 0).normalize();
+ const result = getGridQuaternionFromUpVector(up);
+ expect(result).toBeInstanceOf(Quaternion);
+ // Should not be zero quaternion
+ expect(result.w).not.toBe(0);
+ // Verify it's a valid quaternion
+ const magnitude = Math.sqrt(
+ result.w * result.w +
+ result.x * result.x +
+ result.y * result.y +
+ result.z * result.z
+ );
+ expect(magnitude).toBeCloseTo(1);
+ });
+
+ it("handles diagonal up vector", () => {
+ const up = new Vector3(1, 1, 0).normalize();
+ const result = getGridQuaternionFromUpVector(up);
+ expect(result).toBeInstanceOf(Quaternion);
+ // Should not be zero quaternion
+ expect(result.w).not.toBe(0);
+ // Verify it's a valid quaternion
+ const magnitude = Math.sqrt(
+ result.w * result.w +
+ result.x * result.x +
+ result.y * result.y +
+ result.z * result.z
+ );
+ expect(magnitude).toBeCloseTo(1);
+ });
+
+ it("handles custom target normal", () => {
+ const up = new Vector3(0, 0, 1);
+ const targetNormal = new Vector3(0, 0, 1);
+ const result = getGridQuaternionFromUpVector(up, targetNormal);
+ expect(result).toBeInstanceOf(Quaternion);
+ // Should be identity quaternion when vectors match
+ expect(result.w).toBeCloseTo(1);
+ expect(result.x).toBeCloseTo(0);
+ expect(result.y).toBeCloseTo(0);
+ expect(result.z).toBeCloseTo(0);
+ });
+
+ it("handles antiparallel with custom target normal", () => {
+ const up = new Vector3(0, 0, -1);
+ const targetNormal = new Vector3(0, 0, 1);
+ const result = getGridQuaternionFromUpVector(up, targetNormal);
+ expect(result).toBeInstanceOf(Quaternion);
+ // Should not be zero quaternion (0, 0, 0, 0) - at least one component should be non-zero
+ const hasNonZeroComponent =
+ result.w !== 0 || result.x !== 0 || result.y !== 0 || result.z !== 0;
+ expect(hasNonZeroComponent).toBe(true);
+ // Verify it's a valid quaternion (magnitude should be 1)
+ const magnitude = Math.sqrt(
+ result.w * result.w +
+ result.x * result.x +
+ result.y * result.y +
+ result.z * result.z
+ );
+ expect(magnitude).toBeCloseTo(1);
+ const rotated = up.clone().applyQuaternion(result);
+ expect(rotated.x).toBeCloseTo(targetNormal.x);
+ expect(rotated.y).toBeCloseTo(targetNormal.y);
+ expect(rotated.z).toBeCloseTo(targetNormal.z);
+ });
+});
+
+describe("toNDC", () => {
+ it("converts pointer event to normalized device coordinates", () => {
+ const mockCanvas = {
+ getBoundingClientRect: vi.fn().mockReturnValue({
+ left: 100,
+ top: 50,
+ width: 800,
+ height: 600,
+ }),
+ } as unknown as HTMLCanvasElement;
+
+ const mockEvent = {
+ clientX: 500,
+ clientY: 350,
+ } as PointerEvent;
+
+ const result = toNDC(mockEvent, mockCanvas);
+
+ // Center of canvas should be (0, 0) in NDC
+ expect(result.x).toBeCloseTo(0);
+ expect(result.y).toBeCloseTo(0);
+ });
+
+ it("converts corner coordinates correctly", () => {
+ const mockCanvas = {
+ getBoundingClientRect: vi.fn().mockReturnValue({
+ left: 0,
+ top: 0,
+ width: 100,
+ height: 100,
+ }),
+ } as unknown as HTMLCanvasElement;
+
+ // Top-left corner
+ const topLeftEvent = {
+ clientX: 0,
+ clientY: 0,
+ } as PointerEvent;
+ const topLeftResult = toNDC(topLeftEvent, mockCanvas);
+ expect(topLeftResult.x).toBeCloseTo(-1);
+ expect(topLeftResult.y).toBeCloseTo(1);
+
+ // Bottom-right corner
+ const bottomRightEvent = {
+ clientX: 100,
+ clientY: 100,
+ } as PointerEvent;
+ const bottomRightResult = toNDC(bottomRightEvent, mockCanvas);
+ expect(bottomRightResult.x).toBeCloseTo(1);
+ expect(bottomRightResult.y).toBeCloseTo(-1);
+ });
+});
+
+describe("createPlane", () => {
+ it("creates a plane with normalized normal and negative constant", () => {
+ const normal = new Vector3(2, 0, 0);
+ const constant = 5;
+ const plane = createPlane(normal, constant);
+
+ expect(plane).toBeInstanceOf(Plane);
+ expect(plane.normal.x).toBeCloseTo(1);
+ expect(plane.normal.y).toBeCloseTo(0);
+ expect(plane.normal.z).toBeCloseTo(0);
+ expect(plane.constant).toBe(-5);
+ });
+});
+
+describe("getPlaneIntersection", () => {
+ it("returns intersection point when ray intersects plane", () => {
+ const raycaster = new Raycaster();
+ const camera = new PerspectiveCamera(75, 1, 0.1, 1000);
+ camera.position.set(0, 0, 5);
+ camera.lookAt(0, 0, 0);
+
+ // XY plane at z=0
+ const plane = new Plane(new Vector3(0, 0, 1), 0);
+ // Center of screen
+ const ndc = { x: 0, y: 0 };
+
+ const result = getPlaneIntersection(raycaster, camera, ndc, plane);
+
+ expect(result).toBeInstanceOf(Vector3);
+ expect(result?.x).toBeCloseTo(0);
+ expect(result?.y).toBeCloseTo(0);
+ expect(result?.z).toBeCloseTo(0);
+ });
+
+ it("returns null when ray does not intersect plane", () => {
+ const raycaster = new Raycaster();
+ const camera = new PerspectiveCamera(75, 1, 0.1, 1000);
+ camera.position.set(0, 0, 5);
+ camera.lookAt(0, 0, 0);
+
+ // Create a plane that the ray will miss - plane is behind the camera and ray goes forward
+ // XY plane at z=-10 (behind camera)
+ const plane = new Plane(new Vector3(0, 0, -1), 10);
+ // Center of screen - ray goes forward from z=5 to z=0
+ const ndc = { x: 0, y: 0 };
+
+ const result = getPlaneIntersection(raycaster, camera, ndc, plane);
+
+ expect(result).toBeNull();
+ });
+});
+
+describe("getPlaneFromPositionAndQuaternion", () => {
+ it("creates plane with identity quaternion at origin", () => {
+ const position: [number, number, number] = [0, 0, 0];
+ const quaternion: [number, number, number, number] = [0, 0, 0, 1];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ expect(result.normal.x).toBeCloseTo(0);
+ expect(result.normal.y).toBeCloseTo(0);
+ expect(result.normal.z).toBeCloseTo(1);
+ expect(result.constant).toBeCloseTo(0);
+ });
+
+ it("creates plane with identity quaternion at offset position", () => {
+ const position: [number, number, number] = [1, 2, 3];
+ const quaternion: [number, number, number, number] = [0, 0, 0, 1];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ expect(result.normal.x).toBeCloseTo(0);
+ expect(result.normal.y).toBeCloseTo(0);
+ expect(result.normal.z).toBeCloseTo(1);
+ expect(result.constant).toBeCloseTo(-3);
+ });
+
+ it("creates plane with 90-degree rotation around Y-axis", () => {
+ const position: [number, number, number] = [0, 0, 0];
+ const quaternion: [number, number, number, number] = [
+ 0,
+ Math.sqrt(2) / 2,
+ 0,
+ Math.sqrt(2) / 2,
+ ];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ expect(result.normal.x).toBeCloseTo(1);
+ expect(result.normal.y).toBeCloseTo(0);
+ expect(result.normal.z).toBeCloseTo(0);
+ expect(result.constant).toBeCloseTo(0);
+ });
+
+ it("creates plane with 90-degree rotation around X-axis", () => {
+ const position: [number, number, number] = [0, 0, 0];
+ const quaternion: [number, number, number, number] = [
+ Math.sqrt(2) / 2,
+ 0,
+ 0,
+ Math.sqrt(2) / 2,
+ ];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ expect(result.normal.x).toBeCloseTo(0);
+ expect(result.normal.y).toBeCloseTo(-1);
+ expect(result.normal.z).toBeCloseTo(0);
+ expect(result.constant).toBeCloseTo(0);
+ });
+
+ it("creates plane with 180-degree rotation around X-axis", () => {
+ const position: [number, number, number] = [0, 0, 0];
+ const quaternion: [number, number, number, number] = [1, 0, 0, 0];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ expect(result.normal.x).toBeCloseTo(0);
+ expect(result.normal.y).toBeCloseTo(0);
+ expect(result.normal.z).toBeCloseTo(-1);
+ expect(result.constant).toBeCloseTo(0);
+ });
+
+ it("creates plane with arbitrary rotation and position", () => {
+ const position: [number, number, number] = [2, 3, 4];
+ const quaternion: [number, number, number, number] = [0.1, 0.2, 0.3, 0.9];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ // Verify the normal is normalized
+ const normalMagnitude = Math.sqrt(
+ result.normal.x * result.normal.x +
+ result.normal.y * result.normal.y +
+ result.normal.z * result.normal.z
+ );
+ expect(normalMagnitude).toBeCloseTo(1);
+ });
+
+ it("creates plane with diagonal normal from quaternion", () => {
+ const position: [number, number, number] = [0, 0, 0];
+ // Quaternion that rotates Z-axis to diagonal direction
+ const quaternion: [number, number, number, number] = [0.5, 0.5, 0.5, 0.5];
+ const result = getPlaneFromPositionAndQuaternion(position, quaternion);
+
+ expect(result).toBeInstanceOf(Plane);
+ // Verify the normal is normalized
+ const normalMagnitude = Math.sqrt(
+ result.normal.x * result.normal.x +
+ result.normal.y * result.normal.y +
+ result.normal.z * result.normal.z
+ );
+ expect(normalMagnitude).toBeCloseTo(1);
+ });
});
diff --git a/app/packages/looker-3d/src/utils.ts b/app/packages/looker-3d/src/utils.ts
index 6fe807fc5c0..581f26279e7 100644
--- a/app/packages/looker-3d/src/utils.ts
+++ b/app/packages/looker-3d/src/utils.ts
@@ -1,7 +1,11 @@
import {
type BufferAttribute,
+ Box3,
+ type Camera,
type InterleavedBufferAttribute,
+ Plane,
Quaternion,
+ type Raycaster,
Vector3,
type Vector3Tuple,
type Vector4Tuple,
@@ -86,6 +90,9 @@ class InvalidSceneError extends Error {
/**
* Reads a raw JSON scene from FiftyOne and returns counts
* of different media types in the scene.
+ *
+ * @param scene - The FiftyOne scene JSON object
+ * @returns Object containing counts of different media types
*/
export const getFiftyoneSceneSummary = (scene: FiftyoneSceneRawJson) => {
if (
@@ -136,11 +143,17 @@ export const getFiftyoneSceneSummary = (scene: FiftyoneSceneRawJson) => {
/**
* Converts degrees to radians.
+ *
+ * @param degrees - The angle in degrees
+ * @returns The angle in radians
*/
export const deg2rad = (degrees: number) => degrees * (Math.PI / 180);
/**
* Converts an array of degrees to an array of radians.
+ *
+ * @param degreesArr - Array of angles in degrees
+ * @returns Array of angles in radians
*/
export const toEulerFromDegreesArray = (degreesArr: Vector3Tuple) => {
return degreesArr.map(deg2rad) as Vector3Tuple;
@@ -148,6 +161,9 @@ export const toEulerFromDegreesArray = (degreesArr: Vector3Tuple) => {
/**
* Computes the min and max values for a color buffer attribute.
+ *
+ * @param colorAttribute - The color buffer attribute to analyze
+ * @returns Object containing min and max values
*/
export const computeMinMaxForColorBufferAttribute = (
colorAttribute: BufferAttribute | InterleavedBufferAttribute
@@ -166,6 +182,9 @@ export const computeMinMaxForColorBufferAttribute = (
/**
* Computes the min and max values for a scalar buffer attribute (like intensity in pcd)
+ *
+ * @param attribute - The scalar buffer attribute to analyze
+ * @returns Object containing min and max values
*/
export const computeMinMaxForScalarBufferAttribute = (
attribute: BufferAttribute | InterleavedBufferAttribute
@@ -181,6 +200,12 @@ export const computeMinMaxForScalarBufferAttribute = (
return { min: mn, max: mx };
};
+/**
+ * Gets a color from the color pool based on a string hash.
+ *
+ * @param str - The string to hash
+ * @returns A color from the color pool
+ */
export const getColorFromPoolBasedOnHash = (str: string) => {
const hash = str.split("").reduce((acc, char, idx) => {
const charCode = char.charCodeAt(0);
@@ -189,15 +214,130 @@ export const getColorFromPoolBasedOnHash = (str: string) => {
return COLOR_POOL[hash % COLOR_POOL.length];
};
-export const getGridQuaternionFromUpVector = (upVectorNormalized: Vector3) => {
- // calculate angle between custom up direction and default up direction (y-axis in three-js)
- const angle = Math.acos(upVectorNormalized.dot(new Vector3(0, 1, 0)));
+/**
+ * Calculates a quaternion to align a grid with a custom up vector.
+ *
+ * @param upVectorNormalized - The normalized up vector
+ * @returns A quaternion representing the rotation
+ */
+export const getGridQuaternionFromUpVector = (
+ upVectorNormalized: Vector3,
+ targetNormal: Vector3 = new Vector3(0, 1, 0)
+) => {
+ const from = targetNormal.clone().normalize();
+ const to = upVectorNormalized.clone();
+ return new Quaternion().setFromUnitVectors(from, to);
+};
+
+/**
+ * Converts a pointer event to Normalized Device Coordinates (NDC).
+ * NDC coordinates range from -1 to 1 in both x and y directions,
+ * with (0, 0) at the center of the canvas.
+ *
+ * @param ev - The pointer event to convert
+ * @param canvas - The HTML canvas element to get bounds from
+ * @returns An object with x and y coordinates in NDC space
+ */
+export function toNDC(ev: PointerEvent, canvas: HTMLCanvasElement) {
+ const rect = canvas.getBoundingClientRect();
+ return {
+ x: ((ev.clientX - rect.left) / rect.width) * 2 - 1,
+ y: -((ev.clientY - rect.top) / rect.height) * 2 + 1,
+ };
+}
- // calculate axis perpendicular to both the default up direction and the custom up direction
- const axis = new Vector3()
- .crossVectors(new Vector3(0, 1, 0), upVectorNormalized)
- .normalize();
+/**
+ * Calculates the intersection point between a ray and a plane.
+ *
+ * @param raycaster - The THREE.js raycaster instance
+ * @param camera - The THREE.js camera
+ * @param ndc - Normalized device coordinates
+ * @param plane - The THREE.js plane to intersect with
+ * @returns The intersection point if it exists, null otherwise
+ */
+export function getPlaneIntersection(
+ raycaster: Raycaster,
+ camera: Camera,
+ ndc: { x: number; y: number },
+ plane: Plane
+): Vector3 | null {
+ raycaster.setFromCamera(ndc as any, camera);
+ const point = new Vector3();
+ if (raycaster.ray.intersectPlane(plane, point)) {
+ return point;
+ }
+ return null;
+}
- // quaternion to represent the rotation around an axis perpendicular to both the default up direction and the custom up direction
- return new Quaternion().setFromAxisAngle(axis, angle);
-};
+/**
+ * Creates a THREE.js plane from normal and constant values.
+ *
+ * @param normal - The plane normal vector
+ * @param constant - The plane constant
+ * @returns A new THREE.js plane
+ */
+export function createPlane(normal: Vector3, constant: number): Plane {
+ return new Plane(normal.clone().normalize(), -constant);
+}
+
+/**
+ * Checks if a pointer event matches the specified button.
+ *
+ * @param ev - The pointer event to check
+ * @param button - The button number to match (0 = left, 1 = middle, 2 = right)
+ * @returns True if the button matches, false otherwise
+ */
+export function isButtonMatch(ev: PointerEvent, button: number): boolean {
+ return ev.button === button;
+}
+
+/**
+ * Creates a THREE.js plane from stored position and quaternion state.
+ *
+ * @param position - The plane position as [x, y, z]
+ * @param quaternion - The plane rotation as [x, y, z, w]
+ * @returns A new THREE.js plane
+ */
+export function getPlaneFromPositionAndQuaternion(
+ position: [number, number, number],
+ quaternion: [number, number, number, number]
+): Plane {
+ const pos = new Vector3(...position);
+ const quat = new Quaternion(...quaternion);
+
+ // Extract normal from quaternion by rotating the Z-axis
+ const normal = new Vector3(0, 0, 1).applyQuaternion(quat).normalize();
+
+ // Create plane with normal and position
+ const plane = new Plane();
+ plane.setFromNormalAndCoplanarPoint(normal, pos);
+
+ return plane;
+}
+
+/**
+ * Expands a bounding box by a specified safety margin factor.
+ * The expansion is applied uniformly in all directions from the center.
+ *
+ * @param boundingBox - The original bounding box to expand
+ * @param safetyMargin - The expansion factor (e.g., 1.5 for 1.5x expansion)
+ * @returns A new expanded bounding box
+ */
+export function expandBoundingBox(
+ boundingBox: Box3,
+ safetyMargin: number = 1.5
+): Box3 {
+ if (!boundingBox || boundingBox.isEmpty()) {
+ return boundingBox;
+ }
+
+ const center = boundingBox.getCenter(new Vector3());
+ const size = boundingBox.getSize(new Vector3());
+
+ const expandedSize = size.clone().multiplyScalar(safetyMargin);
+
+ const expandedBox = new Box3();
+ expandedBox.setFromCenterAndSize(center, expandedSize);
+
+ return expandedBox;
+}
diff --git a/app/packages/looker-3d/src/vite-env.d.ts b/app/packages/looker-3d/src/vite-env.d.ts
index 11f02fe2a00..b1f45c78666 100644
--- a/app/packages/looker-3d/src/vite-env.d.ts
+++ b/app/packages/looker-3d/src/vite-env.d.ts
@@ -1 +1,2 @@
///
+///
diff --git a/app/packages/state/src/recoil/atoms.ts b/app/packages/state/src/recoil/atoms.ts
index e13d3b49532..010b9ad8bf1 100644
--- a/app/packages/state/src/recoil/atoms.ts
+++ b/app/packages/state/src/recoil/atoms.ts
@@ -407,3 +407,8 @@ export const editingFieldAtom = atom({
key: "editingFieldAtom",
default: false,
});
+
+export const isInMultiPanelViewAtom = atom({
+ key: "isInMultiPanelViewAtom",
+ default: false,
+});
diff --git a/app/yarn.lock b/app/yarn.lock
index b260dd4d38b..0d895fa901c 100644
--- a/app/yarn.lock
+++ b/app/yarn.lock
@@ -2559,13 +2559,15 @@ __metadata:
"@biomejs/biome": "npm:^1.8.3"
"@emotion/react": "npm:^11.14.0"
"@emotion/styled": "npm:^11.14.0"
- "@react-three/drei": "npm:^10.6.0"
+ "@react-three/drei": "npm:^10.7.6"
"@react-three/fiber": "npm:^8.18.0"
+ "@types/chroma-js": "npm:^3.1.1"
"@types/node": "npm:^20.11.10"
"@types/react": "npm:^18.2.48"
"@types/react-dom": "npm:^18.2.18"
"@types/three": "npm:^0.176.0"
"@vitejs/plugin-react": "npm:^1.3.0"
+ chroma-js: "npm:^3.1.2"
leva: "npm:^0.9.36"
lodash: "npm:^4.17.21"
nodemon: "npm:^3.0.3"
@@ -4434,9 +4436,9 @@ __metadata:
languageName: node
linkType: hard
-"@react-three/drei@npm:^10.6.0":
- version: 10.6.0
- resolution: "@react-three/drei@npm:10.6.0"
+"@react-three/drei@npm:^10.7.6":
+ version: 10.7.6
+ resolution: "@react-three/drei@npm:10.7.6"
dependencies:
"@babel/runtime": "npm:^7.26.0"
"@mediapipe/tasks-vision": "npm:0.10.17"
@@ -4467,7 +4469,7 @@ __metadata:
peerDependenciesMeta:
react-dom:
optional: true
- checksum: 10/df42d7e515bde84f1e546e1757c8fdc115d13ee9e6f79e308edc0ecd411c5cfe570c31ebe50b3715e4902432544d9eba204acf760faefca613b31e857ce4fd99
+ checksum: 10/fd6efdb687cfa0bdceeaf4183dfad99e7a2dc1b153f803d24de861fea8d5c924d2b01f5a659a4942f051f56842f6ac193f62cc4d1e3d0032f0b847a4c6177ddf
languageName: node
linkType: hard
@@ -5377,6 +5379,13 @@ __metadata:
languageName: node
linkType: hard
+"@types/chroma-js@npm:^3.1.1":
+ version: 3.1.1
+ resolution: "@types/chroma-js@npm:3.1.1"
+ checksum: 10/24770bd645425a1d90e3651dbb18bc5977290a2d2fc6e9f32d3892d4c43c1e9f554fb324cd0f4e20a23b734ee652278330a0d331973a7da824e0c15192ba129b
+ languageName: node
+ linkType: hard
+
"@types/color-string@npm:^1.5.0":
version: 1.5.5
resolution: "@types/color-string@npm:1.5.5"
@@ -7840,6 +7849,13 @@ __metadata:
languageName: node
linkType: hard
+"chroma-js@npm:^3.1.2":
+ version: 3.1.2
+ resolution: "chroma-js@npm:3.1.2"
+ checksum: 10/ea09b27d04b477a8fdc3314bfa6dc8de05feab47999ff1acdeba6cf3c32c76338baa5b4562d949d33d8adeca57754e5ea591c15b9985d65583c8771e7e07c2ef
+ languageName: node
+ linkType: hard
+
"ci-info@npm:^3.2.0":
version: 3.9.0
resolution: "ci-info@npm:3.9.0"
diff --git a/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-darwin.png b/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-darwin.png
index dcf074a2f3a..68d6ee9f1b7 100644
Binary files a/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-darwin.png and b/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-darwin.png differ
diff --git a/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-linux.png b/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-linux.png
index dcf074a2f3a..68d6ee9f1b7 100644
Binary files a/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-linux.png and b/e2e-pw/src/oss/specs/3d/fo3d-ply.spec.ts-snapshots/ply-scene-top-view-chromium-linux.png differ