Support mid-drag modifier changes

src/viser/_scene_handles.py

@@ -475,9 +475,16 @@ class SceneNodeDragEvent(Generic[TSceneNodeHandle]):
     """Scene node that is being dragged."""
     phase: DragPhase
     """Drag lifecycle phase: ``"start"`` at press, ``"update"`` on
-    every throttled pointermove (~20Hz), ``"end"`` at release. A
-    single drag fires exactly one ``"start"``, zero or more
-    ``"update"``s, and exactly one ``"end"``."""
+    every throttled pointermove (~20Hz), ``"end"`` at release.
+
+    A gesture is partitioned into one *segment* per held modifier-combo.
+    Each segment fires exactly one ``"start"``, zero or more
+    ``"update"``s, and exactly one ``"end"``. If the user changes the
+    held modifier mid-drag, the current segment ends and a new one
+    starts under the new modifier (see :attr:`modifier`) -- so a single
+    physical drag can produce more than one ``start``/``end`` pair. When
+    the modifier doesn't change, this collapses to the common case of a
+    single ``start`` ... ``end`` per gesture."""
     instance_index: int | None
     """Instance index within a batched scene node (e.g. batched meshes,
     batched GLBs, batched axes); ``None`` for non-batched nodes. Frozen
@@ -498,9 +505,12 @@ class SceneNodeDragEvent(Generic[TSceneNodeHandle]):
     button: Literal["left", "middle", "right"]
     """Mouse button that initiated the drag."""
     modifier: _messages.KeyModifier | None
-    """Modifier-combo held at drag-start (frozen for the drag's
-    lifetime). ``None`` if no modifiers were held; otherwise a
-    canonical :data:`KeyModifier` string."""
+    """Modifier-combo that owns the current drag segment. Constant within
+    a segment and matches the binding this callback was registered for;
+    if the user changes the held modifier mid-drag the segment ends and a
+    new one begins under the new combo (see :attr:`phase`). ``None`` if no
+    modifiers are held; otherwise a canonical :data:`KeyModifier`
+    string."""
 
 
 _VALID_DRAG_BUTTONS: Tuple[_messages.DragButton, ...] = get_args(_messages.DragButton)
@@ -611,12 +621,21 @@ def on_drag(
     ) -> Any:
         """Attach a callback for the full drag lifecycle.
 
-        Fires three times per gesture: once with
-        ``event.phase == "start"`` at press, zero or more times with
-        ``"update"`` (throttled pointermove), once with ``"end"`` at
-        release. ``end`` fires even on cancellation paths (window
-        blur, pointer cancel, node removed mid-drag) so per-drag
-        state can be released.
+        Fires once with ``event.phase == "start"`` at press, zero or
+        more times with ``"update"`` (throttled pointermove), and once
+        with ``"end"`` at release. ``end`` fires even on cancellation
+        paths (window blur, pointer cancel, node removed mid-drag) so
+        per-drag state can be released.
+
+        Modifiers are live: if the user changes the held modifier
+        mid-drag, the current segment ends and a new one begins under
+        the new combo, routing to whichever callback that combo is bound
+        to. A callback therefore sees a clean ``start`` ... ``end`` pair
+        for *its* modifier each time that modifier is engaged, and a
+        single physical drag may fire more than one such pair. To switch
+        behavior mid-drag (e.g. changing the drag plane), register a
+        separate ``on_drag`` for each modifier combo. ``event.modifier``
+        identifies the active segment.
 
         Usable as a bare decorator (``@handle.on_drag``, defaults to
         ``button="left"`` and no modifiers) or with arguments
@@ -629,9 +648,12 @@ def on_drag(
                 ordered ``"+"``-separated string like ``"cmd/ctrl"``,
                 ``"shift"``, or ``"cmd/ctrl+shift"``. ``None`` matches
                 "no modifiers held". Matching is exact: listed modifiers
-                must be held and others must not be. Left-drag on this
-                node intercepts the gesture -- the camera only orbits on
-                empty-space drags.
+                must be held and others must not be. The match is
+                re-evaluated whenever the held modifier changes mid-drag,
+                so this callback is entered and exited as its combo is
+                engaged and released. Left-drag on this node intercepts
+                the gesture -- the camera only orbits on empty-space
+                drags.
 
         Note on ordering: synchronous (``def``) callbacks are submitted
         to a thread pool fire-and-forget and can run out of order -- an

src/viser/client/src/DragLayer.tsx

@@ -31,9 +31,12 @@ import { useThrottledMessageSender } from "./WebsocketUtils";
 import {
   ActiveDragState,
   DragScratches,
+  KeyModifier,
   anyBindingMatches,
   computeInstanceWorldMatrix,
   isInstancedMesh2VendoredMessage,
+  keyModifierFromEvent,
+  planModifierTransition,
 } from "./dragUtils";
 import { DragLayerApi, DragLayerContext } from "./dragLayerContext";
 import { useDragArrow } from "./useDragArrow";
@@ -231,21 +234,58 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
 
   // Build + send a drag message in one call, no-op if ``buildDragMessage``
   // returns null (live grab point unavailable). Wraps the duplicated
-  // null-check pattern at the three send sites. */
+  // null-check pattern at the send sites. Returns whether a message was
+  // actually sent -- callers use this to keep ``segmentActive`` honest
+  // (a ``start`` that couldn't build must not be paired with a later
+  // ``end``). */
   const sendDragMessage = React.useCallback(
     (
       activeDrag: ActiveDragState,
       phase: "start" | "update" | "end",
       throttle: boolean,
-    ) => {
+    ): boolean => {
       const message = buildDragMessage(activeDrag, phase);
-      if (message === null) return;
+      if (message === null) return false;
       if (throttle) sendDragsThrottled(message);
       else viewerMutable.sendMessage(message);
+      return true;
     },
     [buildDragMessage, sendDragsThrottled, viewerMutable],
   );
 
+  // Apply a mid-drag modifier change. Ends the current segment (if any),
+  // switches ownership to ``nextModifier``, and starts a fresh segment
+  // when the new combo is bound. Geometry is untouched, so the drag
+  // continues without a visual jump -- only the addressed callback set
+  // changes. Dormant (unbound) modifiers send nothing; see
+  // ``planModifierTransition``. */
+  const transitionDragModifier = React.useCallback(
+    (activeDrag: ActiveDragState, nextModifier: KeyModifier | null) => {
+      const plan = planModifierTransition(
+        activeDrag.input.modifier,
+        nextModifier,
+        activeDrag.bindings,
+        activeDrag.input.button,
+        activeDrag.segmentActive,
+      );
+      if (plan === null) return;
+      if (plan.emitEnd) {
+        // Flush queued throttled updates so the old segment's pending
+        // update lands *before* its synthetic end -- preserves wire
+        // ordering across the segment boundary.
+        flushDragsThrottled();
+        sendDragMessage(activeDrag, "end", false);
+      }
+      // Switch ownership before emitting the new ``start`` so the start
+      // message carries the new modifier. ``button`` is unchanged.
+      activeDrag.input = { ...activeDrag.input, modifier: nextModifier };
+      activeDrag.segmentActive = plan.emitStart
+        ? sendDragMessage(activeDrag, "start", false)
+        : false;
+    },
+    [flushDragsThrottled, sendDragMessage],
+  );
+
   type EndInfo = {
     clientX: number;
     clientY: number;
@@ -264,15 +304,14 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
       }
 
       flushDragsThrottled();
-      if (sendEndMessage) {
-        // Modifier state is frozen at drag_start: a drag is "owned" by
-        // whichever (button, modifiers) combo was held when the user
-        // pressed the mouse, and stays owned by that combo until release
-        // regardless of modifier changes mid-drag. This guarantees the
-        // drag_start / drag_end callbacks see the same dispatch and
-        // avoids a class of footguns where a key-up arrives a beat
-        // before mouse-up (downgrading the gesture at the last moment)
-        // or a modifier is accidentally pressed mid-drag.
+      if (sendEndMessage && activeDrag.segmentActive) {
+        // End the currently-active segment. A drag is partitioned into
+        // one segment per (button, modifier) combo; the modifier can
+        // switch mid-drag (see ``transitionDragModifier``), and each
+        // switch already emitted the prior segment's ``end``. So here we
+        // only emit when a segment is still active -- a release while
+        // dormant (current modifier matches no binding) has nothing left
+        // to end.
         sendDragMessage(activeDrag, "end", false);
       }
 
@@ -362,16 +401,42 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
           if (event.pointerId !== activeDrag.pointerId) return;
           if (!updateActiveDragEnd(event.clientX, event.clientY)) return;
 
-          // Modifier/button state is frozen at drag_start and reused on
-          // every update/end -- see the note in `stopActiveDrag` above.
+          // A modifier change carried on this move ends the current
+          // segment and starts a new one under the new combo. Run it
+          // *after* refreshing the end position so the synthetic end
+          // reports the latest pointer location, and *before* the update
+          // so the update is attributed to the new segment.
+          const liveModifier = keyModifierFromEvent(event);
+          if (liveModifier !== activeDrag.input.modifier) {
+            transitionDragModifier(activeDrag, liveModifier);
+          }
+
           // ``start_position`` is recomputed live inside buildDragMessage,
           // so the wire payload always reflects the click point's
-          // current world position (tracking the moving object).
-          sendDragMessage(activeDrag, "update", true);
+          // current world position (tracking the moving object). Skip
+          // while dormant -- the current modifier matches no binding.
+          if (activeDrag.segmentActive) {
+            sendDragMessage(activeDrag, "update", true);
+          }
           // The per-frame useFrame updates the arrow tail from target
           // transforms; no manual update needed here.
         };
 
+        // Modifier changes can also arrive with the pointer stationary
+        // (the user taps a modifier key without moving the mouse). Listen
+        // for key transitions during the drag and re-evaluate ownership
+        // using the last-known pointer position. ``keyModifierFromEvent``
+        // reads ``ctrl/meta/shift/alt`` off the KeyboardEvent, so both
+        // keydown and keyup resolve the current combo.
+        const handleWindowKeyChange = (event: KeyboardEvent) => {
+          const activeDrag = activeDragRef.current;
+          if (activeDrag === null) return;
+          const liveModifier = keyModifierFromEvent(event);
+          if (liveModifier !== activeDrag.input.modifier) {
+            transitionDragModifier(activeDrag, liveModifier);
+          }
+        };
+
         const handleWindowPointerUp = (event: PointerEvent) => {
           // Ignore mismatched pointers -- we only end the drag when the
           // *same* pointer that started it lifts up (or cancels).
@@ -393,6 +458,8 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
           window.removeEventListener("pointerup", handleWindowPointerUp);
           window.removeEventListener("pointercancel", handleWindowPointerUp);
           window.removeEventListener("blur", handleWindowBlur);
+          window.removeEventListener("keydown", handleWindowKeyChange);
+          window.removeEventListener("keyup", handleWindowKeyChange);
         };
 
         // Plane parallel to the camera image plane, through the start
@@ -424,6 +491,13 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
           endPointWorld: startWorld.clone(),
           endPointerXy: [pointerXy[0], pointerXy[1]],
           input,
+          bindings,
+          // Set from the initial ``start`` send below. The pointerdown
+          // path only calls ``beginDrag`` when ``input`` matches a
+          // binding, so the opening segment is bound -- but the send can
+          // still fail if the live grab point is unavailable, so we trust
+          // its return value rather than assuming ``true``.
+          segmentActive: false,
           releaseCameraLock: null,
           cleanup,
         };
@@ -434,7 +508,13 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
         window.addEventListener("pointerup", handleWindowPointerUp);
         window.addEventListener("pointercancel", handleWindowPointerUp);
         window.addEventListener("blur", handleWindowBlur);
-        sendDragMessage(activeDragRef.current, "start", false);
+        window.addEventListener("keydown", handleWindowKeyChange);
+        window.addEventListener("keyup", handleWindowKeyChange);
+        activeDragRef.current.segmentActive = sendDragMessage(
+          activeDragRef.current,
+          "start",
+          false,
+        );
         return true;
       },
       stopIfNodeIs: (nodeName) => {
@@ -449,6 +529,7 @@ function DragLayerActive({ children }: { children?: React.ReactNode }) {
       frameScratches,
       sendDragMessage,
       stopActiveDrag,
+      transitionDragModifier,
       updateActiveDragEnd,
       viewer,
       viewerMutable,

src/viser/client/src/dragUtils.test.ts

@@ -7,6 +7,7 @@ import {
   anyBindingMatches,
   hasCmdCtrl,
   motionExceedsThreshold,
+  planModifierTransition,
   MOTION_THRESHOLD_PX,
 } from "./dragUtils";
 
@@ -91,6 +92,58 @@ describe("hasCmdCtrl", () => {
   });
 });
 
+describe("planModifierTransition", () => {
+  // mjviser-style setup: two bound combos on the left button, used to
+  // switch the drag plane mid-gesture.
+  const bindings = [
+    { button: "left" as const, modifier: "cmd/ctrl" as const },
+    { button: "left" as const, modifier: "cmd/ctrl+shift" as const },
+  ];
+
+  it("is a no-op when the modifier is unchanged", () => {
+    expect(
+      planModifierTransition("cmd/ctrl", "cmd/ctrl", bindings, "left", true),
+    ).toBeNull();
+    // Even when nothing is held and nothing changes.
+    expect(planModifierTransition(null, null, bindings, "left", false)).toBeNull();
+  });
+
+  it("ends the old segment and starts a new one between two bound combos", () => {
+    expect(
+      planModifierTransition("cmd/ctrl", "cmd/ctrl+shift", bindings, "left", true),
+    ).toEqual({ emitEnd: true, emitStart: true });
+  });
+
+  it("ends the segment and goes dormant when the new combo is unbound", () => {
+    // ctrl is bound, shift-only is not -- releasing ctrl mid-drag drops
+    // into a dormant gap rather than starting a spurious segment.
+    expect(
+      planModifierTransition("cmd/ctrl", "shift", bindings, "left", true),
+    ).toEqual({ emitEnd: true, emitStart: false });
+  });
+
+  it("starts a fresh segment when re-entering a bound combo from dormant", () => {
+    // Already dormant (segmentActive=false): no end to emit, just the
+    // new start.
+    expect(
+      planModifierTransition("shift", "cmd/ctrl", bindings, "left", false),
+    ).toEqual({ emitEnd: false, emitStart: true });
+  });
+
+  it("stays dormant when moving between two unbound combos", () => {
+    expect(
+      planModifierTransition("shift", "alt", bindings, "left", false),
+    ).toEqual({ emitEnd: false, emitStart: false });
+  });
+
+  it("respects the button when matching the new combo", () => {
+    // The same modifier on a button with no binding is unbound.
+    expect(
+      planModifierTransition(null, "cmd/ctrl", bindings, "right", false),
+    ).toEqual({ emitEnd: false, emitStart: false });
+  });
+});
+
 describe("motionExceedsThreshold", () => {
   it("is false for movement at or under the threshold (L-infinity)", () => {
     expect(motionExceedsThreshold([0, 0], [0, 0])).toBe(false);