-
Notifications
You must be signed in to change notification settings - Fork 2.8k
Expand file tree
/
Copy pathplugin.py
More file actions
1211 lines (1046 loc) · 54 KB
/
Copy pathplugin.py
File metadata and controls
1211 lines (1046 loc) · 54 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
ImGui overlay plugin for joint control and simulation controls.
Requires the ``render`` optional extras: ``pip install 'genesis-world[render]'``.
"""
import os
import time
from typing import TYPE_CHECKING, Any
import numpy as np
from scipy.spatial.transform import Rotation as R
import genesis as gs
import genesis.utils.geom as gu
from genesis.engine.interactive_scene import InteractiveFeature, InteractiveScene
from genesis.ext.pyrender.overlay.style import apply_dark_theme
from genesis.ext.pyrender.overlay.types import build_entity_joint_data, EntityCacheEntry, EntityJointData
from genesis.utils.misc import tensor_to_array
from genesis.vis.viewer_plugins import EVENT_HANDLE_STATE, EVENT_HANDLED, ViewerPlugin
if TYPE_CHECKING:
from genesis.engine.entities.rigid_entity import RigidEntity
from genesis.engine.scene import Scene
from genesis.ext.pyrender.viewer import Viewer
_FPS_HISTORY_SIZE = 30
_MORPH_TYPES = ["URDF", "MJCF", "Mesh", "Box", "Sphere", "Cylinder", "Plane"]
def button_size_with_min(imgui, label: str, min_width: float) -> tuple[float, float]:
"""Return a ``(width, height)`` size tuple for ``imgui.button(label, size=...)`` that auto-fits the label but
never drops below ``min_width``. Height stays 0 so ImGui picks its default."""
text_width = imgui.calc_text_size(label).x
return max(min_width, text_width + 2.0 * imgui.get_style().frame_padding.x), 0.0
def draw_separator(imgui, thickness: int = 2) -> None:
"""Draw a ``thickness`` px-tall horizontal separator at integer-pixel coordinates. ImGui's built-in
``Separator()`` draws a 1 px line centered on a half-pixel boundary, which different OpenGL drivers rasterize
onto different rows; this helper uses a filled rectangle aligned to integer rows so the line is byte-identical
on every renderer."""
draw_list = imgui.get_window_draw_list()
x0, y = imgui.get_cursor_screen_pos()
x0_i, y_i = int(x0), int(y)
x1_i = x0_i + int(imgui.get_content_region_avail().x)
color = imgui.get_color_u32(imgui.Col_.separator.value)
draw_list.add_rect_filled((x0_i, y_i), (x1_i, y_i + thickness), color)
imgui.dummy((0.0, float(thickness)))
class ImGuiOverlayPlugin(ViewerPlugin):
"""
ViewerPlugin that adds an ImGui control panel for simulation and joint control.
Features:
- Simulation controls: play/pause, step, reset
- Joint sliders for each entity (editable only when paused)
- FPS display with rolling average
- Multi-step support
- Custom panel registration API
Limitations:
- Only controls environment 0 in batched simulations
Usage:
scene = gs.Scene(viewer_options=gs.options.ViewerOptions(enable_gui=True), show_viewer=True)
scene.build()
while scene.viewer.is_alive():
scene.step()
The overlay drives play/pause and scene rebuild through an InteractiveScene it wraps the scene with, so
the loop just calls ``scene.step()``; the controls take effect there on the stepping thread.
"""
def __init__(
self,
controlled_env_idx: int = 0,
free_joint_pos_limit: float = 10.0,
panel_width: int | None = None,
):
try:
import imgui_bundle # noqa: F401
except ImportError:
gs.raise_exception(
"ImGuiOverlayPlugin requires the optional 'imgui-bundle' dependency. Install Genesis with the "
"'render' extras (pip install 'genesis-world[render]'). Pre-built wheels are not published for every "
"Python/OS combination (e.g. Python 3.10, Linux aarch64); on those platforms install manually via "
"'pip install imgui-bundle', which builds from source and requires CMake."
)
super().__init__()
# InteractiveScene wrapping the current scene, created and owned by this plugin in build(). It backs
# the scene-editing controls (feature gating + rebuild) so the user never instantiates one manually.
self._interactive_scene = None
self._controlled_env_idx = controlled_env_idx
self._free_joint_pos_limit = free_joint_pos_limit
self._panel_width = panel_width
self._imgui = None
self._ctx = None
self._impl = None
self._io = None
self._available = False
self._init_attempted = False
self._last_time = None
# Number of frames a single "Step" click advances. Play/pause and stepping state itself live on the
# InteractiveScene (the controller); this overlay only toggles them.
self._step_count = 1
self._entity_cache = {}
self._user_panels = []
self._fps_history = []
# Name of the tab to force-select on the next frame, then cleared. None leaves the user's selection.
self._active_tab = None
# The Scene editor panel mutates this local mirror of the live entities; on "Rebuild Scene" it is
# submitted to the InteractiveScene, which applies it on the stepping thread.
self._pending_dirty = False
self._pending_entities_kwargs: dict[str, dict] = {}
self._add_entity_file = ""
self._add_entity_morph_type = 0 # index into _MORPH_TYPES
self._add_entity_pos = [0.0, 0.0, 0.0]
self._add_entity_scale = 1.0
# Type-specific geometry params
self._add_box_size = [0.2, 0.2, 0.2]
self._add_sphere_radius = 0.1
self._add_cylinder_radius = 0.05
self._add_cylinder_height = 0.2
self._add_entity_fixed = True
# File browser state
self._file_browser_open = False
self._file_browser_dir = os.getcwd()
self._file_browser_selected = -1
# Gizmo state
self._gizmo = None # imguizmo.im_guizmo module (lazy loaded)
self._gizmo_operation = None # gizmo.OPERATION.translate
self._gizmo_mode = None # gizmo.MODE.world
self._gizmo_entity_idx = -1 # which free-joint entity is selected for gizmo
self._gizmo_cached_matrix = None # cached 4x4 object matrix while dragging (avoids qpos round-trip jitter)
# Per-entity euler/quat mode: entity_idx -> "euler" or "quat"
self._rotation_mode = {}
# Per-entity wireframe state: entity_idx -> bool
self._wireframe_state = {}
def register_panel(self, callback, section="side"):
"""Register custom UI panel. callback(imgui) called each frame.
Thread-safe: uses copy-on-write list.
Args:
callback: Function taking imgui module as argument, called each frame.
section: "side" adds to main panel, "overlay" creates floating window.
"""
new_list = list(self._user_panels) + [(callback, section)]
self._user_panels = new_list # Atomic reference swap
def build(self, viewer: "Viewer", camera, scene: "Scene"):
"""Store references; ImGui initialization is deferred to on_draw (viewer thread)."""
super().build(viewer, camera, scene)
# Reset ImGui state so it re-initializes in the new viewer thread
# (needed after scene rebuild creates a new viewer/OpenGL context)
# Don't destroy the old context here, it belonged to the old viewer
# thread and is already invalid after scene.destroy().
if self._init_attempted:
self._impl = None
self._io = None
self._available = False
self._init_attempted = False
self._last_time = None
# Non-batched scenes (n_envs == 0) expect envs_idx=None on entity setters and return 1D qpos
# tensors. Collapse the controlled index to None so downstream code can pass it through
# unconditionally without branching on the batched/non-batched shape.
if scene.n_envs == 0:
self._controlled_env_idx = None
elif self._controlled_env_idx is not None and not (0 <= self._controlled_env_idx < scene.n_envs):
gs.raise_exception(
f"controlled_env_idx={self._controlled_env_idx} out of range for scene with n_envs={scene.n_envs}."
)
# Cache entity data now (doesn't require OpenGL)
self._cache_entity_data()
self._capture_pending_entities_kwargs()
# Wrap the scene on first build. The InteractiveScene is the controller (it owns play/pause, stepping
# and rebuild); this overlay is just a view that reads and toggles its state. On a rebuild the plugin is
# re-attached to the reconstructed scene (same object), so the existing wrapper still applies.
if self._interactive_scene is None:
self._interactive_scene = InteractiveScene(scene)
@property
def _supported_features(self) -> frozenset[InteractiveFeature]:
"""Editing features advertised by the plugin's InteractiveScene for the current simulator mode,
queried live. Each scene-editing control gates on its own feature and renders disabled when absent."""
return self._interactive_scene.supported_features
def _refresh_visuals(self):
"""Refresh render transforms after a GUI-driven mutation. Caller must hold the render lock."""
rigid_solver = self.scene.rigid_solver
if not rigid_solver.is_active:
return
rigid_solver.update_geoms_render_T()
rigid_solver.update_vgeoms()
rigid_solver.update_vgeoms_render_T()
ctx = self.viewer.gs_context
ctx.update_link_frame()
ctx.update_rigid()
def _apply_entity_vis_mode(self, entity, mode: str):
"""Switch the entity's rendered mesh between ``"visual"`` and ``"collision"``. Removes the previous
render nodes from the context, swaps ``entity.surface.vis_mode``, then rebuilds nodes from the
appropriate geom set."""
from genesis.ext import pyrender
if not isinstance(entity.surface, gs.surfaces.Surface):
return
old_mode = entity.surface.vis_mode
if old_mode == mode:
return
with self.viewer.render_lock:
ctx = self.viewer.gs_context
rigid_solver = self.scene.rigid_solver
old_geoms = entity.vgeoms if old_mode == "visual" else entity.geoms
for geom in old_geoms:
if geom.uid in ctx.rigid_nodes:
ctx.remove_node(ctx.rigid_nodes[geom.uid])
del ctx.rigid_nodes[geom.uid]
entity.surface.vis_mode = mode
self._refresh_visuals()
is_collision = mode == "collision"
geoms, geoms_T = (
(entity.vgeoms, rigid_solver._vgeoms_render_T)
if mode == "visual"
else (entity.geoms, rigid_solver._geoms_render_T)
)
for geom in geoms:
geom_envs_idx = ctx._get_geom_active_envs_idx(geom, ctx.rendered_envs_idx)
if len(geom_envs_idx) == 0:
continue
ctx.add_rigid_node(
geom,
pyrender.Mesh.from_trimesh(
mesh=geom.get_trimesh(),
poses=geoms_T[geom.idx][geom_envs_idx],
smooth=geom.surface.smooth if not is_collision else False,
double_sided=geom.surface.double_sided if not is_collision else False,
is_floor=isinstance(entity._morph, gs.morphs.Plane),
env_shared=not ctx.env_separate_rigid,
),
)
def _init_imgui(self):
"""Initialize ImGui. Must be called from the viewer thread (e.g., in on_draw)."""
if self._init_attempted:
return
self._init_attempted = True
try:
from imgui_bundle import imgui
from imgui_bundle.python_backends import pyglet_backend
self._imgui = imgui
self._ctx = imgui.create_context()
# Load default font at larger size before renderer builds the atlas
io = imgui.get_io()
io.fonts.clear()
font_cfg = imgui.ImFontConfig()
font_cfg.size_pixels = 18.0
io.fonts.add_font_default(font_cfg)
self._impl = pyglet_backend.create_renderer(self.viewer, attach_callbacks=False)
# Fix: Set window reference for cursor handling (not set when attach_callbacks=False)
self._impl._window = self.viewer
self._io = imgui.get_io()
self._io.set_ini_filename("") # Don't persist window positions
# Render the first frame as if the window is unfocused so ImGui's keyboard nav does not auto-pick
# the first focusable widget and draw a nav highlight on top of it. Pyglet's Win32 backend reports
# the window as focused at startup, which would otherwise leave the first entity header with a
# visible highlight until the user interacts. Subsequent focus events from pyglet (mouse click,
# key press, etc.) restore normal focus behavior on demand.
self._io.add_focus_event(False)
# Set up clipboard (pyglet backend doesn't do this by default)
# Pyglet caches _clipboard_str and only clears it on SelectionClear
# events, which may not be dispatched in time. Invalidate the cache
# before each read so we always get fresh system clipboard content.
window_ref = self.viewer
def _get_clipboard(_ctx):
try:
window_ref._clipboard_str = None
text = window_ref.get_clipboard_text()
return text if text else ""
except Exception:
return ""
def _set_clipboard(_ctx, text):
try:
window_ref.set_clipboard_text(text)
except Exception:
pass
platform_io = imgui.get_platform_io()
platform_io.platform_get_clipboard_text_fn = _get_clipboard
platform_io.platform_set_clipboard_text_fn = _set_clipboard
apply_dark_theme(imgui)
self._available = True
# Try to load ImGuizmo for 3D gizmos
try:
from imgui_bundle import imguizmo
self._gizmo = imguizmo.im_guizmo
self._gizmo_operation = self._gizmo.OPERATION.translate
self._gizmo_mode = self._gizmo.MODE.world
self._gizmo.set_gizmo_size_clip_space(0.15)
self._gizmo.allow_axis_flip(False)
except ImportError:
pass
except ImportError:
print("ImGuiOverlayPlugin: imgui-bundle not found. Install with: pip install imgui-bundle")
except Exception as e:
print(f"ImGuiOverlayPlugin: Failed to initialize ImGui: {e}")
def _cache_entity_data(self):
"""Cache static joint metadata from all rigid entities."""
self._entity_cache.clear()
for entity in self.scene.rigid_solver.entities:
if entity.n_dofs == 0:
# Still include for vis_mode toggle, but no joint data
self._entity_cache[entity.idx] = EntityCacheEntry(
entity=entity,
name=entity.name,
joint_data=EntityJointData(
q_names=[],
q_limits=([], []),
q_is_quaternion=[],
quat_groups=[],
has_free_joint=False,
free_joint_q_start=-1,
),
n_qs=0,
n_dofs=0,
)
continue
jdata = build_entity_joint_data(entity, self._free_joint_pos_limit)
if jdata.q_names:
self._entity_cache[entity.idx] = EntityCacheEntry(
entity=entity,
name=entity.name,
joint_data=jdata,
n_qs=len(jdata.q_names),
n_dofs=entity.n_dofs,
)
def _capture_pending_entities_kwargs(self):
"""Capture current entity construction kwargs into ``self._pending_entities_kwargs`` for the Scene
editor panel. Keyed by entity name; values are the kwargs forwarded to ``scene.add_entity``."""
self._pending_entities_kwargs = {}
for entity in self.scene.entities:
morph = tuple(entity.morphs) if getattr(entity, "_enable_heterogeneous", False) else entity.morph
kwargs: dict[str, Any] = {"morph": morph}
if isinstance(entity, gs.engine.entities.RigidEntity):
kwargs["material"] = entity.material
kwargs["surface"] = entity.surface
kwargs["visualize_contact"] = entity.visualize_contact
self._pending_entities_kwargs[entity.name] = kwargs
def _is_capturing(self) -> bool:
"""Check if ImGui or gizmo wants mouse/keyboard input."""
if not self._available:
return False
return self._io.want_capture_mouse or self._io.want_capture_keyboard or self._is_gizmo_active()
# Event handlers - forward input to ImGui and block when capturing
def on_mouse_press(self, x, y, button, modifiers) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_mouse_press(x, y, button, modifiers)
return EVENT_HANDLED if self._is_capturing() else None
def on_mouse_drag(self, x, y, dx, dy, buttons, modifiers) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_mouse_drag(x, y, dx, dy, buttons, modifiers)
return EVENT_HANDLED if self._is_capturing() else None
def on_mouse_release(self, x, y, button, modifiers) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_mouse_release(x, y, button, modifiers)
return EVENT_HANDLED if self._is_capturing() else None
def on_mouse_scroll(self, x, y, dx, dy) -> EVENT_HANDLE_STATE:
if self._available:
# imgui backend expects: on_mouse_scroll(x, y, mods, scroll)
self._impl.on_mouse_scroll(x, y, 0, dy)
return EVENT_HANDLED if self._is_capturing() else None
def on_mouse_motion(self, x, y, dx, dy) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_mouse_motion(x, y, dx, dy)
return EVENT_HANDLED if self._is_capturing() else None
def on_key_press(self, symbol, modifiers) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_key_press(symbol, modifiers)
return EVENT_HANDLED if self._is_capturing() else None
def on_key_release(self, symbol, modifiers) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_key_release(symbol, modifiers)
return EVENT_HANDLED if self._is_capturing() else None
def on_text(self, text) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_text(text)
return EVENT_HANDLED if self._is_capturing() else None
def on_resize(self, width, height) -> EVENT_HANDLE_STATE:
if self._available:
self._impl.on_resize(width, height)
return None
def on_draw(self) -> None:
"""Render ImGui overlay after scene is drawn."""
# Lazy initialization: must happen in viewer thread (which owns OpenGL context)
if not self._init_attempted:
self._init_imgui()
if not self._available:
return
# Update delta time manually (avoid calling pyglet.clock.tick() which conflicts with viewer loop)
current_time = time.perf_counter()
if self._last_time is not None:
self._io.delta_time = current_time - self._last_time
else:
self._io.delta_time = 1.0 / 60.0
if self._io.delta_time <= 0.0:
self._io.delta_time = 1.0 / 1000.0
self._last_time = current_time
# Track FPS history
if self._io.delta_time > 0:
self._fps_history.append(1.0 / self._io.delta_time)
if len(self._fps_history) > _FPS_HISTORY_SIZE:
self._fps_history = self._fps_history[-_FPS_HISTORY_SIZE:]
self._imgui.new_frame()
# Initialize ImGuizmo for this frame
if self._gizmo is not None:
self._gizmo.begin_frame()
io = self._io
self._gizmo.set_rect(0, 0, io.display_size.x, io.display_size.y)
self._gizmo.set_orthographic(not self.viewer.viewer_flags["use_perspective_cam"])
self._render_control_panel()
# Render 3D gizmos for selected free-joint entity
if self._gizmo is not None and self._gizmo_entity_idx >= 0:
self._render_gizmo()
self._imgui.render()
self._impl.render(self._imgui.get_draw_data())
def _render_control_panel(self):
"""Render unified control panel with all sections."""
imgui = self._imgui
if self._panel_width is not None:
# Pin the width while letting the height autoresize to content.
imgui.set_next_window_size_constraints((self._panel_width, 0.0), (self._panel_width, float("inf")))
imgui.begin("Genesis Control Panel", flags=imgui.WindowFlags_.always_auto_resize)
self._render_sim_controls()
if imgui.begin_tab_bar("##main_tabs"):
for name, render_section in (
("Entities", self._render_entity_browser),
("Visualization", self._render_visualization),
("Camera", self._render_camera_controls),
("Scene", self._render_scene_editor),
):
# Force the requested tab selected for this frame; otherwise honor the user's selection.
flags = imgui.TabItemFlags_.set_selected.value if self._active_tab == name else 0
if imgui.begin_tab_item(name, None, flags)[0]:
render_section()
imgui.end_tab_item()
imgui.end_tab_bar()
self._active_tab = None
# Render user callback panels (side panels)
for callback, section in self._user_panels:
if section == "side":
callback(imgui)
imgui.end()
# Render overlay panels as separate windows
for callback, section in self._user_panels:
if section == "overlay":
callback(imgui)
def _render_sim_controls(self):
"""Render simulation control buttons, time display, and FPS."""
imgui = self._imgui
interactive = self._interactive_scene
# State label
if interactive.paused:
imgui.text_colored((1.0, 0.7, 0.0, 1.0), "Paused")
else:
imgui.text_colored((0.4, 0.9, 0.4, 1.0), "Running")
# Play/Pause and Reset (always visible), Step (only when paused). Auto-fit the label but with a 60-px
# floor so single-word verbs share a consistent baseline width and never get truncated.
play_pause = "Pause" if not interactive.paused else "Play"
if imgui.button(play_pause, size=button_size_with_min(imgui, play_pause, 60.0)):
interactive.resume() if interactive.paused else interactive.pause()
if interactive.paused:
imgui.same_line()
if imgui.button("Step", size=button_size_with_min(imgui, "Step", 60.0)):
interactive.step(self._step_count)
imgui.same_line()
if imgui.button("Reset", size=button_size_with_min(imgui, "Reset", 60.0)):
with self.viewer.render_lock:
self.scene.reset()
self.viewer.gs_context.clear_dynamic_nodes(only_outdated=False)
self._refresh_visuals()
# Time display (frame count * dt = simulation time)
sim_time = self.scene.t * self.scene.sim.dt
imgui.text(f"Time: {sim_time:.3f}s Step: {self.scene.t}")
# FPS display
if self._fps_history:
avg_fps = sum(self._fps_history) / len(self._fps_history)
imgui.same_line()
imgui.text(f" FPS: {avg_fps:.0f}")
if self.scene.n_envs > 1:
imgui.text_colored(
(1.0, 0.7, 0.0, 1.0),
f"Note: Controlling env {self._controlled_env_idx} of {self.scene.n_envs}",
)
draw_separator(imgui)
def _render_visualization(self):
"""Render visualization toggle controls."""
imgui = self._imgui
render_flags = self.viewer.render_flags
gs_context = self.viewer.gs_context
# Shadows
changed, new_val = imgui.checkbox("Shadows", render_flags["shadows"])
if changed:
render_flags["shadows"] = new_val
# World Frame
changed, new_val = imgui.checkbox("World Frame", gs_context.world_frame_shown)
if changed:
(gs_context.on_world_frame if new_val else gs_context.off_world_frame)()
# Link Frame
changed, new_val = imgui.checkbox("Link Frame", gs_context.link_frame_shown)
if changed:
(gs_context.on_link_frame if new_val else gs_context.off_link_frame)()
# Link Frame Size slider
link_size = gs_context.link_frame_size
changed_size, new_size = imgui.slider_float("Frame Size##link_frame_size", link_size, 0.02, 0.5, "%.2f")
if changed_size and link_size > 0 and new_size > 0:
gs_context.link_frame_mesh.vertices *= new_size / link_size
gs_context.link_frame_size = new_size
if gs_context.link_frame_shown:
gs_context.off_link_frame()
gs_context.on_link_frame()
# Camera Frustum
changed, new_val = imgui.checkbox("Camera Frustum", gs_context.camera_frustum_shown)
if changed:
(gs_context.on_camera_frustum if new_val else gs_context.off_camera_frustum)()
# Face Normals
changed, new_val = imgui.checkbox("Face Normals", render_flags["face_normals"])
if changed:
render_flags["face_normals"] = new_val
# Vertex Normals
changed, new_val = imgui.checkbox("Vertex Normals", render_flags["vertex_normals"])
if changed:
render_flags["vertex_normals"] = new_val
draw_separator(imgui)
# Orthographic Camera
is_ortho = not self.viewer.viewer_flags["use_perspective_cam"]
changed, new_ortho = imgui.checkbox("Orthographic Camera", is_ortho)
if changed:
self.viewer.viewer_flags["use_perspective_cam"] = not new_ortho
if new_ortho:
self.viewer._camera_node.camera = self.viewer._default_orth_cam
else:
self.viewer._camera_node.camera = self.viewer._default_persp_cam
def _render_gizmo(self):
"""Render 3D manipulation gizmo for the selected free-joint entity."""
gizmo = self._gizmo
Matrix16 = gizmo.Matrix16
data = self._entity_cache.get(self._gizmo_entity_idx)
if data is None or not data.joint_data.has_free_joint:
return
entity = data.entity
qs = data.joint_data.free_joint_q_start
# While actively dragging, use the cached matrix to avoid qpos round-trip jitter.
# Only read from qpos when not dragging (to pick up external changes).
if gizmo.is_using() and self._gizmo_cached_matrix is not None:
obj_mat = self._gizmo_cached_matrix
else:
# Get current qpos
qpos = tensor_to_array(entity.get_qpos())
if self._controlled_env_idx is not None:
qpos = qpos[self._controlled_env_idx]
# Extract position and quaternion from qpos
pos = qpos[qs : qs + 3]
quat_wxyz = qpos[qs + 3 : qs + 7] # w, x, y, z
rot = R.from_quat([quat_wxyz[1], quat_wxyz[2], quat_wxyz[3], quat_wxyz[0]]) # scipy uses x,y,z,w
obj_mat = np.eye(4)
obj_mat[:3, :3] = rot.as_matrix()
obj_mat[:3, 3] = pos
# ImGuizmo expects column-major (transpose for row-major numpy)
object_matrix = Matrix16(obj_mat.T.flatten().tolist())
# Get view matrix (inverse of camera pose)
cam_pose = self.viewer._trackball._n_pose.copy()
view_mat = np.linalg.inv(cam_pose)
camera_view = Matrix16(view_mat.T.flatten().tolist())
# Get projection matrix
w, h = int(self._io.display_size.x), int(self._io.display_size.y)
if w > 0 and h > 0:
proj = self.camera.camera.get_projection_matrix(width=w, height=h)
camera_proj = Matrix16(proj.T.flatten().tolist())
else:
return
# Draw gizmo
modified = gizmo.manipulate(
camera_view,
camera_proj,
self._gizmo_operation,
self._gizmo_mode,
object_matrix,
)
if modified:
# Extract new transform from modified matrix (column-major -> row-major)
new_mat = np.array(object_matrix.values).reshape(4, 4).T
# Cache the matrix for next frame to avoid qpos round-trip jitter
self._gizmo_cached_matrix = new_mat.copy()
new_pos = new_mat[:3, 3]
new_rot = R.from_matrix(new_mat[:3, :3])
new_quat_xyzw = new_rot.as_quat() # scipy: x,y,z,w
new_quat_wxyz = [new_quat_xyzw[3], new_quat_xyzw[0], new_quat_xyzw[1], new_quat_xyzw[2]]
# Read current qpos for non-free-joint DOFs
qpos = tensor_to_array(entity.get_qpos())
if self._controlled_env_idx is not None:
qpos = qpos[self._controlled_env_idx]
# Update only the free-joint DOFs
new_qpos = list(qpos)
new_qpos[qs : qs + 3] = new_pos.tolist()
new_qpos[qs + 3 : qs + 7] = new_quat_wxyz
# Auto-pause on gizmo edit
self._interactive_scene.pause()
with self.viewer.render_lock:
entity.set_qpos(new_qpos, envs_idx=self._controlled_env_idx)
self._refresh_visuals()
elif not gizmo.is_using():
# Clear cache when drag ends so next interaction reads fresh qpos
self._gizmo_cached_matrix = None
def _is_gizmo_active(self):
"""Check if the gizmo is being used (for input blocking)."""
if self._gizmo is not None:
return self._gizmo.is_using() or self._gizmo.is_over()
return False
def _render_camera_controls(self):
"""Render camera position, lookat, FOV controls."""
imgui = self._imgui
trackball = self.viewer._trackball
# Read current camera state from trackball
pose = trackball._n_pose
pos = [float(pose[0, 3]), float(pose[1, 3]), float(pose[2, 3])]
# Use trackball's actual orbit center as lookat (not derived from z-axis)
target = trackball._n_target
lookat = [float(target[0]), float(target[1]), float(target[2])]
# Position drag
changed_pos, new_pos = imgui.drag_float3("Position##cam_pos", pos, 0.05, -100.0, 100.0, "%.2f")
# Lookat drag
changed_lookat, new_lookat = imgui.drag_float3("Lookat##cam_lookat", lookat, 0.05, -100.0, 100.0, "%.2f")
if changed_pos or changed_lookat:
cam_pos = np.array(list(new_pos)) if changed_pos else np.array(pos)
cam_lookat = np.array(list(new_lookat)) if changed_lookat else np.array(lookat)
# Build pose with fixed world-up to prevent unintuitive roll
world_up = np.array([0.0, 0.0, 1.0])
cam_pose = gu.pos_lookat_up_to_T(cam_pos, cam_lookat, world_up)
self.scene.viewer._camera_up = cam_pose[:3, 1].copy()
trackball.set_camera_pose(cam_pose)
# Sync trackball orbit center so mouse orbiting works correctly after
trackball._n_target = cam_lookat.copy()
trackball._target = cam_lookat.copy()
# FOV slider
fov_deg = float(self.camera.camera.yfov * 180.0 / np.pi)
changed_fov, new_fov = imgui.slider_float("FOV##cam_fov", fov_deg, 15.0, 120.0, "%.1f")
if changed_fov:
self.camera.camera.yfov = new_fov * np.pi / 180.0
# Reset Camera button
if imgui.button("Reset Camera", size=(120, 0)):
self.viewer._reset_view()
_FILE_EXTENSIONS = {
"URDF": {".urdf"},
"MJCF": {".xml"},
"Mesh": {".obj", ".stl", ".ply", ".dae", ".glb", ".gltf"},
}
def _render_file_browser(self, morph_type):
"""Render a file browser popup for selecting asset files."""
imgui = self._imgui
if not self._file_browser_open:
return
imgui.open_popup("File Browser##file_popup")
imgui.set_next_window_size((500, 400))
if imgui.begin_popup_modal("File Browser##file_popup")[0]:
# Current directory display with parent navigation
if imgui.button("^##parent_dir"):
parent = os.path.dirname(self._file_browser_dir)
if parent != self._file_browser_dir:
self._file_browser_dir = parent
self._file_browser_selected = -1
imgui.same_line()
imgui.text(self._file_browser_dir)
draw_separator(imgui)
# List directory contents
valid_exts = self._FILE_EXTENSIONS.get(morph_type, set())
try:
entries = sorted(os.listdir(self._file_browser_dir))
except OSError:
entries = []
dirs = [
e for e in entries if os.path.isdir(os.path.join(self._file_browser_dir, e)) and not e.startswith(".")
]
files = [
e
for e in entries
if os.path.isfile(os.path.join(self._file_browser_dir, e))
and (not valid_exts or os.path.splitext(e)[1].lower() in valid_exts)
]
items = [d + "/" for d in dirs] + files
if imgui.begin_child("file_list", size=(0, -30)):
for idx, item in enumerate(items):
is_dir = item.endswith("/")
selected = idx == self._file_browser_selected
if imgui.selectable(item, selected)[0]:
if is_dir:
self._file_browser_dir = os.path.join(self._file_browser_dir, item[:-1])
self._file_browser_selected = -1
else:
self._file_browser_selected = idx
# Double-click on file to confirm
if not is_dir and imgui.is_item_hovered() and imgui.is_mouse_double_clicked(0):
self._add_entity_file = os.path.join(self._file_browser_dir, item)
self._file_browser_open = False
imgui.close_current_popup()
imgui.end_child()
# OK / Cancel buttons
can_select = self._file_browser_selected >= 0 and self._file_browser_selected >= len(dirs)
if imgui.button("OK", size=(80, 0)) and can_select:
file_name = files[self._file_browser_selected - len(dirs)]
self._add_entity_file = os.path.join(self._file_browser_dir, file_name)
self._file_browser_open = False
imgui.close_current_popup()
imgui.same_line()
if imgui.button("Cancel", size=(80, 0)):
self._file_browser_open = False
imgui.close_current_popup()
imgui.end_popup()
else:
# Popup was closed (e.g. clicking outside)
self._file_browser_open = False
_SCENE_EDIT_DISABLED_TOOLTIP = "This action is unavailable for the current scene configuration."
def _maybe_show_disabled_tooltip(self, disabled: bool):
"""Show the unavailable-feature tooltip when the previous item is disabled and hovered."""
if not disabled:
return
imgui = self._imgui
if imgui.is_item_hovered(imgui.HoveredFlags_.allow_when_disabled.value):
imgui.set_tooltip(self._SCENE_EDIT_DISABLED_TOOLTIP)
def _render_scene_editor(self):
"""Render scene editing controls (entity scale, add entity, rebuild).
Each control maps to an InteractiveFeature and renders in its disabled visual state when the
scene does not support that feature, so the panel layout stays identical across scene types.
"""
imgui = self._imgui
features = self._supported_features
scale_disabled = InteractiveFeature.SCALE_ENTITY not in features
remove_disabled = InteractiveFeature.REMOVE_ENTITY not in features
add_disabled = InteractiveFeature.ADD_ENTITY not in features
rebuild_disabled = InteractiveFeature.REBUILD not in features
# Per-entity scale editing (FileMorph only; primitives carry size/radius/height instead).
to_remove: str | None = None
for name, kwargs in self._pending_entities_kwargs.items():
morph = kwargs["morph"]
morph_name = type(morph).__name__
file_name = morph.file if isinstance(morph, gs.morphs.FileMorph) else ""
imgui.text(f"{name} ({morph_name}): {file_name or '(builtin)'}")
if isinstance(morph, gs.morphs.FileMorph):
scale = morph.scale
scale_val = float(scale[0]) if isinstance(scale, (list, tuple, np.ndarray)) else float(scale)
imgui.begin_disabled(scale_disabled)
changed, new_scale = imgui.drag_float(f"Scale##scale_{name}", scale_val, 0.01, 0.01, 100.0, "%.3f")
imgui.end_disabled()
self._maybe_show_disabled_tooltip(scale_disabled)
if changed and not scale_disabled:
morph.scale = new_scale
self._pending_dirty = True
imgui.same_line()
imgui.begin_disabled(remove_disabled)
remove_clicked = imgui.button(f"X##remove_{name}")
imgui.end_disabled()
self._maybe_show_disabled_tooltip(remove_disabled)
if remove_clicked and not remove_disabled:
to_remove = name
draw_separator(imgui)
if to_remove is not None:
del self._pending_entities_kwargs[to_remove]
self._pending_dirty = True
# Add entity section
imgui.begin_disabled(add_disabled)
add_header_open = imgui.collapsing_header("Add Entity##add_entity")
imgui.end_disabled()
self._maybe_show_disabled_tooltip(add_disabled)
if add_header_open:
imgui.indent()
imgui.begin_disabled(add_disabled)
changed_type, self._add_entity_morph_type = imgui.combo(
"Type##add_type", self._add_entity_morph_type, _MORPH_TYPES
)
morph_type = _MORPH_TYPES[self._add_entity_morph_type]
# Default fixed=True for Plane when type changes
if changed_type and morph_type == "Plane":
self._add_entity_fixed = True
# File path for file-based morphs
if morph_type in ("URDF", "MJCF", "Mesh"):
_, self._add_entity_file = imgui.input_text("File##add_file", self._add_entity_file, 256)
imgui.same_line()
if imgui.button("Browse##add_browse") and not add_disabled:
self._file_browser_open = True
self._file_browser_selected = -1
# Start browsing from current file's directory if set
if self._add_entity_file:
parent = os.path.dirname(self._add_entity_file)
if os.path.isdir(parent):
self._file_browser_dir = parent
self._render_file_browser(morph_type)
_, self._add_entity_scale = imgui.drag_float(
"Scale##add_scale", self._add_entity_scale, 0.01, 0.01, 100.0, "%.3f"
)
# Type-specific geometry params
if morph_type == "Box":
_, self._add_box_size = imgui.drag_float3(
"Size##add_box_size", self._add_box_size, 0.01, 0.01, 100.0, "%.3f"
)
elif morph_type == "Sphere":
_, self._add_sphere_radius = imgui.drag_float(
"Radius##add_sphere_r", self._add_sphere_radius, 0.01, 0.01, 100.0, "%.3f"
)
elif morph_type == "Cylinder":
_, self._add_cylinder_radius = imgui.drag_float(
"Radius##add_cyl_r", self._add_cylinder_radius, 0.01, 0.01, 100.0, "%.3f"
)
_, self._add_cylinder_height = imgui.drag_float(
"Height##add_cyl_h", self._add_cylinder_height, 0.01, 0.01, 100.0, "%.3f"
)
# Position (all types except Plane)
if morph_type != "Plane":
_, self._add_entity_pos = imgui.drag_float3(
"Position##add_pos", self._add_entity_pos, 0.05, -100.0, 100.0, "%.2f"
)
# Fixed checkbox
_, self._add_entity_fixed = imgui.checkbox("Fixed##add_fixed", self._add_entity_fixed)
add_clicked = imgui.button("Add##add_btn")
imgui.end_disabled()
self._maybe_show_disabled_tooltip(add_disabled)
if add_clicked and not add_disabled:
pos = tuple(self._add_entity_pos)
scale = self._add_entity_scale
fixed = self._add_entity_fixed
box_size = tuple(self._add_box_size)
morph_cls_map = {
"URDF": lambda: gs.morphs.URDF(file=self._add_entity_file, pos=pos, scale=scale, fixed=fixed),
"MJCF": lambda: gs.morphs.MJCF(file=self._add_entity_file, pos=pos, scale=scale, fixed=fixed),
"Mesh": lambda: gs.morphs.Mesh(file=self._add_entity_file, pos=pos, scale=scale, fixed=fixed),
"Box": lambda: gs.morphs.Box(pos=pos, size=box_size, fixed=fixed),
"Sphere": lambda: gs.morphs.Sphere(pos=pos, radius=self._add_sphere_radius, fixed=fixed),
"Cylinder": lambda: gs.morphs.Cylinder(
pos=pos, radius=self._add_cylinder_radius, height=self._add_cylinder_height, fixed=fixed
),
"Plane": lambda: gs.morphs.Plane(),
}
new_morph = morph_cls_map[morph_type]()
# Generate a unique name based on the morph type.
base_name = morph_type
suffix = 0
name = base_name
while name in self._pending_entities_kwargs:
suffix += 1
name = f"{base_name}_{suffix}"
self._pending_entities_kwargs[name] = {
"morph": new_morph,
"material": None,
"surface": None,
"visualize_contact": False,
}
self._pending_dirty = True
imgui.unindent()
# Rebuild button. The InteractiveScene performs the actual rebuild on the stepping thread; doing it
# here on the viewer thread would destroy the OpenGL context we are rendering from.
if self._pending_dirty:
imgui.text_colored((1.0, 0.7, 0.0, 1.0), "Changes pending")
imgui.begin_disabled(rebuild_disabled)
rebuild_clicked = imgui.button("Rebuild Scene", size=(150, 0))
imgui.end_disabled()
self._maybe_show_disabled_tooltip(rebuild_disabled)
if rebuild_clicked and not rebuild_disabled:
self._interactive_scene.rebuild(entities_kwargs=self._pending_entities_kwargs)
self._pending_dirty = False
def _render_entity_browser(self):
"""Render entity list with joint sliders."""
imgui = self._imgui
if not self._entity_cache:
imgui.text("No controllable entities")
return
for entity_idx, data in self._entity_cache.items():
entity = data.entity
expanded = imgui.collapsing_header(
f"{data.name}##entity_{entity_idx}", flags=imgui.TreeNodeFlags_.default_open
)
if not expanded:
continue
imgui.indent()