Render improvements (v1.3.1) (princeton_vl/infinigen_internal/princet…

…on-vl#107)

* Attempt to fix white snakes and volume_bounces=0 bugs

* Make videos 4sec not 8, tweak slurm_1h, tweak terrain resolution, speed up cameras for video

* Fix --use_existing for scenes started with --specific_seed

* Increase opengl time, decrease render sample quality

* Noisify camera motions

* Add overhead.gin

* Tweak video length and cam seped

* Drop fps to 16

* Reorganize cycles configuration, overhaul enable_gpu to be more robust and only ever use one device type

* Bump version to 1.3.1, remove scripts/launch

* Tweak ratios, fix typo

* Fix noshortrender typo, rename conf to noisy_video

* Changelog

.gitignore

@@ -42,7 +42,8 @@ Blender-FLIP-Fluids
 *.blend1
 *.out
 profiles_*.npz
-*outputs
+outputs
+outputs_scratch
 snippets
 resources
 times.txt

docs/CHANGELOG.md

@@ -36,3 +36,6 @@ v1.2.6
 - Fix bug where manage_jobs.py would ignore CUDA_VISIBLE_DEVICES that didnt start at 0
 - Add NotImplementedError for dynamic hair.
 
+v1.3.1
+- Fix configuration bug causing massive render slowdown 
+- Create noisier video trajectories optimized for training

infinigen/__init__.py

@@ -1,3 +1,3 @@
 import logging
 
-__version__ = "1.2.6"
+__version__ = "1.3.1"

infinigen/assets/creatures/reptile.py

@@ -395,7 +395,7 @@ def chameleon_postprocessing(body_parts, extras, params):
 def purge_empty_materials(obj):
     with butil.SelectObjects(obj):
         for i, m in enumerate(obj.material_slots):
-            if m.name != '':
+            if m.material is not None:
                 continue
             bpy.context.object.active_material_index = i
             bpy.ops.object.material_slot_remove()

infinigen/core/execute_tasks.py

@@ -283,6 +283,7 @@ def execute_tasks(
     resample_idx=None,
     output_blend_name="scene.blend",
     generate_resolution=(1280,720),
+    fps=24,
     reset_assets=True,
     focal_length=None,
     dryrun=False,
@@ -315,6 +316,7 @@ def execute_tasks(
     bpy.context.scene.frame_start = int(frame_range[0])
     bpy.context.scene.frame_end = int(frame_range[1])
     bpy.context.scene.frame_set(int(frame_range[0]))
+    bpy.context.scene.render.fps = fps
     bpy.context.scene.render.resolution_x = generate_resolution[0]
     bpy.context.scene.render.resolution_y = generate_resolution[1]
     bpy.context.view_layer.update()

infinigen/core/init.py

@@ -29,6 +29,19 @@
 
 logger = logging.getLogger(__name__)
 
+CYCLES_GPUTYPES_PREFERENCE = [
+
+    # key must be a valid cycles device_type
+    # ordering indicate preference - earlier device types will be used over later if both are available
+    #  - e.g most OPTIX gpus will also show up as a CUDA gpu, but we will prefer to use OPTIX due to this list's ordering
+
+    'OPTIX',
+    'CUDA',  
+    'HIP', # untested
+    'ONEAPI', # untested
+    'CPU',
+]
+
 def parse_args_blender(parser):
     if '--' in sys.argv:
         # Running using a blender commandline python. 
@@ -193,15 +206,100 @@ def import_addons(names):
         except Exception:
             logger.warning(f'Could not load addon "{name}"')
 
-def configure_blender():
-    bpy.context.preferences.system.scrollback = 0 
-    bpy.context.preferences.edit.undo_steps = 0
+@gin.configurable
+def configure_render_cycles(
+
+    # supplied by gin.config
+    min_samples,
+    num_samples,
+    time_limit,
+    adaptive_threshold,
+    exposure,
+    denoise
+):
     bpy.context.scene.render.engine = 'CYCLES'
-    bpy.context.scene.cycles.device = 'GPU'
 
+    # For now, denoiser is always turned on, but the  _used_
+    if denoise:
+        try:
+            bpy.context.scene.cycles.use_denoising = True
+            bpy.context.scene.cycles.denoiser = 'OPTIX'
+        except Exception as e:
+            logger.warning(f"Cannot use OPTIX denoiser {e}")
+
+    bpy.context.scene.cycles.samples = num_samples # i.e. infinity
+    bpy.context.scene.cycles.adaptive_min_samples = min_samples
+    bpy.context.scene.cycles.adaptive_threshold = adaptive_threshold # i.e. noise threshold
+    bpy.context.scene.cycles.time_limit = time_limit
+    bpy.context.scene.cycles.film_exposure = exposure
     bpy.context.scene.cycles.volume_step_rate = 0.1
     bpy.context.scene.cycles.volume_preview_step_rate = 0.1
     bpy.context.scene.cycles.volume_max_steps = 32
+    bpy.context.scene.cycles.volume_bounces = 4
+
+@gin.configurable
+def configure_cycles_devices(
+    use_gpu=True
+):
+
+    if use_gpu is False:
+        logger.info(f'Render will use CPU-only due to {use_gpu=}')
+        bpy.context.scene.cycles.device = 'CPU'
+        return
+
+    assert bpy.context.scene.render.engine == 'CYCLES'
+    bpy.context.scene.cycles.device = 'GPU'
+    prefs = bpy.context.preferences.addons['cycles'].preferences
+
+    # Necessary to "remind" cycles that the devices exist? Not sure. Without this no devices are found.
+    for dt in prefs.get_device_types(bpy.context):
+        prefs.get_devices_for_type(dt[0])
+
+    assert len(prefs.devices) != 0, prefs.devices
+
+    types = list(d.type for d in prefs.devices)
+
+    types = sorted(types, key=CYCLES_GPUTYPES_PREFERENCE.index)
+    logger.info(f'Available devices have {types=}')
+    use_device_type = types[0]
+
+    if use_device_type == 'CPU':
+        logger.warning(f'Render will use CPU-only, only found {types=}')
+        bpy.context.scene.cycles.device = 'CPU'
+        return
+
+    bpy.context.preferences.addons['cycles'].preferences.compute_device_type = use_device_type
+    use_devices = [d for d in prefs.devices if d.type == use_device_type]
+
+
+    logger.info(f'Cycles will use {use_device_type=}, {len(use_devices)=}')
+
+    for d in prefs.devices:
+        d.use = False
+    for d in use_devices:
+        d.use = True
+
+    return use_devices
+
+@gin.configurable
+def configure_blender(
+    render_engine='CYCLES',
+    motion_blur=False,
+    motion_blur_shutter=0.5,
+):
+    bpy.context.preferences.system.scrollback = 0 
+    bpy.context.preferences.edit.undo_steps = 0
+
+    if render_engine == 'CYCLES':
+        configure_render_cycles()
+        configure_cycles_devices()
+    else:
+        raise ValueError(f'Unrecognized {render_engine=}')
+
+    bpy.context.scene.render.use_motion_blur = motion_blur
+    if motion_blur: 
+        bpy.context.scene.cycles.motion_blur_position = 'START'
+        bpy.context.scene.render.motion_blur_shutter = motion_blur_shutter
 
     import_addons(['ant_landscape', 'real_snow'])
 

infinigen/core/placement/animation_policy.py

@@ -170,15 +170,17 @@ class AnimPolicyRandomWalkLookaround:
     def __init__(
         self, 
         speed=('uniform', 1, 2.5), 
-        yaw_range=(-20, 20), 
-        step_range=(10, 15),
+        step_speed_mult=('uniform', 0.5, 2),
+        yaw_sampler=('uniform',-20, 20), 
+        step_range=('clip_gaussian', 3, 5, 0.5, 10),
         rot_vars=(5, 0, 5),
         motion_dir_zoff=('clip_gaussian', 0, 90, 0, 180)
     ):
 
         self.speed = random_general(speed)
 
-        self.yaw_range = yaw_range
+        self.step_speed_mult = step_speed_mult
+        self.yaw_sampler = yaw_sampler
         self.step_range = step_range
         self.rot_vars = rot_vars
 
@@ -195,15 +197,16 @@ def __call__(self, obj, frame_curr, bvh, retry_pct):
         orig_motion_dir_euler = copy(self.motion_dir_euler)
         def sampler():
             self.motion_dir_euler = copy(orig_motion_dir_euler)
-            self.motion_dir_euler[2] += np.deg2rad(U(*self.yaw_range))
-            step = U(*self.step_range)
+            self.motion_dir_euler[2] += np.deg2rad(random_general(self.yaw_sampler))
+            step = random_general(self.step_range)
             off = Euler(self.motion_dir_euler, 'XYZ').to_matrix() @ Vector((0, 0, -step))
             off.z = 0
             return off
 
         pos = walk_same_altitude(obj.location, sampler, bvh)
 
-        time = np.linalg.norm(pos - obj.location) / self.speed
+        step_speed = self.speed * random_general(self.step_speed_mult)
+        time = np.linalg.norm(pos - obj.location) / step_speed
         rot = np.array(obj.rotation_euler) + np.deg2rad(N(0, self.rot_vars, 3))
 
         return Vector(pos), Vector(rot), time, 'BEZIER'

infinigen/core/placement/camera.py

@@ -498,8 +498,8 @@ def animate_cameras(
         terrain_tags_ratio=scene_preprocessed['terrain_tags_ratio'] if strict_selection else {},
     )
 
-
     for cam_rig in cam_rigs:
+
         if policy_registry is None:  
             if U() < follow_poi_chance and pois is not None and len(pois):
                 policy = animation_policy.AnimPolicyFollowObject(
@@ -511,7 +511,9 @@ def animate_cameras(
                 policy = animation_policy.AnimPolicyRandomWalkLookaround()
         else:
             policy = policy_registry()
+
         logger.info(f'Animating {cam_rig=} using {policy=}')
+
         animation_policy.animate_trajectory(
             cam_rig, 
             scene_preprocessed['terrain_bvh'],