From f6c312d140c718523e2ab4eabbc1b5bf1735a2c8 Mon Sep 17 00:00:00 2001 From: Alexander Raistrick Date: Tue, 8 Oct 2024 14:03:00 -0400 Subject: [PATCH 1/2] Move gin initialization inside slurm-submitted function, so slurm jobs recieve gin configs --- .../generate_individual_assets.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/infinigen_examples/generate_individual_assets.py b/infinigen_examples/generate_individual_assets.py index 6762b5b2..a4c85796 100644 --- a/infinigen_examples/generate_individual_assets.py +++ b/infinigen_examples/generate_individual_assets.py @@ -236,6 +236,25 @@ def build_and_save_asset(payload: dict): output_folder.mkdir(exist_ok=True) + init.apply_gin_configs( + ["infinigen_examples/configs_indoor", "infinigen_examples/configs_nature"], + configs=args.configs, + overrides=args.overrides, + skip_unknown=True, + ) + + if args.debug is not None: + for name in logging.root.manager.loggerDict: + if not name.startswith("infinigen"): + continue + if len(args.debug) == 0 or any(name.endswith(x) for x in args.debug): + logging.getLogger(name).setLevel(logging.DEBUG) + + init.configure_blender() + + if args.gpu: + init.configure_render_cycles() + logger.info(f"Building scene for {factory_name} {idx}") if args.seed > 0: @@ -454,25 +473,6 @@ def mapfunc( def main(args): bpy.context.window.workspace = bpy.data.workspaces["Geometry Nodes"] - init.apply_gin_configs( - ["infinigen_examples/configs_indoor", "infinigen_examples/configs_nature"], - configs=args.configs, - overrides=args.overrides, - skip_unknown=True, - ) - - if args.debug is not None: - for name in logging.root.manager.loggerDict: - if not name.startswith("infinigen"): - continue - if len(args.debug) == 0 or any(name.endswith(x) for x in args.debug): - logging.getLogger(name).setLevel(logging.DEBUG) - - init.configure_blender() - - if args.gpu: - init.configure_render_cycles() - if args.output_folder is None: outputs = Path("outputs") assert outputs.exists(), outputs From 2bfad931808e4cc9326bb29f25ee3f0949ef5b56 Mon Sep 17 00:00:00 2001 From: Alexander Raistrick Date: Wed, 28 Aug 2024 12:59:41 -0400 Subject: [PATCH 2/2] Fix wandb alert title overflow, fix show_gpu_table account names --- infinigen/datagen/manage_jobs.py | 2 +- infinigen/datagen/util/show_gpu_table.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/infinigen/datagen/manage_jobs.py b/infinigen/datagen/manage_jobs.py index 8d19bae6..0e82bfa4 100644 --- a/infinigen/datagen/manage_jobs.py +++ b/infinigen/datagen/manage_jobs.py @@ -679,7 +679,7 @@ def manage_datagen_jobs(all_scenes, elapsed, num_concurrent, disk_sleep_threshol print(message) if wandb is not None: wandb.alert( - title=f"{args.output_folder} full", + title=f"{args.output_folder.name} sleeping for full disk", text=message, wait_duration=3 * 60 * 60, ) diff --git a/infinigen/datagen/util/show_gpu_table.py b/infinigen/datagen/util/show_gpu_table.py index e8aaf0a5..87bb92dd 100644 --- a/infinigen/datagen/util/show_gpu_table.py +++ b/infinigen/datagen/util/show_gpu_table.py @@ -35,7 +35,7 @@ def get_gpu_nodes(): for line in sinfo_output.splitlines(): node, group, gres, totalmem = line.split() if group != "all": - if group in {"pvl", "cs*"}: + if group in {"pvl", "cs*", "pnlp"}: num_cpus = int(cpu_regex(gres).group(1)) shared_node_mem[node] = int((int(totalmem) / 1024) / num_cpus) if gres_regex(gres):