From 1574d42335d4cd1b412d3826fcaa439e899d72b8 Mon Sep 17 00:00:00 2001 From: Janne Hellsten Date: Wed, 17 Feb 2021 18:02:45 +0200 Subject: [PATCH] dataset_tool.py docs update, tf32 disable for calc_metrics.py - Add a justification for using uncompressed zip and uncompressed png (#22) - Clarify class label usage with dataset_tool.py (#18) - Disable tf32 in calc_metrics as this has not been tested. --- calc_metrics.py | 2 ++ dataset_tool.py | 22 +++++++++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/calc_metrics.py b/calc_metrics.py index cdd6dd381..03e828195 100755 --- a/calc_metrics.py +++ b/calc_metrics.py @@ -47,6 +47,8 @@ def subprocess_fn(rank, args, temp_dir): # Print network summary. device = torch.device('cuda', rank) torch.backends.cudnn.benchmark = True + torch.backends.cuda.matmul.allow_tf32 = False + torch.backends.cudnn.allow_tf32 = False G = copy.deepcopy(args.G).eval().requires_grad_(False).to(device) if rank == 0 and args.verbose: z = torch.empty([1, G.z_dim], device=device) diff --git a/dataset_tool.py b/dataset_tool.py index d663211ff..c59e62928 100755 --- a/dataset_tool.py +++ b/dataset_tool.py @@ -331,14 +331,34 @@ def convert_dataset( --source path/ Recursively load all images from path/ --source dataset.zip Recursively load all images from dataset.zip - The output dataset format can be either an image folder or a zip archive. Specifying the output format and path: \b --dest /path/to/dir Save output files under /path/to/dir --dest /path/to/dataset.zip Save output files into /path/to/dataset.zip + The output dataset format can be either an image folder or an uncompressed zip archive. + Zip archives makes it easier to move datasets around file servers and clusters, and may + offer better training performance on network file systems. + Images within the dataset archive will be stored as uncompressed PNG. + Uncompresed PNGs can be efficiently decoded in the training loop. + + Class labels are stored in a file called 'dataset.json' that is stored at the + dataset root folder. This file has the following structure: + + \b + { + "labels": [ + ["00000/img00000000.png",6], + ["00000/img00000001.png",9], + ... repeated for every image in the datase + ["00049/img00049999.png",1] + ] + } + + If the 'dataset.json' file cannot be found, the dataset is interpreted as + not containing class labels. Image scale/crop and resolution requirements: