fix bugs of data preprocessing with multiple json keys

NVIDIA · Dec 25, 2024 · 90a7760 · 90a7760
1 parent 2da43ef
commit 90a7760
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/tools/preprocess_data.py b/tools/preprocess_data.py
@@ -184,7 +184,8 @@ def process_json_file(self, file_name):
             self.print_processing_stats(i, proc_start, total_bytes_processed)
 
         fin.close()
-        builders[key].finalize(output_idx_files[key])
+        for key in builders.keys():
+            builders[key].finalize(output_idx_files[key])
 
 
 def get_args():