Add Parallel Processing Input to Item Chip Generator (#413)

* Add Parallel Processing Input to Item Chip Generator

* use int

* update changelog

* update changelog

Author: vayunalapati

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to the [Nucleus Python Client](https://github.com/scaleapi/n
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.16.12](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.12) - 2023-11-27
+
+### Added
+
+- Added `num_processes` parameter to `dataset.items_and_annotation_chip_generator()` to specify parallel processing.
 
 ## [0.16.11](https://github.com/scaleapi/nucleus-python-client/releases/tag/v0.16.11) - 2023-11-22
 

nucleus/dataset.py

@@ -1485,6 +1485,7 @@ def items_and_annotation_chip_generator(
         stride_size: int,
         cache_directory: str,
         query: Optional[str] = None,
+        num_processes: int = 0,
     ) -> Iterable[Dict[str, str]]:
         """Provides a generator of chips for all DatasetItems and BoxAnnotations in the dataset.
 
@@ -1498,6 +1499,7 @@ def items_and_annotation_chip_generator(
             cache_directory: The s3 or local directory to store the image and annotations of a chip.
               s3 directories must be in the format s3://s3-bucket/s3-key
             query: Structured query compatible with the `Nucleus query language <https://nucleus.scale.com/docs/query-language-reference>`_.
+            num_processes: The number of worker processes to use to chip and upload images. If unset, no parallel processing will occur.
 
         Returns:
             Generator where each element is a dict containing the location of the image chip (jpeg) and its annotations (json).
@@ -1522,22 +1524,26 @@ def items_and_annotation_chip_generator(
             annotations = item[BOX_TYPE]
             item_ref_id = item[ITEM_KEY][REFERENCE_ID_KEY]
             offsets = generate_offsets(w, h, chip_size, stride_size)
-            with Pool() as pool:
-                chip_args = [
-                    (
-                        offset,
-                        chip_size,
-                        w,
-                        h,
-                        item_ref_id,
-                        cache_directory,
-                        image,
-                        annotations,
-                    )
-                    for offset in offsets
-                ]
-                for chip_result in pool.imap(process_chip, chip_args):
-                    yield chip_result
+            chip_args = [
+                (
+                    offset,
+                    chip_size,
+                    w,
+                    h,
+                    item_ref_id,
+                    cache_directory,
+                    image,
+                    annotations,
+                )
+                for offset in offsets
+            ]
+            if num_processes:
+                with Pool(num_processes) as pool:
+                    for chip_result in pool.imap(process_chip, chip_args):
+                        yield chip_result
+            else:
+                for chip_arg in chip_args:
+                    yield process_chip(chip_arg)
 
     def export_embeddings(
         self,