Skip to content

Commit

Permalink
Introduce a --no-parallel argument
Browse files Browse the repository at this point in the history
  • Loading branch information
matthiask committed Jan 9, 2025
1 parent f3921dd commit 9087c8e
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 8 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ Next version

- Rewrote ``process_imagefields`` to use the multiprocessing module, which
hopefully improves compatibility on macOS.
- Introduced a ``--no-parallel`` argument to ``process_imagefields``.


0.21 (2024-12-09)
Expand Down
38 changes: 30 additions & 8 deletions imagefield/management/commands/process_imagefields.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,12 @@ def add_arguments(self, parser):
default="",
help="Run house-keeping tasks.",
)
parser.add_argument(
"--no-parallel",
action="store_true",
dest="no_parallel",
help="Process images sequentially without parallelization.",
)
parser.add_argument(
"field",
nargs="*",
Expand Down Expand Up @@ -87,18 +93,16 @@ def _process_field(self, field, options):
force=options.get("force"),
)

pool = mp.Pool()

fn = partial(
_process_instance,
field=field,
housekeep=options.get("housekeep"),
force=options.get("force"),
)

try:
if options.get("no_parallel"):
for index, (instance, errors) in enumerate(
pool.imap(fn, queryset.iterator(chunk_size=100))
map(fn, queryset.iterator(chunk_size=100))
):
if errors:
self.stderr.write("\n".join(errors))
Expand All @@ -112,10 +116,28 @@ def _process_field(self, field, options):
# if not done already
instance._skip_generate_files = True
instance.save()

finally:
pool.close()
pool.join()
else:
pool = mp.Pool()
try:
for index, (instance, errors) in enumerate(
pool.imap(fn, queryset.iterator(chunk_size=100))
):
if errors:
self.stderr.write("\n".join(errors))

progress = "*" * (50 * index // count)
self.stdout.write(
f"\r|{progress.ljust(50)}| {index + 1}/{count}", ending=""
)

# Save instance once for good measure; fills in width/height
# if not done already
instance._skip_generate_files = True
instance.save()

finally:
pool.close()
pool.join()

self.stdout.write("\r|{}| {}/{}".format("*" * 50, count, count))

Expand Down

0 comments on commit 9087c8e

Please sign in to comment.