Skip to content

Commit

Permalink
Rewrite the code to use multiprocessing
Browse files Browse the repository at this point in the history
  • Loading branch information
matthiask committed Jan 9, 2025
1 parent f80d27c commit f3921dd
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 3 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@ Change log
Next version
~~~~~~~~~~~~

- Rewrote ``process_imagefields`` to use the multiprocessing module, which
hopefully improves compatibility on macOS.


0.21 (2024-12-09)
~~~~~~~~~~~~~~~~~
Expand Down
12 changes: 9 additions & 3 deletions imagefield/management/commands/process_imagefields.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import multiprocessing as mp
import sys
from concurrent.futures import ProcessPoolExecutor
from fnmatch import fnmatch
from functools import partial

Expand Down Expand Up @@ -87,16 +87,18 @@ def _process_field(self, field, options):
force=options.get("force"),
)

pool = mp.Pool()

fn = partial(
_process_instance,
field=field,
housekeep=options.get("housekeep"),
force=options.get("force"),
)

with ProcessPoolExecutor() as executor:
try:
for index, (instance, errors) in enumerate(
executor.map(fn, queryset.iterator(chunk_size=100))
pool.imap(fn, queryset.iterator(chunk_size=100))
):
if errors:
self.stderr.write("\n".join(errors))
Expand All @@ -111,6 +113,10 @@ def _process_field(self, field, options):
instance._skip_generate_files = True
instance.save()

finally:
pool.close()
pool.join()

self.stdout.write("\r|{}| {}/{}".format("*" * 50, count, count))


Expand Down

0 comments on commit f3921dd

Please sign in to comment.