diff --git a/imagefield/management/commands/process_imagefields.py b/imagefield/management/commands/process_imagefields.py index b2c66a2..42de7e4 100644 --- a/imagefield/management/commands/process_imagefields.py +++ b/imagefield/management/commands/process_imagefields.py @@ -1,5 +1,5 @@ import sys -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor from fnmatch import fnmatch from functools import partial @@ -88,16 +88,19 @@ def _process_field(self, field, options): ) fn = partial( - self._process_instance, + _process_instance, field=field, housekeep=options.get("housekeep"), force=options.get("force"), ) - with ThreadPoolExecutor() as executor: - for index, instance in enumerate( + with ProcessPoolExecutor() as executor: + for index, (instance, errors) in enumerate( executor.map(fn, queryset.iterator(chunk_size=100)) ): + if errors: + self.stderr.write("\n".join(errors)) + progress = "*" * (50 * index // count) self.stdout.write( f"\r|{progress.ljust(50)}| {index + 1}/{count}", ending="" @@ -110,16 +113,18 @@ def _process_field(self, field, options): self.stdout.write("\r|{}| {}/{}".format("*" * 50, count, count)) - def _process_instance(self, instance, field, housekeep, **kwargs): - fieldfile = getattr(instance, field.name) - for key in field.formats: - try: - fieldfile.process(key, **kwargs) - except Exception as exc: - self.stderr.write( - f"Error while processing {fieldfile.name} ({field.field_label}, #{instance.pk}):\n{exc}\n" - ) - if housekeep == "blank-on-failure": - field.save_form_data(instance, "") - return instance +def _process_instance(instance, field, housekeep, **kwargs): + fieldfile = getattr(instance, field.name) + for key in field.formats: + try: + fieldfile.process(key, **kwargs) + except Exception as exc: + if housekeep == "blank-on-failure": + field.save_form_data(instance, "") + + return instance, [ + f"Error while processing {fieldfile.name} ({field.field_label}, #{instance.pk}):\n{exc}\n" + ] + + return instance, None