[#310] Run Ruff check and format on codebase

.github/workflows/ruff-action.yaml

@@ -0,0 +1,26 @@
+name: ruff-action
+
+on: pull_request
+
+jobs:
+  ruff-format:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Ruff formatter (check only)
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "format --check --diff"
+
+  ruff-check:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Run Ruff check (check only)
+        uses: astral-sh/ruff-action@v3
+        with:
+          args: "check --diff"

irods_capability_automated_ingest/celery.py

@@ -1,5 +1,6 @@
-from . import custom_event_handler, sync_logging
+from . import sync_logging
 
+from billiard import current_process
 from celery import Celery
 from celery.signals import task_prerun, task_postrun
 

irods_capability_automated_ingest/char_map_util.py

@@ -1,4 +1,3 @@
-import base64
 import hashlib
 import logging
 import re
@@ -16,9 +15,9 @@ def _re_wrapper(regex):
     return lambda ch: type(ch) is str and len(ch) == 1 and regex.match(ch)
 
 
-_string_replace = lambda _string, _map: _string.translate(
-    {ord(k): v for k, v in _map.items() if v is not None}
-)
+def _string_replace(_string, _map):
+    return _string.translate({ord(k): v for k, v in _map.items() if v is not None})
+
 
 _logger = logging.getLogger("char_map_util")
 
@@ -72,10 +71,12 @@ def _fallback(name=None):
         return h.digest()
 
 
-_change_encoding_test = lambda c: c
-_change_encoding_default = lambda c: (
-    chr(c).encode("utf8") if type(c) is int else c.encode("utf8")
-)
+def _change_encoding_test(c):
+    return c
+
+
+def _change_encoding_default(c):
+    return chr(c).encode("utf8") if isinstance(c, int) else c.encode("utf8")
 
 
 # must be called after first use of _encoded_differences()
@@ -109,7 +110,6 @@ def translate_string(s, mp):
 
 
 def _encoded_differences(filename, MapFn=None, xfunc=_change_encoding_default):
-    rx = _allowed_of_type("radixchars", map_fn=MapFn)
     newname = translate_string(filename, MapFn)
     gen = (
         (tuple(xfunc(_) for _ in a), b)

irods_capability_automated_ingest/examples/coll_create_pre_and_post.py

@@ -1,5 +1,3 @@
-import os
-
 from irods_capability_automated_ingest.core import Core
 from irods_capability_automated_ingest.utils import Operation
 

irods_capability_automated_ingest/examples/coll_modify_pre_and_post.py

@@ -1,5 +1,3 @@
-import os
-
 from irods_capability_automated_ingest.core import Core
 from irods_capability_automated_ingest.utils import Operation
 

irods_capability_automated_ingest/examples/data_obj_create_pre_and_post.py

@@ -1,5 +1,3 @@
-import os
-
 from irods_capability_automated_ingest.core import Core
 from irods_capability_automated_ingest.utils import Operation
 

irods_capability_automated_ingest/examples/data_obj_modify_pre_and_post.py

@@ -1,5 +1,3 @@
-import os
-
 from irods_capability_automated_ingest.core import Core
 from irods_capability_automated_ingest.utils import Operation
 

irods_capability_automated_ingest/examples/no_retry.py

@@ -10,7 +10,6 @@ def operation(session, meta, **options):
 
     @staticmethod
     def pre_data_obj_create(hdlr_mod, logger, session, meta, *args, **options):
-        target = meta["target"]
         path = meta["path"]
 
         r = get_redis(meta["config"])

irods_capability_automated_ingest/examples/retry.py

@@ -19,7 +19,6 @@ def operation(session, meta, **options):
     @staticmethod
     def pre_data_obj_create(hdlr_mod, logger, session, meta, *args, **options):
         path = meta["path"]
-        target = meta["target"]
 
         r = get_redis(meta["config"])
         failures = r.get("failures:" + path)

irods_capability_automated_ingest/irods/filesystem.py

@@ -192,7 +192,6 @@ def update_metadata(hdlr_mod, logger, session, meta, **options):
 
     data_obj_info = {"objPath": dest_dataobj_logical_fullpath}
 
-    outdated_repl_nums = []
     found = False
 
     resc_name = event_handler.to_resource(session, **options)
@@ -412,7 +411,6 @@ def sync_dir_meta(hdlr_mod, logger, session, meta, **options):
 
 def sync_data_from_dir(hdlr_mod, meta, logger, content, **options):
     target = meta["target"]
-    path = meta["path"]
 
     event_handler = custom_event_handler.custom_event_handler(meta)
     session = irods_utils.irods_session(

irods_capability_automated_ingest/irods/irods_utils.py

@@ -1,17 +1,17 @@
 from .. import custom_event_handler, sync_logging
 from ..redis_utils import get_redis
-from ..utils import DeleteMode, Operation
+from ..utils import DeleteMode
 
-from irods.exception import CollectionDoesNotExist, NetworkException
+from irods.exception import NetworkException
 from irods.models import Collection, DataObject, Resource
 from irods.session import iRODSSession
 
-import base64
 import json
 import os
 import redis_lock
 import ssl
 import threading
+import time
 
 irods_session_map = {}
 irods_session_timer_map = {}
@@ -195,7 +195,6 @@ def create_dirs(logger, session, meta, **options):
 
 def create_dir(hdlr_mod, logger, session, meta, **options):
     target = meta["target"]
-    path = meta["path"]
     logger.info("creating collection " + target)
     session.collections.create(target)
 

irods_capability_automated_ingest/irods/s3_bucket.py

@@ -2,15 +2,19 @@
 from .. import custom_event_handler
 from ..utils import Operation
 
+from irods.parallel import _Multipart_close_manager
 from irods.models import Resource, DataObject, Collection
 import irods.keywords as kw
 
 from minio import Minio
 
 import base64
+import concurrent
 import hashlib
 import io
+import multiprocessing
 import os
+import threading
 
 
 def parallel_upload_from_S3(
@@ -390,17 +394,12 @@ def no_op(hdlr_mod, logger, session, meta, **options):
 
 def sync_file(hdlr_mod, logger, session, meta, scanner, op, **options):
     dest_dataobj_logical_fullpath = meta["target"]
-    source_physical_fullpath = meta["path"]
-    b64_path_str = meta.get("b64_path_str")
 
     event_handler = custom_event_handler.custom_event_handler(meta)
     resc_name = event_handler.to_resource(session, **options)
     if resc_name is not None:
         options["destRescName"] = resc_name
 
-    if b64_path_str is not None:
-        source_physical_fullpath = base64.b64decode(b64_path_str)
-
     logger.info(
         "syncing object %s, options = %s" % (dest_dataobj_logical_fullpath, options)
     )
@@ -441,7 +440,6 @@ def update_metadata(hdlr_mod, logger, session, meta, **options):
 
     data_obj_info = {"objPath": dest_dataobj_logical_fullpath}
 
-    outdated_repl_nums = []
     found = False
 
     resc_name = event_handler.to_resource(session, **options)
@@ -661,7 +659,6 @@ def sync_dir_meta(hdlr_mod, logger, session, meta, **options):
 
 def sync_data_from_dir(hdlr_mod, meta, logger, content, **options):
     target = meta["target"]
-    path = meta["path"]
 
     event_handler = custom_event_handler.custom_event_handler(meta)
     session = irods_utils.irods_session(

irods_capability_automated_ingest/irods_sync.py

@@ -166,7 +166,7 @@ def add_arguments(parser):
 
 def handle_start(args):
     ex_file_arg = args.exclude_file_type
-    if ex_file_arg != None:
+    if ex_file_arg is not None:
         ex_arg_list = [x.strip() for x in ex_file_arg[0].split(",")]
 
     check_event_handler(args.event_handler)

irods_capability_automated_ingest/redis_key.py

@@ -1,6 +1,4 @@
-import json
 import time
-import traceback
 
 MAX_RETRIES = 10
 
@@ -22,7 +20,7 @@ def retry(self, func, *args, max_retries=MAX_RETRIES):
             try:
                 res = func(*args)
                 return res
-            except Exception as err:
+            except Exception:
                 retries += 1
 
                 # logger.info('Retrying. retries=' + str(retries), max_retries=max_retries, func=func, args=args, err=err, stacktrace=traceback.extract_tb(err.__traceback__))

irods_capability_automated_ingest/sync_actions.py

@@ -138,10 +138,7 @@ def start_job(data):
     restart_queue = data["restart_queue"]
     sychronous = data["synchronous"]
     progress = data["progress"]
-    s3_region_name = data["s3_region_name"]
-    s3_endpoint_domain = data["s3_endpoint_domain"]
     s3_keypair = data["s3_keypair"]
-    s3_multipart_chunksize = data["s3_multipart_chunksize_in_mib"]
     logger = sync_logging.get_sync_logger(logging_config)
     data_copy = data.copy()
 

irods_capability_automated_ingest/tasks/filesystem_tasks.py

@@ -11,8 +11,6 @@
 
 from irods.exception import (
     CollectionDoesNotExist,
-    DataObjectDoesNotExist,
-    PycommandsException,
 )
 
 # See https://github.com/celery/celery/issues/5362 for information about billiard and Celery.

irods_capability_automated_ingest/tasks/s3_bucket_tasks.py

@@ -15,9 +15,7 @@
 import base64
 import datetime
 import os
-import re
 import redis_lock
-import stat
 import time
 import traceback
 
@@ -156,11 +154,6 @@ def s3_bucket_sync_path(self, meta):
                     index=current_process().index,
                 )
 
-        exclude_file_name = meta["exclude_file_name"]
-        exclude_directory_name = meta["exclude_directory_name"]
-        file_regex = [re.compile(r) for r in exclude_file_name]
-        dir_regex = [re.compile(r) for r in exclude_directory_name]
-
         for obj in itr:
             obj_stats = {}
 

irods_capability_automated_ingest/test/test_delete_modes.py

@@ -8,17 +8,13 @@
 import textwrap
 import time
 
-from irods.data_object import irods_dirname, irods_basename
 from irods.exception import CollectionDoesNotExist
-from irods.meta import iRODSMeta
-from irods.models import Collection, DataObject
 from irods.session import iRODSSession
 
 from irods_capability_automated_ingest.celery import app
 from irods_capability_automated_ingest.redis_utils import get_redis
 from irods_capability_automated_ingest.sync_job import sync_job
 from irods_capability_automated_ingest.utils import DeleteMode, Operation
-import irods_capability_automated_ingest.examples
 
 from . import test_lib
 
@@ -559,10 +555,10 @@ def test_PUT_APPEND_and_DO_NOT_DELETE_does_not_delete_collections(self):
         # Case 20
         self.do_DO_NOT_DELETE_does_not_delete_collections(Operation.PUT_APPEND)
 
-    def test_PUT_SYNC_and_UNREGISTER_are_incompatible(self):
+    def test_PUT_APPEND_and_UNREGISTER_are_incompatible(self):
         # Case 21
         self.do_incompatible_operation_and_delete_mode(
-            Operation.PUT_SYNC, DeleteMode.UNREGISTER
+            Operation.PUT_APPEND, DeleteMode.UNREGISTER
         )
 
     def test_PUT_APPEND_and_TRASH_deletes_collections(self):