ArraySubmitterHelper - broken

Author: Ladme

src/qq_lib/core/array_spec.py

@@ -1,6 +1,8 @@
 # Released under MIT License.
 # Copyright (c) 2025-2026 Ladislav Bartos and Robert Vacha Lab
 
+from collections.abc import Iterator, Sequence
+
 from qq_lib.core.error import QQError
 
 type ArrayElement = int | tuple[int, int] | tuple[int, int, int]
@@ -11,23 +13,36 @@ class ArraySpec:
     Specification for job-array task indices.
 
     Args:
-        elements (list[ArrayElement]): Non-empty list of indices and ranges.
+        elements (Sequence[ArrayElement]): Non-empty list of indices and ranges.
 
     Raises:
         QQError: If the list is empty or any element violates constraints.
     """
 
-    def __init__(self, elements: list[ArrayElement]):
+    def __init__(self, elements: Sequence[ArrayElement]):
         _validate_elements(elements)
         self.elements = _merge_elements(elements)
 
-
-def _validate_elements(elements: list[ArrayElement]) -> None:
+    def __iter__(self) -> Iterator[int]:
+        """
+        Yields individual task indices from the array specification.
+        """
+        for element in self.elements:
+            match element:
+                case int(value):
+                    yield value
+                case (int(start), int(stop)):
+                    yield from range(start, stop + 1)
+                case (int(start), int(stop), int(step)):
+                    yield from range(start, stop + 1, step)
+
+
+def _validate_elements(elements: Sequence[ArrayElement]) -> None:
     """
     Validate a list of array elements.
 
     Args:
-        elements (list[ArrayElement]): Non-empty list of indices and ranges.
+        elements (Sequence[ArrayElement]): Non-empty sequence of indices and ranges.
 
     Raises:
         QQError: If an element has an unsupported type.
@@ -72,7 +87,7 @@ def _validate_elements(elements: list[ArrayElement]) -> None:
                 )
 
 
-def _merge_elements(elements: list[ArrayElement]) -> list[ArrayElement]:
+def _merge_elements(elements: Sequence[ArrayElement]) -> list[ArrayElement]:
     """
     Merge array elements into a shorter equivalent list.
 
@@ -82,7 +97,7 @@ def _merge_elements(elements: list[ArrayElement]) -> list[ArrayElement]:
     will not be collapsed into `(1, 5, 2)`.
 
     Args:
-        elements (list[ArrayElement]): Non-empty list of indices and ranges.
+        elements (Sequence[ArrayElement]): Non-empty sequence of indices and ranges.
 
     Returns:
         list[ArrayElement]: Equivalent list with overlapping and adjacent

src/qq_lib/core/common.py

@@ -9,6 +9,7 @@
 """
 
 import re
+from collections.abc import Sequence
 from datetime import timedelta
 from functools import lru_cache
 from pathlib import Path
@@ -99,7 +100,7 @@ def get_info_file(directory: Path) -> Path:
     """
     Locate the qq job info file in a directory.
 
-    This function searches for files matching the `QQ_INFO_SUFFIX` in the
+    This function searches for files with suffix `CFG.suffixes.qq_info` in the
     provided directory. It raises an error if none or multiple info files are found.
 
     Args:
@@ -124,7 +125,7 @@ def get_info_files(directory: Path) -> list[Path]:
     """
     Retrieve all qq job info files in a directory.
 
-    This function searches for files matching the `QQ_INFO_SUFFIX` in the
+    This function searches for files with suffix `CFG.suffixes.qq_info` in the
     provided directory. The files are sorted by their last modification time
     (with the newest modified file being last in the list).
 
@@ -140,6 +141,51 @@ def get_info_files(directory: Path) -> list[Path]:
     return sorted(info_files, key=lambda f: f.stat().st_mtime)
 
 
+def get_array_file(directory: Path) -> Path:
+    """
+    Located the qq array file in a directory.
+
+    This function searches for files with suffix `CFG.suffixes.qq_array` in the
+    provided directory. It raises an error if none or multiple array files are found.
+
+    Args:
+        directory (Path): The directory to search in.
+
+    Returns:
+        Path: The Path object of the detected qq array file.
+
+    Raises:
+        QQError: If no array file is found or multiple array files are detected.
+    """
+    array_files = get_array_files(directory)
+    if len(array_files) == 0:
+        raise QQError("No qq array file found.")
+    if len(array_files) > 1:
+        raise QQError("Multiple qq array files found.")
+
+    return array_files[0]
+
+
+def get_array_files(directory: Path) -> list[Path]:
+    """
+    Retrieve all qq array files in a directory.
+
+    This function searches for files with suffix `CFG.suffixes.qq_array` in the
+    provided directory. The files are sorted by their last modification time
+    (with the newest modified file being last in the list).
+
+    Args:
+        directory (Path): The directory to search in.
+
+    Returns:
+        list[Path]: A list of Path objects representing the detected qq array files.
+    """
+    array_files = get_files_with_suffix(directory, CFG.suffixes.qq_array)
+    logger.debug(f"Detected the following qq array files: {array_files}.")
+
+    return sorted(array_files, key=lambda f: f.stat().st_mtime)
+
+
 def get_info_file_from_job_id(job_id: str) -> Path:
     """
     Get path to the qq info file corresponding to a job with the given ID.
@@ -766,3 +812,30 @@ def default_resubmit_from_hosts() -> str:
     # if no batch system is available
     except QQError:
         return "??? (no batch system detected)"
+
+
+def subset_indices(a: Sequence[str | Path], b: Sequence[str | Path]) -> list[int]:
+    """
+    Return indices in *a* corresponding to each element of *b*.
+
+    *b* must be a subset of *a*; both lists are assumed to contain no
+    duplicates.
+
+    Args:
+        a (Sequence[str | Path]): The reference sequence of strings or Paths.
+        b (Sequence[str | Path]): The query list whose elements must all appear in *a*.
+
+    Returns:
+        list[int]: A list of the same length as *b* where the i-th entry is
+            the index of `b[i]` in *a*.
+
+    Raises:
+        QQError: If *b* contains elements not present in *a*.
+    """
+    index_of: dict[str | Path, int] = {value: idx for idx, value in enumerate(a)}
+
+    missing: list[str | Path] = [item for item in b if item not in index_of]
+    if missing:
+        raise QQError(f"b is not a subset of a. Missing elements: {missing}.")
+
+    return [index_of[item] for item in b]

src/qq_lib/info/array_informer.py

@@ -40,6 +40,16 @@ def batch_system(self) -> type[BatchInterface]:
         """
         return self.array_info.batch_system
 
+    @property
+    def tasks(self) -> list[Informer | None]:
+        """
+        Return the list of Informers for the tasks in this array.
+
+        Returns:
+            list[Informer | None]: The list of Informers for the tasks in this array.
+        """
+        return self._tasks
+
     @classmethod
     def from_file(cls, file: Path, host: str | None = None) -> Self:
         """
@@ -179,13 +189,18 @@ def all_tasks_in_state(self, states: list[RealState]) -> bool:
         if not self._batch_info_loaded:
             self.load_batch_info()
 
-        return all(informer.get_real_state() in states for informer in self._tasks)
+        return all(
+            informer is not None and informer.get_real_state() in states
+            for informer in self._tasks
+        )
 
     @staticmethod
-    def _get_informers_for_tasks(array_info: ArrayInfo) -> list[Informer]:
+    def _get_informers_for_tasks(array_info: ArrayInfo) -> list[Informer | None]:
         """
         Create informers for each task in the array job.
+
         The informers are returned in the order corresponding to the directory order.
+        If an informer cannot be created for a task, it is replaced with `None`,
 
         Batch job info is NOT automatically loaded for the informers.
 
@@ -199,7 +214,10 @@ def _get_informers_for_tasks(array_info: ArrayInfo) -> list[Informer]:
         informers = []
         for dir in array_info.task_dirs:
             info_path = construct_info_file_path(dir, array_info.job_name)
-            informers.append(Informer.from_file(info_path))
+            try:
+                informers.append(Informer.from_file(info_path))
+            except QQError:
+                informers.append(None)
         return informers
 
     def _get_batch_tasks(self) -> list[BatchJobInterface]:
@@ -272,6 +290,10 @@ def _match_batch_tasks_to_informers(
         Match batch tasks to informers based on task numbers.
         """
         for informer in self._tasks:
+            # skip directories for which informers are missing
+            if informer is None:
+                continue
+
             # get task number from the informer to match it with the batch system info
             if (task_info := informer.info.task_info) is None:
                 raise QQError(