Merge pull request #11 from VachaLab/v0.12

v0.12

Author: Ladme

.github/workflows/release.yml

@@ -12,9 +12,14 @@ jobs:
     runs-on: [self-hosted, Linux, X64]
 
     steps:
+      - name: Clean workspace
+        run: |
+          rm -rf "$GITHUB_WORKSPACE/.git"
+
       - uses: actions/checkout@v5
         with:
           fetch-depth: 0
+          ref: ${{ github.event.release.tag_name }}
 
       - name: Fix git ownership
         run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

.gitignore

@@ -18,4 +18,7 @@ htmlcov/
 docs/
 
 # Version
-src/qq_lib/_version.py
+src/qq_lib/_version.py
+
+# Playground
+src/_playground*.py

.readthedocs.yaml

@@ -1,11 +1,11 @@
 version: 2
 
 build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-  commands:
-    - pip install uv
-    - uv pip install --system pdoc
-    - uv pip install --system .
-    - pdoc src/qq_lib -d google -o $READTHEDOCS_OUTPUT/html
+    os: ubuntu-22.04
+    tools:
+        python: "3.13"
+    commands:
+        - pip install uv
+        - uv pip install --system pdoc
+        - uv pip install --system .
+        - pdoc src/qq_lib -d google -o $READTHEDOCS_OUTPUT/html

CHANGELOG.md

@@ -1,3 +1,18 @@
+## Version 0.12.0
+### Job collections
+- **BREAKING CHANGE**: The `-s` option of `qq info` now corresponds to the `--server` option, not to the `--short` flag, for consistency with other commands. Use `--brief`/`-b` or the legacy long variant `--short` instead to get a brief summary of the job.
+- The `qq killall` command has been deprecated in favor of `qq kill --all`, which has very similar behavior. It can still be used but will be completely removed in a future version of qq.
+- `qq info`, `qq kill`, `qq sync`, `qq respawn`, `qq wipe`, and `qq go` can now accept one or more directories and operate on jobs within them. Job resolution is parallelized and therefore much faster than calling these commands individually. Some of these commands also accept the `--all` flag to operate on all unfinished jobs on the given batch server.
+- `qq submit` now supports submitting multiple scripts at once. Submission is parallelized and therefore much faster than submitting scripts one by one.
+- `qq clear` can now clear runtime files from multiple directories at once.
+- *For more information about job collections, read [the manual](https://vachalab.github.io/qq-manual/job_collections.html).*
+
+### Other changes
+- If a loop job does not create any archive files for the next cycle, `qq` creates an empty `.init` file fulfilling the loop job's requirements.
+- If a loop job-specific or continuous job-specific option is used either on the command line or inside the submitted script, a warning is printed to inform the user that the option will be ignored.
+
+***
+
 ## Version 0.11
 ### qq respawn
 - Failed or killed jobs can now be easily "respawned" using `qq respawn`. When respawning a job, qq will remove the working directory of the failed job, clear all runtime files, and resubmit the job with the same parameters as before.

scripts/qq_scripts/gmx-eta

@@ -5,7 +5,7 @@
 
 """
 Get the estimated time of a Gromacs simulation finishing.
-Version qq 0.11.0.
+Version qq 0.12.0.
 Requires `uv`: https://docs.astral.sh/uv
 """
 
@@ -16,7 +16,7 @@ Requires `uv`: https://docs.astral.sh/uv
 # ]
 #
 # [tool.uv.sources]
-# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.11.0" }
+# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.12.0" }
 # ///
 
 import argparse

scripts/qq_scripts/multi-check

@@ -5,7 +5,8 @@
 
 """
 Check the state of qq jobs in multiple directories.
-Version qq 0.11.0.
+THIS SCRIPT IS DEPRECATED. YOU CAN USE `qq info -d <directories>` INSTEAD.
+Version qq 0.12.0.
 Requires `uv`: https://docs.astral.sh/uv
 """
 
@@ -16,7 +17,7 @@ Requires `uv`: https://docs.astral.sh/uv
 # ]
 #
 # [tool.uv.sources]
-# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.11.0" }
+# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.12.0" }
 # ///
 
 import argparse

scripts/qq_scripts/multi-kill

@@ -5,7 +5,8 @@
 
 """
 Kill qq jobs in multiple directories.
-Version qq 0.11.0.
+THIS SCRIPT IS DEPRECATED. YOU CAN USE `qq kill -d <directories>` INSTEAD.
+Version qq 0.12.0.
 Requires `uv`: https://docs.astral.sh/uv
 """
 
@@ -16,7 +17,7 @@ Requires `uv`: https://docs.astral.sh/uv
 # ]
 #
 # [tool.uv.sources]
-# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.11.0" }
+# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.12.0" }
 # ///
 
 import argparse

scripts/qq_scripts/multi-submit

@@ -5,7 +5,8 @@
 
 """
 Submit qq jobs from multiple directories.
-Version qq 0.11.0.
+THIS SCRIPT IS DEPRECATED. YOU CAN USE `qq submit <directories>/script.sh` INSTEAD.
+Version qq 0.12.0.
 Requires `uv`: https://docs.astral.sh/uv
 """
 
@@ -16,7 +17,7 @@ Requires `uv`: https://docs.astral.sh/uv
 # ]
 #
 # [tool.uv.sources]
-# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.11.0" }
+# qq = { git = "https://github.com/VachaLab/qq.git", tag = "v0.12.0" }
 # ///
 
 import argparse
@@ -65,7 +66,7 @@ def submit_job(base_submitter: Submitter, directory: str) -> bool:
 
     with suppress(QQError):
         # remove runtime files in the directory, if possible
-        clearer = Clearer(Path(directory))
+        clearer = Clearer([Path(directory)])
         clearer.clear()
 
     # make sure that the directory is suitable for submission

src/qq_lib/archive/archiver.py

@@ -74,7 +74,7 @@ def from_archive(self, dir: Path, cycle: int | None = None) -> None:
             QQError: If file transfer fails.
         """
         if not (
-            files := self._get_files(
+            files := self.get_files_matching_pattern(
                 self._archive, self._input_machine, self._archive_format, cycle, False
             )
         ):
@@ -108,7 +108,11 @@ def to_archive(self, dir: Path) -> None:
         Raises:
             QQError: If file transfer or removal fails.
         """
-        if not (files := self._get_files(dir, None, self._archive_format, None, False)):
+        if not (
+            files := self.get_files_matching_pattern(
+                dir, None, self._archive_format, None, False
+            )
+        ):
             logger.debug("Nothing to archive.")
             return
 
@@ -153,7 +157,7 @@ def archive_runtime_files(self, job_name: str, cycle: int) -> None:
             QQError: If moving the runtime files fails.
         """
         if not (
-            files := self._get_files(
+            files := self.get_files_matching_pattern(
                 self._input_dir,
                 self._input_machine,
                 # only use the stem of the job name, the extension will not be matched
@@ -183,7 +187,7 @@ def archive_runtime_files(self, job_name: str, cycle: int) -> None:
             wait_seconds=CFG.archiver.retry_wait,
         ).run()
 
-    def _get_files(
+    def get_files_matching_pattern(
         self,
         directory: Path,
         host: str | None,
@@ -205,7 +209,7 @@ def _get_files(
             include_qq_files (bool): Whether to include qq runtime files. Defaults to False.
 
         Returns:
-            list[Path]: A list of absolute paths to matching files.
+            list[Path]: A list of absolute (logical) paths to matching files.
         """
         if cycle and is_printf_pattern(pattern):
             try:
@@ -248,6 +252,16 @@ def _get_files(
             if regex.search(f.stem) and f.suffix not in CFG.suffixes.all_suffixes
         ]
 
+    def create_init_file(self, cycle: int) -> None:
+        """
+        Create an empty init file for the given cycle.
+        Used as a fallback when no valid archive file is produced, ensuring
+        the next iteration of the loop job can proceed normally.
+        Args:
+            cycle (int): The index of the next cycle of the loop job.
+        """
+        Path(f"{self._archive_format % cycle}.init").touch()
+
     @staticmethod
     def _prepare_regex_pattern(pattern: str) -> re.Pattern[str]:
         """

src/qq_lib/batch/interface/interface.py

@@ -296,6 +296,29 @@ def get_batch_job(cls, job_id: str) -> TBatchJob:
             f"get_batch_job method is not implemented for {cls.__name__}"
         )
 
+    @classmethod
+    def get_batch_jobs_from_ids(cls, job_ids: list[str]) -> list[TBatchJob]:
+        """
+        Retrieve information about multiple jobs from the batch system.
+
+        Batch jobs should be returned in the same order as they appear in `job_ids`.
+        A TBatchJob object should be returned for each job id, even if the job
+        no longer exists or its information is unavailable.
+
+        Array jobs should NOT be expanded into their individual tasks.
+
+        The default implementation is to call `get_batch_job` for each job id.
+        This implementation may be inefficient for large numbers of job ids and
+        should be overriden by subclasses.
+
+        Args:
+            job_ids (list[str]): List of job identifiers.
+
+        Returns:
+            list[TBatchJob]: List of TBatchJob objects, one for each job id.
+        """
+        return [cls.get_batch_job(id) for id in job_ids]
+
     @classmethod
     def get_unfinished_batch_jobs(
         cls, user: str, server: str | None = None