From 8ef1efd5f7e65c27f952f9f23bf856d73e5091e3 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:37:28 +0100 Subject: [PATCH 01/30] ci: add test for sra download --- .github/workflows/main.yml | 12 +++++++++++ .test/config_sra/config.yaml | 39 ++++++++++++++++++++++++++++++++++++ .test/config_sra/samples.tsv | 3 +++ .test/config_sra/units.tsv | 3 +++ 4 files changed, 57 insertions(+) create mode 100644 .test/config_sra/config.yaml create mode 100644 .test/config_sra/samples.tsv create mode 100644 .test/config_sra/units.tsv diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7ff41d23..49d9c816 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -68,3 +68,15 @@ jobs: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_complex/config.yaml --report report.zip" + - name: Test workflow (sra file download, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + - name: Test report (sra file download, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_sra/config.yaml --report report.zip" diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml new file mode 100644 index 00000000..35a83e5c --- /dev/null +++ b/.test/config_sra/config.yaml @@ -0,0 +1,39 @@ +# For a fully commented config.yaml file, see the main `config/config.yaml` +# example file. + +samples: config_sra/samples.tsv +units: config_sra/units.tsv + + +ref: + species: gallus_gallus + release: "115" + build: GRCg7b + + +trimming: + activate: True + +mergeReads: + activate: False + +pca: + activate: True + labels: + - treatment + +diffexp: + variables_of_interest: + condition: + base_level: PBS_0H + batch_effects: "" + contrasts: + treated-vs-untreated: + variable_of_interest: treatment + level_of_interest: H5N1_2.2_12H + model: ~treatment + +params: + star: + index: "" + align: "" diff --git a/.test/config_sra/samples.tsv b/.test/config_sra/samples.tsv new file mode 100644 index 00000000..8780c693 --- /dev/null +++ b/.test/config_sra/samples.tsv @@ -0,0 +1,3 @@ +sample_name treatment +H5N1_2.2_12H_brain_1 H5N1_2.2_12H +PBS_0H_brain_1 PBS_0H \ No newline at end of file diff --git a/.test/config_sra/units.tsv b/.test/config_sra/units.tsv new file mode 100644 index 00000000..8f3be7e5 --- /dev/null +++ b/.test/config_sra/units.tsv @@ -0,0 +1,3 @@ +sample_name unit_name fq1 fq2 sra fastp_adapters fastp_extra strandedness +H5N1_2.2_12H_brain_1 lane1 ERR5043451 +PBS_0H_brain_1 lane1 ERR5043769 \ No newline at end of file From 23ec3c32a63989dafd0772ef5d107f4201a2fe51 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:37:58 +0100 Subject: [PATCH 02/30] docs: copy some nice detailed explanations in example config.yaml from the rna-seq-kallisto-sleuth workflow --- config/config.yaml | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index b0b9266b..9f1337bd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,11 +7,35 @@ units: config/units.tsv ref: - # Ensembl species name + # ensembl species name: + # This species needs to be available for download via the Ensembl FTP site. + # For a quick check, see the Ensembl species list: + # https://www.ensembl.org/info/about/species.html + # For full valid species names, consult the respective table for the release + # you specify, for example for ‘115’ this is at: + # https://ftp.ensembl.org/pub/release-115/species_EnsemblVertebrates.txt + # And to browse available downloads in more detail, see the FTP server: + # https://ftp.ensembl.org/pub/ species: homo_sapiens - # Ensembl release (make sure to take one where snpeff data is available, check 'snpEff databases' output) - release: 100 - # Genome build + # ensembl release version: + # Update this to the latest working version, when you first set up a new + # analysis on a dataset. You can usually find the latest release in the + # Ensembl blog, by looking at the latest posts of the release category: + # https://www.ensembl.info/category/01-release/ + # Later, it only makes sense to update (or downgrade) the release versions + # if either (a) the version you are using consistently fails to download + # (some Ensembl release versions are just broken) or (b) you know that a + # newer version will include changes that will fix some error or adds + # transcripts that will be relevant to your analysis. + release: "115" + # genome build: + # Usually, this should just be the main build listed in: + # https://ftp.ensembl.org/pub/release-115/species_EnsemblVertebrates.txt + # For example, for homo_sapiens, you strip the assembly column entry + # "GRCh38.p12" down to "GRCh38". If in doubt, navigate to the respective + # cdna folder on the FTP server, and look for the correct build in the + # file names there. For example "GRCh38" in: + # https://ftp.ensembl.org/pub/release-115/fasta/homo_sapiens/cdna/ build: GRCh38 trimming: From 64a651880ef221baa57367f6cf5db89fab191f6f Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:41:19 +0100 Subject: [PATCH 03/30] chore: prettier yaml files --- .github/workflows/main.yml | 118 ++++++++++++++++++------------------- config/config.yaml | 2 - 2 files changed, 59 insertions(+), 61 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 49d9c816..92cfb4ca 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,28 +11,28 @@ jobs: formatting: runs-on: ubuntu-latest steps: - - name: Checkout with submodules - uses: actions/checkout@v3 - with: - submodules: recursive - fetch-depth: 0 - - name: Formatting - uses: github/super-linter@v5 - env: - VALIDATE_ALL_CODEBASE: false - DEFAULT_BRANCH: master - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VALIDATE_SNAKEMAKE_SNAKEFMT: true + - name: Checkout with submodules + uses: actions/checkout@v3 + with: + submodules: recursive + fetch-depth: 0 + - name: Formatting + uses: github/super-linter@v5 + env: + VALIDATE_ALL_CODEBASE: false + DEFAULT_BRANCH: master + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VALIDATE_SNAKEMAKE_SNAKEFMT: true linting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: Linting - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml --lint" + - uses: actions/checkout@v3 + - name: Linting + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_complex/config.yaml --lint" run-workflow: runs-on: ubuntu-latest @@ -40,43 +40,43 @@ jobs: - linting - formatting steps: - - name: Checkout repository with submodules - uses: actions/checkout@v3 - with: - submodules: recursive - - name: Test workflow (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - - name: Test report (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --report report.zip" - - name: Test workflow (multiple variables_of_interest, include batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - - name: Test report (multiple variables_of_interest, include batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml --report report.zip" - - name: Test workflow (sra file download, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - - name: Test report (sra file download, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_sra/config.yaml --report report.zip" + - name: Checkout repository with submodules + uses: actions/checkout@v3 + with: + submodules: recursive + - name: Test workflow (basic model, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_basic/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + - name: Test report (basic model, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_basic/config.yaml --report report.zip" + - name: Test workflow (multiple variables_of_interest, include batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + - name: Test report (multiple variables_of_interest, include batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_complex/config.yaml --report report.zip" + - name: Test workflow (sra file download, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + - name: Test report (sra file download, no batch_effects) + uses: snakemake/snakemake-github-action@v1.22.0 + with: + directory: .test + snakefile: workflow/Snakefile + args: "--configfile .test/config_sra/config.yaml --report report.zip" diff --git a/config/config.yaml b/config/config.yaml index 9f1337bd..1e7b976b 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -5,7 +5,6 @@ samples: config/samples.tsv # sample). units: config/units.tsv - ref: # ensembl species name: # This species needs to be available for download via the Ensembl FTP site. @@ -88,7 +87,6 @@ diffexp: # model: ~jointly_handled + treatment_1 + treatment_2 model: "" - # passing extra parameters to respective rules in the workflow params: star: From f2a1ac8c7e522563001aed20ecba37da2f764de4 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:43:49 +0100 Subject: [PATCH 04/30] fix: correct read infix in get_units_fastq() for sra-downloaded fastq files --- workflow/rules/common.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 0757353b..581fb217 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -57,7 +57,7 @@ def get_units_fastqs(wildcards): return expand( "sra/{accession}_{read}.fastq", accession=accession, - read=["R1", "R2"], + read=["1", "2"], ) if not is_paired_end(wildcards.sample): return [ From 9854d5958662e4fa50c6b6aeede1dcb77d19e92c Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:57:30 +0100 Subject: [PATCH 05/30] fix: use existing versions of star snakemake wrappers (3.3.7), to make report creation work --- workflow/rules/align.smk | 2 +- workflow/rules/ref.smk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/align.smk b/workflow/rules/align.smk index 0360c687..45b364d3 100644 --- a/workflow/rules/align.smk +++ b/workflow/rules/align.smk @@ -22,4 +22,4 @@ rule star_align: ), threads: 24 wrapper: - "v7.2.0/bio/star/align" + "v3.3.7/bio/star/align" diff --git a/workflow/rules/ref.smk b/workflow/rules/ref.smk index 7c77d7b3..c66cf5bc 100644 --- a/workflow/rules/ref.smk +++ b/workflow/rules/ref.smk @@ -67,4 +67,4 @@ rule star_index: extra=lookup(within=config, dpath="params/star/index", default=""), threads: 4 wrapper: - "v7.2.0/bio/star/index" + "v3.3.7/bio/star/index" From c54fa67093a1ff0af37853624b24ff55888d6540 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:15:10 +0100 Subject: [PATCH 06/30] ci: update all the GitHub Actions versions --- .github/workflows/conventional-prs.yaml | 2 +- .github/workflows/main.yml | 20 ++++++++++---------- .github/workflows/release-please.yaml | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/conventional-prs.yaml b/.github/workflows/conventional-prs.yaml index 210988fa..2a544c53 100644 --- a/.github/workflows/conventional-prs.yaml +++ b/.github/workflows/conventional-prs.yaml @@ -13,6 +13,6 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@v6 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 92cfb4ca..b79a377e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout with submodules - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive fetch-depth: 0 @@ -26,9 +26,9 @@ jobs: linting: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v6 - name: Linting - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile @@ -41,41 +41,41 @@ jobs: - formatting steps: - name: Checkout repository with submodules - uses: actions/checkout@v3 + uses: actions/checkout@v6 with: submodules: recursive - name: Test workflow (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_basic/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_basic/config.yaml --report report.zip" - name: Test workflow (multiple variables_of_interest, include batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (multiple variables_of_interest, include batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_complex/config.yaml --report report.zip" - name: Test workflow (sra file download, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (sra file download, no batch_effects) - uses: snakemake/snakemake-github-action@v1.22.0 + uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index 0a16d028..aec137a5 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -13,7 +13,7 @@ jobs: release-please: runs-on: ubuntu-latest steps: - - uses: google-github-actions/release-please-action@v3 + - uses: google-github-actions/release-please-action@v4 with: release-type: simple token: ${{ secrets.GITHUB_TOKEN }} From fe87a0f661e0a68c22be18ec283864729cc7184e Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:27:15 +0100 Subject: [PATCH 07/30] Revert "fix: use existing versions of star snakemake wrappers (3.3.7), to make report creation work" This reverts commit 9854d5958662e4fa50c6b6aeede1dcb77d19e92c. --- workflow/rules/align.smk | 2 +- workflow/rules/ref.smk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/rules/align.smk b/workflow/rules/align.smk index 45b364d3..0360c687 100644 --- a/workflow/rules/align.smk +++ b/workflow/rules/align.smk @@ -22,4 +22,4 @@ rule star_align: ), threads: 24 wrapper: - "v3.3.7/bio/star/align" + "v7.2.0/bio/star/align" diff --git a/workflow/rules/ref.smk b/workflow/rules/ref.smk index c66cf5bc..7c77d7b3 100644 --- a/workflow/rules/ref.smk +++ b/workflow/rules/ref.smk @@ -67,4 +67,4 @@ rule star_index: extra=lookup(within=config, dpath="params/star/index", default=""), threads: 4 wrapper: - "v3.3.7/bio/star/index" + "v7.2.0/bio/star/index" From 34e50d1cabc20d47c3dd76dfa805647e13fc99e5 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:28:37 +0100 Subject: [PATCH 08/30] ci: fix release-please-action namespace to new googleapis --- .github/workflows/release-please.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-please.yaml b/.github/workflows/release-please.yaml index aec137a5..546a7103 100644 --- a/.github/workflows/release-please.yaml +++ b/.github/workflows/release-please.yaml @@ -13,7 +13,7 @@ jobs: release-please: runs-on: ubuntu-latest steps: - - uses: google-github-actions/release-please-action@v4 + - uses: googleapis/release-please-action@v4 with: release-type: simple token: ${{ secrets.GITHUB_TOKEN }} From b42bbed9b5f5bcdc1f870e066ee815480f5d1d98 Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:30:11 +0100 Subject: [PATCH 09/30] fix: ensure `release:` in `config.yaml` files is integer --- .test/config_sra/config.yaml | 2 +- config/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index 35a83e5c..8054b222 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -7,7 +7,7 @@ units: config_sra/units.tsv ref: species: gallus_gallus - release: "115" + release: 115 build: GRCg7b diff --git a/config/config.yaml b/config/config.yaml index 1e7b976b..4b593948 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -26,7 +26,7 @@ ref: # (some Ensembl release versions are just broken) or (b) you know that a # newer version will include changes that will fix some error or adds # transcripts that will be relevant to your analysis. - release: "115" + release: 115 # genome build: # Usually, this should just be the main build listed in: # https://ftp.ensembl.org/pub/release-115/species_EnsemblVertebrates.txt From 22ee73ea2915de1da2c3d6f7e40b2687a046582a Mon Sep 17 00:00:00 2001 From: dlaehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:46:04 +0100 Subject: [PATCH 10/30] ci: fix gallus_gallus genome build to bGalGal1.mat.broiler.GRCg7b --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 4b593948..ca54ccea 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -35,7 +35,7 @@ ref: # cdna folder on the FTP server, and look for the correct build in the # file names there. For example "GRCh38" in: # https://ftp.ensembl.org/pub/release-115/fasta/homo_sapiens/cdna/ - build: GRCh38 + build: bGalGal1.mat.broiler.GRCg7b trimming: # If you activate trimming by setting this to `True`, this will trim adapters From cc8f46bb8d64b80946a4f90c4f2a93103a903009 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Tue, 25 Nov 2025 13:55:13 +0100 Subject: [PATCH 11/30] fix: the right config.yaml :facepalm: --- .test/config_sra/config.yaml | 2 +- config/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index 8054b222..db38d5af 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -8,7 +8,7 @@ units: config_sra/units.tsv ref: species: gallus_gallus release: 115 - build: GRCg7b + build: bGalGal1.mat.broiler.GRCg7b trimming: diff --git a/config/config.yaml b/config/config.yaml index ca54ccea..4b593948 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -35,7 +35,7 @@ ref: # cdna folder on the FTP server, and look for the correct build in the # file names there. For example "GRCh38" in: # https://ftp.ensembl.org/pub/release-115/fasta/homo_sapiens/cdna/ - build: bGalGal1.mat.broiler.GRCg7b + build: GRCh38 trimming: # If you activate trimming by setting this to `True`, this will trim adapters From 7faf652e57ba3cd02fcfda28eab548f801ebe188 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 27 Nov 2025 14:16:43 +0100 Subject: [PATCH 12/30] ci: use yeeast dataset for SRA download testing, as gallus_gallus genome too big for GitHub Actions based star indexing --- .test/config_sra/config.yaml | 19 +++++++++++-------- .test/config_sra/samples.tsv | 8 +++++--- .test/config_sra/units.tsv | 6 ++++-- 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index db38d5af..2ad9a0fc 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -2,13 +2,16 @@ # example file. samples: config_sra/samples.tsv +# This data comes from: +# * https://www.ncbi.nlm.nih.gov/bioproject/PRJNA1269045 +# * https://doi.org/10.7554/eLife.107157.3 units: config_sra/units.tsv ref: - species: gallus_gallus + species: saccharomyces_cerevisiae release: 115 - build: bGalGal1.mat.broiler.GRCg7b + build: R64-1-1 trimming: @@ -24,14 +27,14 @@ pca: diffexp: variables_of_interest: - condition: - base_level: PBS_0H + mutational_status: + base_level: wildtype batch_effects: "" contrasts: - treated-vs-untreated: - variable_of_interest: treatment - level_of_interest: H5N1_2.2_12H - model: ~treatment + apj1d_vs_wildtype: + variable_of_interest: mutational_status + level_of_interest: apj1d + model: ~mutational_status params: star: diff --git a/.test/config_sra/samples.tsv b/.test/config_sra/samples.tsv index 8780c693..78072b96 100644 --- a/.test/config_sra/samples.tsv +++ b/.test/config_sra/samples.tsv @@ -1,3 +1,5 @@ -sample_name treatment -H5N1_2.2_12H_brain_1 H5N1_2.2_12H -PBS_0H_brain_1 PBS_0H \ No newline at end of file +sample_name mutational_status +by4741_wt_1 wildtype +by4741_wt_2 wildtype +by4741_apj1d_1 apj1d +by4741_apj1d_2 apj1d \ No newline at end of file diff --git a/.test/config_sra/units.tsv b/.test/config_sra/units.tsv index 8f3be7e5..3c865081 100644 --- a/.test/config_sra/units.tsv +++ b/.test/config_sra/units.tsv @@ -1,3 +1,5 @@ sample_name unit_name fq1 fq2 sra fastp_adapters fastp_extra strandedness -H5N1_2.2_12H_brain_1 lane1 ERR5043451 -PBS_0H_brain_1 lane1 ERR5043769 \ No newline at end of file +by4741_wt_1 lane1 SRR33920892 +by4741_wt_2 lane1 SRR33920893 +by4741_apj1d_1 lane1 SRR33920890 +by4741_apj1d_2 lane1 SRR33920891 \ No newline at end of file From cc2bed35d7f2cc291f2df411dabd718403911e18 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 27 Nov 2025 15:12:25 +0100 Subject: [PATCH 13/30] ci: use some paired-end example data for sra download testing --- .test/config_sra/config.yaml | 16 ++++++++-------- .test/config_sra/samples.tsv | 10 +++++----- .test/config_sra/units.tsv | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index 2ad9a0fc..6860eb4a 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -3,8 +3,8 @@ samples: config_sra/samples.tsv # This data comes from: -# * https://www.ncbi.nlm.nih.gov/bioproject/PRJNA1269045 -# * https://doi.org/10.7554/eLife.107157.3 +# * https://www.ncbi.nlm.nih.gov/bioproject/PRJNA1359129 +# * https://www.ncbi.nlm.nih.gov/Traces/study/?acc=PRJNA1359129&o=acc_s%3Aa units: config_sra/units.tsv @@ -27,14 +27,14 @@ pca: diffexp: variables_of_interest: - mutational_status: - base_level: wildtype + medium: + base_level: glycerol batch_effects: "" contrasts: - apj1d_vs_wildtype: - variable_of_interest: mutational_status - level_of_interest: apj1d - model: ~mutational_status + trehalose_vs_glycerol: + variable_of_interest: medium + level_of_interest: trehalose + model: ~medium params: star: diff --git a/.test/config_sra/samples.tsv b/.test/config_sra/samples.tsv index 78072b96..a42b06f0 100644 --- a/.test/config_sra/samples.tsv +++ b/.test/config_sra/samples.tsv @@ -1,5 +1,5 @@ -sample_name mutational_status -by4741_wt_1 wildtype -by4741_wt_2 wildtype -by4741_apj1d_1 apj1d -by4741_apj1d_2 apj1d \ No newline at end of file +sample_name medium +trehalose_1 trehalose +trehalose_2 trehalose +glycerol_1 glycerol +glycerol_2 glycerol \ No newline at end of file diff --git a/.test/config_sra/units.tsv b/.test/config_sra/units.tsv index 3c865081..160fe0f3 100644 --- a/.test/config_sra/units.tsv +++ b/.test/config_sra/units.tsv @@ -1,5 +1,5 @@ sample_name unit_name fq1 fq2 sra fastp_adapters fastp_extra strandedness -by4741_wt_1 lane1 SRR33920892 -by4741_wt_2 lane1 SRR33920893 -by4741_apj1d_1 lane1 SRR33920890 -by4741_apj1d_2 lane1 SRR33920891 \ No newline at end of file +trehalose_1 lane1 SRR35987766 +trehalose_2 lane1 SRR35987767 +glycerol_1 lane1 SRR35987768 +glycerol_2 lane1 SRR35987769 \ No newline at end of file From 7278404c990d74aed8e6c42848dbf8e0b83bbdb5 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:03:44 +0100 Subject: [PATCH 14/30] ci: try freeing disk space in GitHub Actions runner --- .github/workflows/main.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b79a377e..3af48cbe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,10 +40,30 @@ jobs: - linting - formatting steps: + - name: update apt + run: sudo apt-get update + + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@v1.3.0 + with: + # this might remove tools that are actually needed, + # if set to "true" but frees about 6 GB + tool-cache: false + + # all of these default to true, but feel free to set to + # "false" if necessary for your workflow + android: true + dotnet: true + haskell: true + large-packages: true + docker-images: false + swap-storage: true + - name: Checkout repository with submodules uses: actions/checkout@v6 with: submodules: recursive + - name: Test workflow (basic model, no batch_effects) uses: snakemake/snakemake-github-action@v2 with: From 63319ca8636681c5761c411762a41907269a4b0a Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:06:53 +0100 Subject: [PATCH 15/30] ci: use same reference genome in all GitHub Actions tests --- .test/config_basic/config.yaml | 2 +- .test/config_complex/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.test/config_basic/config.yaml b/.test/config_basic/config.yaml index 4e784c40..de5e5d2f 100644 --- a/.test/config_basic/config.yaml +++ b/.test/config_basic/config.yaml @@ -7,7 +7,7 @@ units: config_basic/units.tsv ref: species: saccharomyces_cerevisiae - release: 100 + release: 115 build: R64-1-1 diff --git a/.test/config_complex/config.yaml b/.test/config_complex/config.yaml index e3afb8e4..bb95fa58 100644 --- a/.test/config_complex/config.yaml +++ b/.test/config_complex/config.yaml @@ -7,7 +7,7 @@ units: config_complex/units.tsv ref: species: saccharomyces_cerevisiae - release: 100 + release: 115 build: R64-1-1 From f3a6ee9067c9bc7098ddaf962b1fe2e10bfa7e5e Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:08:12 +0100 Subject: [PATCH 16/30] ci: always remove previous results after testing report generation --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3af48cbe..83f49249 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,6 +81,7 @@ jobs: with: directory: .test snakefile: workflow/Snakefile + stagein: 'rm -r .test/results/' args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (multiple variables_of_interest, include batch_effects) uses: snakemake/snakemake-github-action@v2 @@ -93,6 +94,7 @@ jobs: with: directory: .test snakefile: workflow/Snakefile + stagein: 'rm -r .test/results/' args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (sra file download, no batch_effects) uses: snakemake/snakemake-github-action@v2 From df8e1647c359b882cb60df9108bfdc9bcd1ed9bc Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Fri, 28 Nov 2025 13:57:45 +0100 Subject: [PATCH 17/30] ci: try getting fasterq-dump to use GitHub Actions runner's temp directory --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 83f49249..8b2df1f1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -94,7 +94,7 @@ jobs: with: directory: .test snakefile: workflow/Snakefile - stagein: 'rm -r .test/results/' + stagein: "rm -r .test/results/; export TMPDIR=${{ runner.temp }}" args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (sra file download, no batch_effects) uses: snakemake/snakemake-github-action@v2 From 48fb793e212fe3ddd5f56d99f5ffbddbe8e112fe Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Fri, 28 Nov 2025 14:40:00 +0100 Subject: [PATCH 18/30] ci: report fasterq-dump disk size limits --- workflow/rules/trim.smk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/workflow/rules/trim.smk b/workflow/rules/trim.smk index 96048e2a..e9380981 100644 --- a/workflow/rules/trim.smk +++ b/workflow/rules/trim.smk @@ -4,6 +4,8 @@ rule get_sra: "sra/{accession}_2.fastq", log: "logs/get-sra/{accession}.log", + params: + extra="-x", wrapper: "v7.2.0/bio/sra-tools/fasterq-dump" From 9a6995a65bce99f77b4dbe534f1b0409da5e8db8 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 1 Dec 2025 17:13:44 +0100 Subject: [PATCH 19/30] ci: try with yet another dataset, to reduce ci disk usage footprint --- .test/config_sra/config.yaml | 16 ++++++++-------- .test/config_sra/samples.tsv | 10 +++++----- .test/config_sra/units.tsv | 12 ++++++++---- 3 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index 6860eb4a..ede446de 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -3,8 +3,8 @@ samples: config_sra/samples.tsv # This data comes from: -# * https://www.ncbi.nlm.nih.gov/bioproject/PRJNA1359129 -# * https://www.ncbi.nlm.nih.gov/Traces/study/?acc=PRJNA1359129&o=acc_s%3Aa +# * https://www.ncbi.nlm.nih.gov/bioproject/PRJEB30262 +# * https://www.ncbi.nlm.nih.gov/Traces/study/?acc=PRJEB30262&o=acc_s%3Aa units: config_sra/units.tsv @@ -27,14 +27,14 @@ pca: diffexp: variables_of_interest: - medium: - base_level: glycerol + genotype: + base_level: control batch_effects: "" contrasts: - trehalose_vs_glycerol: - variable_of_interest: medium - level_of_interest: trehalose - model: ~medium + stb5_vs_control: + variable_of_interest: genotype + level_of_interest: stb5 + model: ~genotype params: star: diff --git a/.test/config_sra/samples.tsv b/.test/config_sra/samples.tsv index a42b06f0..d29b5e20 100644 --- a/.test/config_sra/samples.tsv +++ b/.test/config_sra/samples.tsv @@ -1,5 +1,5 @@ -sample_name medium -trehalose_1 trehalose -trehalose_2 trehalose -glycerol_1 glycerol -glycerol_2 glycerol \ No newline at end of file +sample_name genotype +control_g_1 control +control_g_2 control +stb5_g_1 stb5 +stb5_g_2 stb5 \ No newline at end of file diff --git a/.test/config_sra/units.tsv b/.test/config_sra/units.tsv index 160fe0f3..80d235a2 100644 --- a/.test/config_sra/units.tsv +++ b/.test/config_sra/units.tsv @@ -1,5 +1,9 @@ sample_name unit_name fq1 fq2 sra fastp_adapters fastp_extra strandedness -trehalose_1 lane1 SRR35987766 -trehalose_2 lane1 SRR35987767 -glycerol_1 lane1 SRR35987768 -glycerol_2 lane1 SRR35987769 \ No newline at end of file +control_g_1 lane1 ERR2985811 +control_g_1 lane2 ERR2985812 +control_g_2 lane1 ERR2985815 +control_g_2 lane2 ERR2985816 +stb5_g_1 lane1 ERR2985827 +stb5_g_1 lane2 ERR2985828 +stb5_g_2 lane1 ERR2985831 +stb5_g_2 lane2 ERR2985832 \ No newline at end of file From 1e163076e5f18cf67436e2fdc9a9b7be4f5efee2 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:33:14 +0100 Subject: [PATCH 20/30] chore: prettier .github/workflows/main.yml --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8b2df1f1..a9afb212 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: # this might remove tools that are actually needed, # if set to "true" but frees about 6 GB tool-cache: false - + # all of these default to true, but feel free to set to # "false" if necessary for your workflow android: true @@ -81,7 +81,7 @@ jobs: with: directory: .test snakefile: workflow/Snakefile - stagein: 'rm -r .test/results/' + stagein: "rm -r .test/results/" args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" - name: Test report (multiple variables_of_interest, include batch_effects) uses: snakemake/snakemake-github-action@v2 From 67a6fcea3056dfbc5f055fc3e95da8c926dc3ae2 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:34:20 +0100 Subject: [PATCH 21/30] chore: prettier .test/ config.yaml files --- .test/config_basic/config.yaml | 2 -- .test/config_complex/config.yaml | 3 --- .test/config_sra/config.yaml | 2 -- 3 files changed, 7 deletions(-) diff --git a/.test/config_basic/config.yaml b/.test/config_basic/config.yaml index de5e5d2f..7f3b4181 100644 --- a/.test/config_basic/config.yaml +++ b/.test/config_basic/config.yaml @@ -4,13 +4,11 @@ samples: config_basic/samples.tsv units: config_basic/units.tsv - ref: species: saccharomyces_cerevisiae release: 115 build: R64-1-1 - trimming: activate: True diff --git a/.test/config_complex/config.yaml b/.test/config_complex/config.yaml index bb95fa58..4f4f376b 100644 --- a/.test/config_complex/config.yaml +++ b/.test/config_complex/config.yaml @@ -4,13 +4,11 @@ samples: config_complex/samples.tsv units: config_complex/units.tsv - ref: species: saccharomyces_cerevisiae release: 115 build: R64-1-1 - trimming: activate: False @@ -44,4 +42,3 @@ params: star: index: "" align: "" - diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index ede446de..f29c7f72 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -7,13 +7,11 @@ samples: config_sra/samples.tsv # * https://www.ncbi.nlm.nih.gov/Traces/study/?acc=PRJEB30262&o=acc_s%3Aa units: config_sra/units.tsv - ref: species: saccharomyces_cerevisiae release: 115 build: R64-1-1 - trimming: activate: True From f2be7b0ec472235031ae7a122a0c25f109e35b48 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:34:51 +0100 Subject: [PATCH 22/30] ci: update github/super-linter to v8 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a9afb212..f28798f2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: Formatting - uses: github/super-linter@v5 + uses: github/super-linter@v8 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: master From 461b8a1948475ed695368f2445ae712f5ce8db96 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:40:53 +0100 Subject: [PATCH 23/30] ci: don't trust coderabbitai on versions of stuff, it just confidently makes up stuff --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f28798f2..d931364f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: Formatting - uses: github/super-linter@v8 + uses: github/super-linter@v7 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: master From ea04b91721b2e3b33fc0a792ce8cacc484acc34f Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:44:31 +0100 Subject: [PATCH 24/30] ci: switch to super-linter-super-linter action, which does have a version 8 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d931364f..e90897a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: submodules: recursive fetch-depth: 0 - name: Formatting - uses: github/super-linter@v7 + uses: super-linter/super-linter@v8 env: VALIDATE_ALL_CODEBASE: false DEFAULT_BRANCH: master From 0ae268f17b00c74769d21ee59611409db9204dfe Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:09:02 +0100 Subject: [PATCH 25/30] ci: consistently include `--software-deployment-method conda`, even in report generation, and increase cores to 8 (GitHub Action runners should currently have 4: https://docs.github.com/en/actions/reference/runners/github-hosted-runners#standard-github-hosted-runners-for-public-repositories - snakemake will restrict to what's available) --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e90897a6..951a6a33 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -69,36 +69,36 @@ jobs: with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + args: "--configfile .test/config_basic/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - name: Test report (basic model, no batch_effects) uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --report report.zip" + args: "--configfile .test/config_basic/config.yaml --software-deployment-method conda --report report.zip" - name: Test workflow (multiple variables_of_interest, include batch_effects) uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile stagein: "rm -r .test/results/" - args: "--configfile .test/config_complex/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + args: "--configfile .test/config_complex/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - name: Test report (multiple variables_of_interest, include batch_effects) uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml --report report.zip" + args: "--configfile .test/config_complex/config.yaml ---software-deployment-method conda -report report.zip" - name: Test workflow (sra file download, no batch_effects) uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile stagein: "rm -r .test/results/; export TMPDIR=${{ runner.temp }}" - args: "--configfile .test/config_sra/config.yaml --use-conda --show-failed-logs --cores 2 --conda-cleanup-pkgs cache" + args: "--configfile .test/config_sra/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - name: Test report (sra file download, no batch_effects) uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_sra/config.yaml --report report.zip" + args: "--configfile .test/config_sra/config.yaml --software-deployment-method conda --report report.zip" From 07854fb6fd10f8b3294309136e61b35a93305af8 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 16:57:20 +0100 Subject: [PATCH 26/30] ci: fix copy-pasta misplacement --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 951a6a33..00104422 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -88,7 +88,7 @@ jobs: with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml ---software-deployment-method conda -report report.zip" + args: "--configfile .test/config_complex/config.yaml --software-deployment-method conda --report report.zip" - name: Test workflow (sra file download, no batch_effects) uses: snakemake/snakemake-github-action@v2 with: From a100b78451513864fe693cf25fe12d260defc1d7 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:13:45 +0100 Subject: [PATCH 27/30] ci: switch to matrix setup for fail-fast and to try to circumenvent the mamba install failure --- .github/workflows/main.yml | 75 ++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 00104422..7f8c7846 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,16 +34,42 @@ jobs: snakefile: workflow/Snakefile args: "--configfile .test/config_complex/config.yaml --lint" - run-workflow: + testing: runs-on: ubuntu-latest needs: - linting - formatting + strategy: + fail-fast: true + matrix: + case: + - name: basic model, no batch_effects + args: >- + --configfile .test/config_basic/config.yaml + report: true + free_disk_space: false + + - name: multiple variables_of_interest, include batch_effects + uses: snakemake/snakemake-github-action@v2 + args: >- + --configfile .test/config_complex/config.yaml + report: true + free_disk_space: false + - name: sra file download, no batch_effects + uses: snakemake/snakemake-github-action@v2 + args: >- + --configfile .test/config_sra/config.yaml + report: true + free_disk_space: true + + name: test ${{ matrix.case.name }} + steps: - name: update apt run: sudo apt-get update - name: Free Disk Space (Ubuntu) + if: ${{ matrix.case.free_disk_space }} uses: jlumbroso/free-disk-space@v1.3.0 with: # this might remove tools that are actually needed, @@ -64,41 +90,28 @@ jobs: with: submodules: recursive - - name: Test workflow (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v2 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - - name: Test report (basic model, no batch_effects) - uses: snakemake/snakemake-github-action@v2 - with: - directory: .test - snakefile: workflow/Snakefile - args: "--configfile .test/config_basic/config.yaml --software-deployment-method conda --report report.zip" - - name: Test workflow (multiple variables_of_interest, include batch_effects) - uses: snakemake/snakemake-github-action@v2 - with: - directory: .test - snakefile: workflow/Snakefile - stagein: "rm -r .test/results/" - args: "--configfile .test/config_complex/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - - name: Test report (multiple variables_of_interest, include batch_effects) + - name: run workflow uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_complex/config.yaml --software-deployment-method conda --report report.zip" - - name: Test workflow (sra file download, no batch_effects) - uses: snakemake/snakemake-github-action@v2 - with: - directory: .test - snakefile: workflow/Snakefile - stagein: "rm -r .test/results/; export TMPDIR=${{ runner.temp }}" - args: "--configfile .test/config_sra/config.yaml --software-deployment-method conda --show-failed-logs --cores 8 --conda-cleanup-pkgs cache" - - name: Test report (sra file download, no batch_effects) + args: >- + --conda-cleanup-pkgs cache + --software-deployment-method conda + --show-failed-logs + --cores 8 + ${{ matrix.case.args }} + show-disk-usage-on-error: true + + - name: generate report + if: ${{ matrix.case.report }} uses: snakemake/snakemake-github-action@v2 with: directory: .test snakefile: workflow/Snakefile - args: "--configfile .test/config_sra/config.yaml --software-deployment-method conda --report report.zip" + args: >- + --software-deployment-method conda + --cores 1 + --report report.zip + ${{ matrix.case.args }} + show-disk-usage-on-error: true From 6e117715c1061b1438a9c79b3cec854d2f1b799c Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:17:17 +0100 Subject: [PATCH 28/30] ci: skip updating apt if free-disk-space job is not run --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f8c7846..13e965cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,6 +66,7 @@ jobs: steps: - name: update apt + if: ${{ matrix.case.free_disk_space }} run: sudo apt-get update - name: Free Disk Space (Ubuntu) From 37ffd249dfed28c7d1fb30c07464411334163971 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Thu, 4 Dec 2025 17:18:12 +0100 Subject: [PATCH 29/30] ci: cleanup leftovers from previous setup --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 13e965cc..f3d83378 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -50,13 +50,11 @@ jobs: free_disk_space: false - name: multiple variables_of_interest, include batch_effects - uses: snakemake/snakemake-github-action@v2 args: >- --configfile .test/config_complex/config.yaml report: true free_disk_space: false - name: sra file download, no batch_effects - uses: snakemake/snakemake-github-action@v2 args: >- --configfile .test/config_sra/config.yaml report: true From 273832e4bbb78b2f98a5f9765d015f9a8806adb5 Mon Sep 17 00:00:00 2001 From: David Laehnemann <1379875+dlaehnemann@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:16:57 +0100 Subject: [PATCH 30/30] fix: config_sra uses existing column for pca --- .test/config_sra/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.test/config_sra/config.yaml b/.test/config_sra/config.yaml index f29c7f72..22cc23f5 100644 --- a/.test/config_sra/config.yaml +++ b/.test/config_sra/config.yaml @@ -21,7 +21,7 @@ mergeReads: pca: activate: True labels: - - treatment + - genotype diffexp: variables_of_interest: