From 0f7ff12c15b816b5ccd5c842c2ad001ac2e1ab8c Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 29 Oct 2024 10:51:33 +0100 Subject: [PATCH 01/10] Add CI config for parallellized and economical nf-test runs, ripped from nf-test docs --- .github/workflows/nf-test.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/nf-test.yml diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml new file mode 100644 index 0000000..cf8def0 --- /dev/null +++ b/.github/workflows/nf-test.yml @@ -0,0 +1,34 @@ +name: nf-test + +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + shard: [1, 2, 3, 4] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + + - name: Setup Nextflow latest-edge + uses: nf-core/setup-nextflow@v1 + with: + version: "latest-edge" + + - name: Install nf-test + run: | + wget -qO- https://get.nf-test.com | bash + sudo mv nf-test /usr/local/bin/ + + - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) + run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} --changed-since HEAD^ From 389c50b615846b7ba5133ef93e27ce04009ba83d Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 29 Oct 2024 10:53:07 +0100 Subject: [PATCH 02/10] Add nf-test config to trigger full test on changes to critical files --- .nf-test.config | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .nf-test.config diff --git a/.nf-test.config b/.nf-test.config new file mode 100644 index 0000000..359a8b5 --- /dev/null +++ b/.nf-test.config @@ -0,0 +1,5 @@ +config { + ... + triggers 'nextflow.config', 'nf-test.config', 'test-data/**/*' + ... +} From 38b0e4564aba0f3f2817e26a0edf721bbbce38d5 Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 29 Oct 2024 10:57:02 +0100 Subject: [PATCH 03/10] prettier lint --- .github/workflows/nf-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index cf8def0..d7e8184 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -17,8 +17,8 @@ jobs: - name: Set up JDK 11 uses: actions/setup-java@v2 with: - java-version: '11' - distribution: 'adopt' + java-version: "11" + distribution: "adopt" - name: Setup Nextflow latest-edge uses: nf-core/setup-nextflow@v1 From b452763ce87231b509300193cb4911ab2ad2f9aa Mon Sep 17 00:00:00 2001 From: kedhammar Date: Tue, 29 Oct 2024 11:00:58 +0100 Subject: [PATCH 04/10] Bump changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b0b12de..c682d31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ Initial release of nf-core/seqinspector, created with the [nf-core](https://nf-c - [#20](https://github.com/nf-core/seqinspector/pull/20) Use tags to generate group reports - [#13](https://github.com/nf-core/seqinspector/pull/13) Generate reports per run, per project and per lane. - [#49](https://github.com/nf-core/seqinspector/pull/49) Merge with template 3.0.2. +- [#51](https://github.com/nf-core/seqinspector/pull/51) Add nf-test to CI. ### `Fixed` From 89ca390fb2a947d0cffe25215b4ec49125a6f347 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 10:34:45 +0000 Subject: [PATCH 05/10] Add modules testdata base path to test config and trim out resource limits --- tests/nextflow.config | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/tests/nextflow.config b/tests/nextflow.config index 8d9ef46..1e56f44 100644 --- a/tests/nextflow.config +++ b/tests/nextflow.config @@ -8,14 +8,8 @@ params { config_profile_name = 'nf-test profile' config_profile_description = 'Configuration profile to use for nf-test.' - // Limit resources so that this can run on GitHub Actions - max_cpus = 2 - max_memory = '3.GB' - max_time = '2.h' - pipelines_testdata_base_path = 'https://raw.githubusercontent.com/nf-core/test-datasets/' + modules_testdata_base_path = 's3://ngi-igenomes/testdata/nf-core/modules/' validationSchemaIgnoreParams = 'genomes,igenomes_base,pipelines_testdata_base_path' - - } From 34437348868aa0f4f7aa18424f35738903987a94 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 13:39:41 +0000 Subject: [PATCH 06/10] nf-core subworkflows update to apply fix --- modules.json | 4 ++-- .../tests/main.workflow.nf.test | 10 ++++++---- .../nf-core/utils_nfschema_plugin/tests/main.nf.test | 4 ++-- workflows/seqinspector.nf | 4 ++-- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/modules.json b/modules.json index 8e632d5..a27c750 100644 --- a/modules.json +++ b/modules.json @@ -21,7 +21,7 @@ "nf-core": { "utils_nextflow_pipeline": { "branch": "master", - "git_sha": "3aa0aec1d52d492fe241919f0c6100ebf0074082", + "git_sha": "56372688d8979092cafbe0c5c3895b491166ca1c", "installed_by": ["subworkflows"] }, "utils_nfcore_pipeline": { @@ -31,7 +31,7 @@ }, "utils_nfschema_plugin": { "branch": "master", - "git_sha": "bbd5a41f4535a8defafe6080e00ea74c45f4f96c", + "git_sha": "2fd2cd6d0e7b273747f32e465fdc6bcc3ae0814e", "installed_by": ["subworkflows"] } } diff --git a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test index ca964ce..02dbf09 100644 --- a/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test +++ b/subworkflows/nf-core/utils_nextflow_pipeline/tests/main.workflow.nf.test @@ -52,10 +52,12 @@ nextflow_workflow { } then { - assertAll( - { assert workflow.success }, - { assert workflow.stdout.contains("nextflow_workflow v9.9.9") } - ) + expect { + with(workflow) { + assert success + assert "nextflow_workflow v9.9.9" in stdout + } + } } } diff --git a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test index 842dc43..8fb3016 100644 --- a/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test +++ b/subworkflows/nf-core/utils_nfschema_plugin/tests/main.nf.test @@ -42,7 +42,7 @@ nextflow_workflow { params { test_data = '' - outdir = 1 + outdir = null } workflow { @@ -94,7 +94,7 @@ nextflow_workflow { params { test_data = '' - outdir = 1 + outdir = null } workflow { diff --git a/workflows/seqinspector.nf b/workflows/seqinspector.nf index ea62811..d1fb289 100644 --- a/workflows/seqinspector.nf +++ b/workflows/seqinspector.nf @@ -138,9 +138,9 @@ workflow SEQINSPECTOR { ) emit: - global_report = MULTIQC_GLOBAL.out.report.toList() // channel: /path/to/multiqc_report.html + global_report = MULTIQC_GLOBAL.out.report.toList() // channel: /path/to/multiqc_report.html grouped_reports = MULTIQC_PER_TAG.out.report.toList() // channel: [ /path/to/multiqc_report.html ] - versions = ch_versions // channel: [ path(versions.yml) ] + versions = ch_versions // channel: [ path(versions.yml) ] } /* From ee42aa9b0a5de78419451666b571e8d3575c2498 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 13:43:56 +0000 Subject: [PATCH 07/10] simplify CI, always run --- .github/workflows/nf-test.yml | 4 +--- .nf-test.config | 5 ----- 2 files changed, 1 insertion(+), 8 deletions(-) delete mode 100644 .nf-test.config diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index d7e8184..b6276bf 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -11,8 +11,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - with: - fetch-depth: 0 - name: Set up JDK 11 uses: actions/setup-java@v2 @@ -31,4 +29,4 @@ jobs: sudo mv nf-test /usr/local/bin/ - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) - run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} --changed-since HEAD^ + run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} diff --git a/.nf-test.config b/.nf-test.config deleted file mode 100644 index 359a8b5..0000000 --- a/.nf-test.config +++ /dev/null @@ -1,5 +0,0 @@ -config { - ... - triggers 'nextflow.config', 'nf-test.config', 'test-data/**/*' - ... -} From 47053765d465c663e4e90c7d2dff8f745a54cd28 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 13:48:10 +0000 Subject: [PATCH 08/10] try specifying cwd as arg in nf-test command --- .github/workflows/nf-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nf-test.yml b/.github/workflows/nf-test.yml index b6276bf..b0d7b8e 100644 --- a/.github/workflows/nf-test.yml +++ b/.github/workflows/nf-test.yml @@ -29,4 +29,4 @@ jobs: sudo mv nf-test /usr/local/bin/ - name: Run Tests (Shard ${{ matrix.shard }}/${{ strategy.job-total }}) - run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} + run: nf-test test --ci --shard ${{ matrix.shard }}/${{ strategy.job-total }} . From 222a986914543fde7cdedf891c20bfe85320f787 Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 14:34:32 +0000 Subject: [PATCH 09/10] harmonize syntax and correct comment --- workflows/seqinspector.nf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflows/seqinspector.nf b/workflows/seqinspector.nf index d1fb289..ab6cda4 100644 --- a/workflows/seqinspector.nf +++ b/workflows/seqinspector.nf @@ -87,10 +87,10 @@ workflow SEQINSPECTOR { .mix(ch_multiqc_extra_files) .collect(), ch_multiqc_config.toList(), - Channel.empty().toList(), + [], ch_multiqc_logo.toList(), - Channel.empty().toList(), - Channel.empty().toList() + [], + [] ) ch_tags = ch_multiqc_files @@ -138,7 +138,7 @@ workflow SEQINSPECTOR { ) emit: - global_report = MULTIQC_GLOBAL.out.report.toList() // channel: /path/to/multiqc_report.html + global_report = MULTIQC_GLOBAL.out.report.toList() // channel: [ /path/to/multiqc_report.html ] grouped_reports = MULTIQC_PER_TAG.out.report.toList() // channel: [ /path/to/multiqc_report.html ] versions = ch_versions // channel: [ path(versions.yml) ] } From 69825df5aa0763953398bcca78fe9ba86699ff3f Mon Sep 17 00:00:00 2001 From: Alfred Kedhammar Date: Tue, 29 Oct 2024 14:42:03 +0000 Subject: [PATCH 10/10] remove multiqc versions file from snapshot assertion, will fail if tests run on different nextflow versions --- tests/MiSeq.main.nf.test | 1 - tests/MiSeq.main.nf.test.snap | 5 ++--- tests/NovaSeq6000.main.nf.test | 5 ----- tests/NovaSeq6000.main.nf.test.snap | 9 ++------- tests/PromethION.main.nf.test | 1 - tests/PromethION.main.nf.test.snap | 5 ++--- 6 files changed, 6 insertions(+), 20 deletions(-) diff --git a/tests/MiSeq.main.nf.test b/tests/MiSeq.main.nf.test index 8fbff4a..246a05a 100644 --- a/tests/MiSeq.main.nf.test +++ b/tests/MiSeq.main.nf.test @@ -21,7 +21,6 @@ nextflow_pipeline { path("$outputDir/multiqc/global_report/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/global_report/multiqc_data/multiqc_software_versions.txt"), ).match() } ) diff --git a/tests/MiSeq.main.nf.test.snap b/tests/MiSeq.main.nf.test.snap index 4613d52..0742b5a 100644 --- a/tests/MiSeq.main.nf.test.snap +++ b/tests/MiSeq.main.nf.test.snap @@ -3,8 +3,7 @@ "content": [ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,7b1b7fd457b60404768045b148d4c0a8", - "multiqc_general_stats.txt:md5,5b28a83b14cb2fe88d084d08900ebdbf", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3" + "multiqc_general_stats.txt:md5,5b28a83b14cb2fe88d084d08900ebdbf" ], "meta": { "nf-test": "0.9.0", @@ -12,4 +11,4 @@ }, "timestamp": "2024-10-28T13:18:10.3675973" } -} \ No newline at end of file +} diff --git a/tests/NovaSeq6000.main.nf.test b/tests/NovaSeq6000.main.nf.test index d050399..e24637c 100644 --- a/tests/NovaSeq6000.main.nf.test +++ b/tests/NovaSeq6000.main.nf.test @@ -21,27 +21,22 @@ nextflow_pipeline { path("$outputDir/multiqc/global_report/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/global_report/multiqc_data/multiqc_software_versions.txt"), path("$outputDir/multiqc/group_reports/lane1/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/group_reports/lane1/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/group_reports/lane1/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/group_reports/lane1/multiqc_data/multiqc_software_versions.txt"), path("$outputDir/multiqc/group_reports/group1/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/group_reports/group1/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/group_reports/group1/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/group_reports/group1/multiqc_data/multiqc_software_versions.txt"), path("$outputDir/multiqc/group_reports/group2/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/group_reports/group2/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/group_reports/group2/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/group_reports/group2/multiqc_data/multiqc_software_versions.txt"), path("$outputDir/multiqc/group_reports/test/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/group_reports/test/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/group_reports/test/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/group_reports/test/multiqc_data/multiqc_software_versions.txt"), ).match() }, ) diff --git a/tests/NovaSeq6000.main.nf.test.snap b/tests/NovaSeq6000.main.nf.test.snap index ee3c22b..fc38ac5 100644 --- a/tests/NovaSeq6000.main.nf.test.snap +++ b/tests/NovaSeq6000.main.nf.test.snap @@ -4,23 +4,18 @@ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,3730f9046b20ac5c17a86db0a33f8d5d", "multiqc_general_stats.txt:md5,25abe0f6a35eb4a3b056fc3cf5c13732", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,8284e25ccc21041cf3b5a32eb6a51e78", "multiqc_general_stats.txt:md5,90ee35137492b80aab36ef67f72d8921", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,f38ffdc112c73af3a41ed15848a3761f", "multiqc_general_stats.txt:md5,d62a2fc39e674d98783d408791803148", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,7ff71ceb8ecdf086331047f8860c3347", "multiqc_general_stats.txt:md5,2f09b8f199ac40cf67ba50843cebd29c", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3", "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,519ff344a896ac369bba4d5c5b8be7b5", - "multiqc_general_stats.txt:md5,6a1c16f068d7ba3a9225a17eb570ed9a", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3" + "multiqc_general_stats.txt:md5,6a1c16f068d7ba3a9225a17eb570ed9a" ], "meta": { "nf-test": "0.9.0", @@ -28,4 +23,4 @@ }, "timestamp": "2024-10-28T13:19:13.226135825" } -} \ No newline at end of file +} diff --git a/tests/PromethION.main.nf.test b/tests/PromethION.main.nf.test index 8fec8b3..ad6fe6d 100644 --- a/tests/PromethION.main.nf.test +++ b/tests/PromethION.main.nf.test @@ -21,7 +21,6 @@ nextflow_pipeline { path("$outputDir/multiqc/global_report/multiqc_data/multiqc_citations.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_fastqc.txt"), path("$outputDir/multiqc/global_report/multiqc_data/multiqc_general_stats.txt"), - path("$outputDir/multiqc/global_report/multiqc_data/multiqc_software_versions.txt"), ).match() }, ) diff --git a/tests/PromethION.main.nf.test.snap b/tests/PromethION.main.nf.test.snap index 026a8cd..875673c 100644 --- a/tests/PromethION.main.nf.test.snap +++ b/tests/PromethION.main.nf.test.snap @@ -3,8 +3,7 @@ "content": [ "multiqc_citations.txt:md5,4c806e63a283ec1b7e78cdae3a923d4f", "multiqc_fastqc.txt:md5,1a4b472e13cadc770832b0e20d1de7b0", - "multiqc_general_stats.txt:md5,409cefc7f17f95d176ced6032bf8fb32", - "multiqc_software_versions.txt:md5,a3698a2d32e8695c38d50e3d17de5fe3" + "multiqc_general_stats.txt:md5,409cefc7f17f95d176ced6032bf8fb32" ], "meta": { "nf-test": "0.9.0", @@ -12,4 +11,4 @@ }, "timestamp": "2024-10-28T13:19:57.261730412" } -} \ No newline at end of file +}