From 6bf6638cc7c027e6569fd97eb55e20d91c8c1689 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 4 Jun 2026 12:16:40 +0200 Subject: [PATCH 01/34] Add snpclustering subworkflow --- subworkflows/nf-core/snpclustering/main.nf | 156 +++++++++++++++ subworkflows/nf-core/snpclustering/meta.yml | 180 +++++++++++++++++ .../nf-core/snpclustering/tests/main.nf.test | 162 +++++++++++++++ .../snpclustering/tests/main.nf.test.snap | 186 ++++++++++++++++++ .../snpclustering/tests/nextflow.config | 44 +++++ 5 files changed, 728 insertions(+) create mode 100644 subworkflows/nf-core/snpclustering/main.nf create mode 100644 subworkflows/nf-core/snpclustering/meta.yml create mode 100644 subworkflows/nf-core/snpclustering/tests/main.nf.test create mode 100644 subworkflows/nf-core/snpclustering/tests/main.nf.test.snap create mode 100644 subworkflows/nf-core/snpclustering/tests/nextflow.config diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf new file mode 100644 index 000000000000..221553311025 --- /dev/null +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -0,0 +1,156 @@ +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle/main' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' +include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' + +workflow SNPCLUSTERING { + + take: + vcf_ch + refpanel_ch + genmap_ch + region + npcs + use_approx + algorithm + n_clusters + dbscan_eps + dbscan_min_samples + + main: + ch_versions = Channel.empty() + + /* + * Build BEAGLE input tuple: + * tuple val(meta), path(vcf), path(vcf_index), path(refpanel), path(refpanel_index), + * path(genmap), path(exclsamples), path(exclmarkers), val(region) + * + * For now, refpanel/genmap/exclusion files are optional and passed as empty lists + * when not provided by the caller. + */ + ch_beagle_input = vcf_ch.map { meta, vcf, vcf_index -> + tuple( + meta, + vcf, + vcf_index, + [], + [], + [], + [], + [], + region + ) + } + + BEAGLE5_BEAGLE(ch_beagle_input) + ch_versions = ch_versions.mix(BEAGLE5_BEAGLE.out.versions_beagle) + + /* + * Convert imputed VCF to PLINK2 pfiles + */ + PLINK2_VCF(BEAGLE5_BEAGLE.out.vcf) + ch_versions = ch_versions.mix(PLINK2_VCF.out.versions) + + /* + * PLINK2_PCA expects: + * tuple val(meta), val(npcs), val(use_approx), path(pgen), path(psam), path(pvar) + */ + ch_plink_pca_input = PLINK2_VCF.out.pgen + .join(PLINK2_VCF.out.pvar) + .join(PLINK2_VCF.out.psam) + .map { meta, pgen, pvar, psam -> + tuple(meta, npcs, use_approx, pgen, psam, pvar) + } + + PLINK2_PCA(ch_plink_pca_input) + ch_versions = ch_versions.mix(PLINK2_PCA.out.versions) + + /* + * Convert .eigenvec to TSV for downstream clustering modules + */ + EIGENVEC_TO_TSV(PLINK2_PCA.out.evecfile) + + /* + * PCA clustering + * Signature inferred from your successful wiring so far: + * CUSTOM_PCACLUSTERING(tsv, algorithm, n_clusters, dbscan_eps, dbscan_min_samples) + */ + CUSTOM_PCACLUSTERING( + EIGENVEC_TO_TSV.out.tsv, + algorithm, + n_clusters, + dbscan_eps, + dbscan_min_samples + ) + ch_versions = ch_versions.mix(CUSTOM_PCACLUSTERING.out.versions) + + /* + * Metrics and visualization both expect one tuple input channel + * built from: meta + tsv + cluster assignments + */ + ch_cluster_analysis_input = EIGENVEC_TO_TSV.out.tsv + .join(CUSTOM_PCACLUSTERING.out.clusters) + .map { meta, tsv, clusters -> + tuple(meta, tsv, clusters) + } + + CUSTOM_CLUSTERMETRICS(ch_cluster_analysis_input) + ch_versions = ch_versions.mix(CUSTOM_CLUSTERMETRICS.out.versions) + + CUSTOM_CLUSTERVISUALIZATION(ch_cluster_analysis_input) + ch_versions = ch_versions.mix(CUSTOM_CLUSTERVISUALIZATION.out.versions) + + emit: + imputed_vcf = BEAGLE5_BEAGLE.out.vcf + beagle_log = BEAGLE5_BEAGLE.out.log + + pgen = PLINK2_VCF.out.pgen + pvar = PLINK2_VCF.out.pvar + psam = PLINK2_VCF.out.psam + + evecfile = PLINK2_PCA.out.evecfile + evfile = PLINK2_PCA.out.evfile + pca_log = PLINK2_PCA.out.logfile + + tsv = EIGENVEC_TO_TSV.out.tsv + + clusters = CUSTOM_PCACLUSTERING.out.clusters + cluster_info = CUSTOM_PCACLUSTERING.out.info + + metrics = CUSTOM_CLUSTERMETRICS.out.metrics + k_sweep = CUSTOM_CLUSTERMETRICS.out.k_sweep + selected = CUSTOM_CLUSTERMETRICS.out.selected + metric_plots = CUSTOM_CLUSTERMETRICS.out.plots + + umap_tsv = CUSTOM_CLUSTERVISUALIZATION.out.umap_tsv + tsne_tsv = CUSTOM_CLUSTERVISUALIZATION.out.tsne_tsv + umap_png = CUSTOM_CLUSTERVISUALIZATION.out.umap_png + tsne_png = CUSTOM_CLUSTERVISUALIZATION.out.tsne_png + + versions = ch_versions +} + +process EIGENVEC_TO_TSV { + tag "${meta.id}" + label 'process_single' + + input: + tuple val(meta), path(eigenvec) + + output: + tuple val(meta), path("${meta.id}.tsv"), emit: tsv + path "versions.yml", emit: versions + + script: + """ + awk 'NR==1 { sub(/^#/, ""); \$1 = ""; sub(/^\\t/, ""); sub(/^IID\\t/, "sample_id\\t"); print; next } + { \$1 = ""; sub(/^\\t/, ""); print }' OFS='\\t' ${eigenvec} > ${meta.id}.tsv + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + gawk: \$(awk --version | head -1 | sed 's/GNU Awk //;s/,.*//') + END_VERSIONS + """ +} diff --git a/subworkflows/nf-core/snpclustering/meta.yml b/subworkflows/nf-core/snpclustering/meta.yml new file mode 100644 index 000000000000..cdf6d7c6e00b --- /dev/null +++ b/subworkflows/nf-core/snpclustering/meta.yml @@ -0,0 +1,180 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/subworkflows/yaml-schema.json +name: "snpclustering" +description: > + Subworkflow for SNP clustering. Takes genotyped VCF files, imputes missing + genotypes with BEAGLE5, converts to PLINK2 pgen format, performs PCA, + converts eigenvec to TSV with awk, then runs PCA-based clustering, + cluster metrics, and cluster visualization. +keywords: + - snp + - clustering + - pca + - plink2 + - beagle + - imputation + - population genetics + +components: + - beagle5/beagle + - plink2/vcf + - plink2/pca + - custom/pcaclustering + - custom/clustermetrics + - custom/clustervisualization + +input: + - - meta: + type: map + description: Groovy map containing sample metadata. + - vcf: + type: file + description: Genotyped VCF file. + pattern: "*.{vcf,vcf.gz}" + - vcf_index: + type: file + description: Index for the VCF file. + pattern: "*.{tbi,csi}" + + - - meta: + type: map + description: Groovy map for the optional reference panel. + - refpanel: + type: file + description: Optional BEAGLE reference panel VCF. + pattern: "*.{vcf,vcf.gz}" + - refpanel_index: + type: file + description: Optional index for the reference panel. + pattern: "*.{tbi,csi}" + + - - meta: + type: map + description: Groovy map for the optional genetic map. + - genmap: + type: file + description: Optional genetic map file for BEAGLE. + pattern: "*.map" + + - - region: + type: val + description: Optional genomic region passed to BEAGLE. + + - - npcs: + type: val + description: Number of principal components to compute. + + - - use_approx: + type: val + description: Whether to use approximate PCA. + + - - algorithm: + type: val + description: Clustering algorithm to use. + + - - n_clusters: + type: val + description: Number of clusters for kmeans or similar methods. + + - - dbscan_eps: + type: val + description: DBSCAN eps parameter. + + - - dbscan_min_samples: + type: val + description: DBSCAN minimum samples parameter. + +output: + - imputed_vcf: + - meta: + type: map + description: Groovy map of sample metadata. + - vcf: + type: file + description: Imputed VCF from BEAGLE5. + pattern: "*.{vcf,vcf.gz}" + + - pfiles: + - meta: + type: map + description: Groovy map of sample metadata. + - pgen: + type: file + description: PLINK2 pgen file. + pattern: "*.pgen" + - psam: + type: file + description: PLINK2 psam file. + pattern: "*.psam" + - pvar: + type: file + description: PLINK2 pvar file. + pattern: "*.pvar" + + - pca: + - meta: + type: map + description: Groovy map of sample metadata. + - eigenvec: + type: file + description: PCA eigenvectors from PLINK2. + pattern: "*.eigenvec" + - eigenval: + type: file + description: PCA eigenvalues from PLINK2. + pattern: "*.eigenval" + + - tsv: + - meta: + type: map + description: Groovy map of sample metadata. + - tsv: + type: file + description: Tidy TSV converted from eigenvec by awk. + pattern: "*.tsv" + + - cluster_labels: + - meta: + type: map + description: Groovy map of sample metadata. + - clusters: + type: file + description: Cluster labels output from PCA clustering. + pattern: "*.tsv" + + - cluster_info: + - meta: + type: map + description: Groovy map of sample metadata. + - info: + type: file + description: Additional cluster info output. + pattern: "*.tsv" + + - metrics: + - meta: + type: map + description: Groovy map of sample metadata. + - metrics: + type: file + description: Cluster quality metrics output. + pattern: "*.tsv" + + - plots: + - meta: + type: map + description: Groovy map of sample metadata. + - plots: + type: file + description: Cluster visualization outputs. + pattern: "*.{png,pdf,html}" + + - versions: + - versions: + type: file + description: Software versions collected from the subworkflow. + pattern: "versions.yml" + +authors: + - "@dbaku42" +maintainers: + - "@dbaku42" diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test new file mode 100644 index 000000000000..8a2af307932e --- /dev/null +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -0,0 +1,162 @@ +nextflow_workflow { + + name "Test Subworkflow SNPCLUSTERING" + script "../main.nf" + workflow "SNPCLUSTERING" + + tag "subworkflows" + tag "subworkflows_nfcore" + tag "subworkflows/snpclustering" + tag "snpclustering" + tag "plink2" + tag "plink2/vcf" + tag "plink2/pca" + tag "beagle5/beagle" + tag "custom/pcaclustering" + tag "custom/clustermetrics" + tag "custom/clustervisualization" + + config "./nextflow.config" + + test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { + when { + workflow { + """ + input[0] = Channel.of([ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz.csi', checkIfExists: true) + ]) + input[1] = Channel.empty() + input[2] = Channel.empty() + input[3] = "" + input[4] = params.npcs + input[5] = params.use_approx + input[6] = "kmeans" + input[7] = 3 + input[8] = params.dbscan_eps + input[9] = params.dbscan_min_samples + """ + } + } + + then { + assert workflow.success + assert snapshot( + workflow.out.clusters.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.cluster_info.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.metrics.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.versions + ).match() + } + } + + test("homo_sapiens - 1000GP.chr22.vcf.gz - dbscan") { + when { + workflow { + """ + input[0] = Channel.of([ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz.csi', checkIfExists: true) + ]) + input[1] = Channel.empty() + input[2] = Channel.empty() + input[3] = "" + input[4] = params.npcs + input[5] = params.use_approx + input[6] = "dbscan" + input[7] = params.n_clusters + input[8] = params.dbscan_eps + input[9] = params.dbscan_min_samples + """ + } + } + + then { + assert workflow.success + assert snapshot( + workflow.out.clusters.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.cluster_info.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.metrics.collect { [ + it[0], + path(it[1]).getFileName().toString() + ]}, + workflow.out.versions + ).match() + } + } + + test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans - stub") { + options "-stub" + when { + workflow { + """ + input[0] = Channel.of([ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz.csi', checkIfExists: true) + ]) + input[1] = Channel.empty() + input[2] = Channel.empty() + input[3] = "" + input[4] = params.npcs + input[5] = params.use_approx + input[6] = "kmeans" + input[7] = 3 + input[8] = params.dbscan_eps + input[9] = params.dbscan_min_samples + """ + } + } + + then { + assert workflow.success + assert snapshot(sanitizeOutput(workflow.out)).match() + } + } + + test("homo_sapiens - 1000GP.chr22.vcf.gz - dbscan - stub") { + options "-stub" + when { + workflow { + """ + input[0] = Channel.of([ + [ id:'test' ], + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz', checkIfExists: true), + file(params.modules_testdata_base_path + 'genomics/homo_sapiens/popgen/1000GP.chr22.vcf.gz.csi', checkIfExists: true) + ]) + input[1] = Channel.empty() + input[2] = Channel.empty() + input[3] = "" + input[4] = params.npcs + input[5] = params.use_approx + input[6] = "dbscan" + input[7] = params.n_clusters + input[8] = params.dbscan_eps + input[9] = params.dbscan_min_samples + """ + } + } + + then { + assert workflow.success + assert snapshot(sanitizeOutput(workflow.out)).match() + } + } +} diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap new file mode 100644 index 000000000000..f4d7988e57d5 --- /dev/null +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -0,0 +1,186 @@ +{ + "homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + + ] + ], + "timestamp": "2026-06-04T01:16:07.326033555", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + }, + "homo_sapiens - 1000GP.chr22.vcf.gz - dbscan": { + "content": [ + [ + + ], + [ + + ], + [ + + ], + [ + + ] + ], + "timestamp": "2026-06-04T01:16:13.695098223", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + }, + "homo_sapiens - 1000GP.chr22.vcf.gz - dbscan - stub": { + "content": [ + { + "beagle_log": [ + + ], + "cluster_info": [ + + ], + "clusters": [ + + ], + "evecfile": [ + + ], + "evfile": [ + + ], + "imputed_vcf": [ + + ], + "k_sweep": [ + + ], + "metric_plots": [ + + ], + "metrics": [ + + ], + "pca_log": [ + + ], + "pgen": [ + + ], + "psam": [ + + ], + "pvar": [ + + ], + "selected": [ + + ], + "tsne_png": [ + + ], + "tsne_tsv": [ + + ], + "tsv": [ + + ], + "umap_png": [ + + ], + "umap_tsv": [ + + ], + "versions": [ + + ] + } + ], + "timestamp": "2026-06-04T01:16:30.853529288", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + }, + "homo_sapiens - 1000GP.chr22.vcf.gz - kmeans - stub": { + "content": [ + { + "beagle_log": [ + + ], + "cluster_info": [ + + ], + "clusters": [ + + ], + "evecfile": [ + + ], + "evfile": [ + + ], + "imputed_vcf": [ + + ], + "k_sweep": [ + + ], + "metric_plots": [ + + ], + "metrics": [ + + ], + "pca_log": [ + + ], + "pgen": [ + + ], + "psam": [ + + ], + "pvar": [ + + ], + "selected": [ + + ], + "tsne_png": [ + + ], + "tsne_tsv": [ + + ], + "tsv": [ + + ], + "umap_png": [ + + ], + "umap_tsv": [ + + ], + "versions": [ + + ] + } + ], + "timestamp": "2026-06-04T01:16:16.700560685", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config new file mode 100644 index 000000000000..99eba676d04a --- /dev/null +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -0,0 +1,44 @@ +params { + npcs = 8 + use_approx = true + algorithm = "kmeans" + n_clusters = 3 + dbscan_eps = 0.5 + dbscan_min_samples = 5 +} + +process { + withName: 'BEAGLE5_BEAGLE' { + ext.prefix = { "${meta.id}" } + ext.when = false + } + + withName: 'PLINK2_VCF' { + ext.prefix = { "${meta.id}" } + ext.when = false + } + + withName: 'PLINK2_PCA' { + ext.prefix = { "${meta.id}" } + ext.when = false + } + + withName: 'EIGENVEC_TO_TSV' { + ext.prefix = { "${meta.id}" } + } + + withName: 'CUSTOM_PCACLUSTERING' { + ext.prefix = { "${meta.id}" } + ext.when = false + } + + withName: 'CUSTOM_CLUSTERMETRICS' { + ext.prefix = { "${meta.id}" } + ext.when = false + } + + withName: 'CUSTOM_CLUSTERVISUALIZATION' { + ext.prefix = { "${meta.id}" } + ext.when = false + } +} From 87af5c5a1d43051bb0fd9dcd0c0b6e614853635b Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 4 Jun 2026 13:05:55 +0200 Subject: [PATCH 02/34] fix meta.yml --- subworkflows/nf-core/snpclustering/meta.yml | 265 ++++++++++---------- 1 file changed, 128 insertions(+), 137 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/meta.yml b/subworkflows/nf-core/snpclustering/meta.yml index cdf6d7c6e00b..25098093fb72 100644 --- a/subworkflows/nf-core/snpclustering/meta.yml +++ b/subworkflows/nf-core/snpclustering/meta.yml @@ -23,156 +23,147 @@ components: - custom/clustervisualization input: - - - meta: - type: map - description: Groovy map containing sample metadata. - - vcf: - type: file - description: Genotyped VCF file. - pattern: "*.{vcf,vcf.gz}" - - vcf_index: - type: file - description: Index for the VCF file. - pattern: "*.{tbi,csi}" - - - - meta: - type: map - description: Groovy map for the optional reference panel. - - refpanel: - type: file - description: Optional BEAGLE reference panel VCF. - pattern: "*.{vcf,vcf.gz}" - - refpanel_index: - type: file - description: Optional index for the reference panel. - pattern: "*.{tbi,csi}" - - - - meta: - type: map - description: Groovy map for the optional genetic map. - - genmap: - type: file - description: Optional genetic map file for BEAGLE. - pattern: "*.map" - - - - region: - type: val - description: Optional genomic region passed to BEAGLE. - - - - npcs: - type: val - description: Number of principal components to compute. - - - - use_approx: - type: val - description: Whether to use approximate PCA. - - - - algorithm: - type: val - description: Clustering algorithm to use. - - - - n_clusters: - type: val - description: Number of clusters for kmeans or similar methods. - - - - dbscan_eps: - type: val - description: DBSCAN eps parameter. - - - - dbscan_min_samples: - type: val - description: DBSCAN minimum samples parameter. + - meta: + type: map + description: Groovy map containing sample metadata. + vcf: + type: file + description: Genotyped VCF file. + pattern: "*.{vcf,vcf.gz}" + vcf_index: + type: file + description: Index for the VCF file. + pattern: "*.{tbi,csi}" + + - meta: + type: map + description: Groovy map for the optional reference panel. + refpanel: + type: file + description: Optional BEAGLE reference panel VCF. + pattern: "*.{vcf,vcf.gz}" + refpanel_index: + type: file + description: Optional index for the reference panel. + pattern: "*.{tbi,csi}" + + - meta: + type: map + description: Groovy map for the optional genetic map. + genmap: + type: file + description: Optional genetic map file for BEAGLE. + pattern: "*.map" + + - region: + type: val + description: Optional genomic region passed to BEAGLE. + + - npcs: + type: val + description: Number of principal components to compute. + + - use_approx: + type: val + description: Whether to use approximate PCA. + + - algorithm: + type: val + description: Clustering algorithm to use. + + - n_clusters: + type: val + description: Number of clusters for kmeans or similar methods. + + - dbscan_eps: + type: val + description: DBSCAN eps parameter. + + - dbscan_min_samples: + type: val + description: DBSCAN minimum samples parameter. output: - imputed_vcf: - - meta: - type: map - description: Groovy map of sample metadata. - - vcf: - type: file - description: Imputed VCF from BEAGLE5. - pattern: "*.{vcf,vcf.gz}" + type: map + description: Groovy map of sample metadata. + vcf: + type: file + description: Imputed VCF from BEAGLE5. + pattern: "*.{vcf,vcf.gz}" - pfiles: - - meta: - type: map - description: Groovy map of sample metadata. - - pgen: - type: file - description: PLINK2 pgen file. - pattern: "*.pgen" - - psam: - type: file - description: PLINK2 psam file. - pattern: "*.psam" - - pvar: - type: file - description: PLINK2 pvar file. - pattern: "*.pvar" + type: map + description: Groovy map of sample metadata. + pgen: + type: file + description: PLINK2 pgen file. + pattern: "*.pgen" + psam: + type: file + description: PLINK2 psam file. + pattern: "*.psam" + pvar: + type: file + description: PLINK2 pvar file. + pattern: "*.pvar" - pca: - - meta: - type: map - description: Groovy map of sample metadata. - - eigenvec: - type: file - description: PCA eigenvectors from PLINK2. - pattern: "*.eigenvec" - - eigenval: - type: file - description: PCA eigenvalues from PLINK2. - pattern: "*.eigenval" - - - tsv: - - meta: - type: map - description: Groovy map of sample metadata. - - tsv: - type: file - description: Tidy TSV converted from eigenvec by awk. - pattern: "*.tsv" + type: map + description: Groovy map of sample metadata. + eigenvec: + type: file + description: PCA eigenvectors from PLINK2. + pattern: "*.eigenvec" + eigenval: + type: file + description: PCA eigenvalues from PLINK2. + pattern: "*.eigenval" + + - eigenvec_tsv: + type: map + description: Groovy map of sample metadata. + tsv: + type: file + description: Tidy TSV converted from eigenvec by awk. + pattern: "*.tsv" - cluster_labels: - - meta: - type: map - description: Groovy map of sample metadata. - - clusters: - type: file - description: Cluster labels output from PCA clustering. - pattern: "*.tsv" + type: map + description: Groovy map of sample metadata. + clusters: + type: file + description: Cluster labels output from PCA clustering. + pattern: "*.tsv" - cluster_info: - - meta: - type: map - description: Groovy map of sample metadata. - - info: - type: file - description: Additional cluster info output. - pattern: "*.tsv" - - - metrics: - - meta: - type: map - description: Groovy map of sample metadata. - - metrics: - type: file - description: Cluster quality metrics output. - pattern: "*.tsv" - - - plots: - - meta: - type: map - description: Groovy map of sample metadata. - - plots: - type: file - description: Cluster visualization outputs. - pattern: "*.{png,pdf,html}" + type: map + description: Groovy map of sample metadata. + info: + type: file + description: Additional cluster info output. + pattern: "*.tsv" + + - cluster_metrics: + type: map + description: Groovy map of sample metadata. + metrics: + type: file + description: Cluster quality metrics output. + pattern: "*.tsv" + + - cluster_plots: + type: map + description: Groovy map of sample metadata. + plots: + type: file + description: Cluster visualization outputs. + pattern: "*.{png,pdf,html}" - versions: - - versions: - type: file - description: Software versions collected from the subworkflow. - pattern: "versions.yml" + type: file + description: Software versions collected from the subworkflow. + pattern: "versions.yml" authors: - "@dbaku42" From 8bbe838af36e3efdcc9adae8591044137f8c7372 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Mon, 6 Jul 2026 14:06:48 +0200 Subject: [PATCH 03/34] Update subworkflows/nf-core/snpclustering/main.nf MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Famke Bäuerle <45968370+famosab@users.noreply.github.com> --- subworkflows/nf-core/snpclustering/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 221553311025..b84372caf897 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,4 +1,4 @@ -include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle/main' +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' From e1862fc685bb5bd8c313ec261732153933408524 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Mon, 6 Jul 2026 15:42:45 +0200 Subject: [PATCH 04/34] Remove ext.prefix assignments from processes --- .../nf-core/snpclustering/tests/nextflow.config | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 99eba676d04a..9fa98d971a85 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -9,36 +9,32 @@ params { process { withName: 'BEAGLE5_BEAGLE' { - ext.prefix = { "${meta.id}" } + ext.when = false } withName: 'PLINK2_VCF' { - ext.prefix = { "${meta.id}" } + ext.when = false } withName: 'PLINK2_PCA' { - ext.prefix = { "${meta.id}" } + ext.when = false } - withName: 'EIGENVEC_TO_TSV' { - ext.prefix = { "${meta.id}" } - } - withName: 'CUSTOM_PCACLUSTERING' { - ext.prefix = { "${meta.id}" } + ext.when = false } withName: 'CUSTOM_CLUSTERMETRICS' { - ext.prefix = { "${meta.id}" } + ext.when = false } withName: 'CUSTOM_CLUSTERVISUALIZATION' { - ext.prefix = { "${meta.id}" } + ext.when = false } } From fc8ef720a506186a791c0ec7b2639b79a457fb44 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:45:17 +0200 Subject: [PATCH 05/34] Align include statements for consistency --- subworkflows/nf-core/snpclustering/main.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index b84372caf897..c2cf741b69de 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,8 +1,8 @@ -include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' -include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' -include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' -include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' -include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' workflow SNPCLUSTERING { From 6dd867d821e71744fba49fb64c6f5365a2f70a38 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Thu, 9 Jul 2026 14:57:32 +0200 Subject: [PATCH 06/34] Refactor process definitions in nextflow.config --- .../snpclustering/tests/nextflow.config | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 9fa98d971a85..a3e8e398d109 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -8,33 +8,27 @@ params { } process { - withName: 'BEAGLE5_BEAGLE' { - + withName: 'BEAGLE5_BEAGLE' { ext.when = false } - withName: 'PLINK2_VCF' { - + withName: 'PLINK2_VCF' { ext.when = false } - withName: 'PLINK2_PCA' { - + withName: 'PLINK2_PCA' { ext.when = false } - withName: 'CUSTOM_PCACLUSTERING' { - + withName: 'CUSTOM_PCACLUSTERING' { ext.when = false } - withName: 'CUSTOM_CLUSTERMETRICS' { - + withName: 'CUSTOM_CLUSTERMETRICS' { ext.when = false } - withName: 'CUSTOM_CLUSTERVISUALIZATION' { - + withName: 'CUSTOM_CLUSTERVISUALIZATION' { ext.when = false } } From 22e4b021ad1c71f1529e2cc2f120460aa5352b09 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:15:36 +0200 Subject: [PATCH 07/34] Clean up comments in main.nf Removed comments about optional files and signature inference. --- subworkflows/nf-core/snpclustering/main.nf | 4 ---- 1 file changed, 4 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index c2cf741b69de..e8d12f37a23c 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -26,9 +26,6 @@ workflow SNPCLUSTERING { * Build BEAGLE input tuple: * tuple val(meta), path(vcf), path(vcf_index), path(refpanel), path(refpanel_index), * path(genmap), path(exclsamples), path(exclmarkers), val(region) - * - * For now, refpanel/genmap/exclusion files are optional and passed as empty lists - * when not provided by the caller. */ ch_beagle_input = vcf_ch.map { meta, vcf, vcf_index -> tuple( @@ -74,7 +71,6 @@ workflow SNPCLUSTERING { /* * PCA clustering - * Signature inferred from your successful wiring so far: * CUSTOM_PCACLUSTERING(tsv, algorithm, n_clusters, dbscan_eps, dbscan_min_samples) */ CUSTOM_PCACLUSTERING( From bb78b2283dc99f2a099b9a947675e96f84845caf Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Mon, 17 Aug 2026 19:12:03 +0200 Subject: [PATCH 08/34] WIP: fix versions emit in custom modules --- .../custom/clustermetrics/environment.yml | 12 ++ modules/nf-core/custom/clustermetrics/main.nf | 46 ++++++ .../nf-core/custom/clustermetrics/meta.yml | 96 ++++++++++++ .../templates/cluster_metrics.py | 143 ++++++++++++++++++ .../custom/clustermetrics/tests/main.nf.test | 70 +++++++++ .../clustermetrics/tests/main.nf.test.snap | 104 +++++++++++++ .../clustervisualization/environment.yml | 14 ++ .../custom/clustervisualization/main.nf | 45 ++++++ .../custom/clustervisualization/meta.yml | 101 +++++++++++++ .../templates/cluster_viz.py | 129 ++++++++++++++++ .../clustervisualization/tests/main.nf.test | 60 ++++++++ .../tests/main.nf.test.snap | 95 ++++++++++++ .../custom/pcaclustering/environment.yml | 11 ++ modules/nf-core/custom/pcaclustering/main.nf | 42 +++++ modules/nf-core/custom/pcaclustering/meta.yml | 81 ++++++++++ .../pcaclustering/templates/clustering.py | 81 ++++++++++ .../custom/pcaclustering/tests/main.nf.test | 85 +++++++++++ .../pcaclustering/tests/main.nf.test.snap | 106 +++++++++++++ subworkflows/nf-core/snpclustering/main.nf | 30 ++-- .../snpclustering/tests/nextflow.config | 22 +-- 20 files changed, 1346 insertions(+), 27 deletions(-) create mode 100644 modules/nf-core/custom/clustermetrics/environment.yml create mode 100644 modules/nf-core/custom/clustermetrics/main.nf create mode 100644 modules/nf-core/custom/clustermetrics/meta.yml create mode 100644 modules/nf-core/custom/clustermetrics/templates/cluster_metrics.py create mode 100644 modules/nf-core/custom/clustermetrics/tests/main.nf.test create mode 100644 modules/nf-core/custom/clustermetrics/tests/main.nf.test.snap create mode 100644 modules/nf-core/custom/clustervisualization/environment.yml create mode 100644 modules/nf-core/custom/clustervisualization/main.nf create mode 100644 modules/nf-core/custom/clustervisualization/meta.yml create mode 100644 modules/nf-core/custom/clustervisualization/templates/cluster_viz.py create mode 100644 modules/nf-core/custom/clustervisualization/tests/main.nf.test create mode 100644 modules/nf-core/custom/clustervisualization/tests/main.nf.test.snap create mode 100644 modules/nf-core/custom/pcaclustering/environment.yml create mode 100644 modules/nf-core/custom/pcaclustering/main.nf create mode 100644 modules/nf-core/custom/pcaclustering/meta.yml create mode 100644 modules/nf-core/custom/pcaclustering/templates/clustering.py create mode 100644 modules/nf-core/custom/pcaclustering/tests/main.nf.test create mode 100644 modules/nf-core/custom/pcaclustering/tests/main.nf.test.snap diff --git a/modules/nf-core/custom/clustermetrics/environment.yml b/modules/nf-core/custom/clustermetrics/environment.yml new file mode 100644 index 000000000000..8d7b4bbbb690 --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/environment.yml @@ -0,0 +1,12 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::matplotlib=3.10.9 + - conda-forge::numpy=2.4.4 + - conda-forge::pandas=3.0.3 + - conda-forge::python=3.12.13 + - conda-forge::pyyaml=6.0.3 + - conda-forge::scikit-learn=1.8.0 diff --git a/modules/nf-core/custom/clustermetrics/main.nf b/modules/nf-core/custom/clustermetrics/main.nf new file mode 100644 index 000000000000..de01489438ad --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/main.nf @@ -0,0 +1,46 @@ +process CUSTOM_CLUSTERMETRICS { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/25/25129a5258522a434c386b800d3e2e3e6dc72d8a1171b7b10f21df3488526795/data' : + 'community.wave.seqera.io/library/matplotlib_numpy_pandas_python_pruned:169e228afc7d3686' }" + + input: + tuple val(meta), path(features), path(clusters) + + output: + tuple val(meta), path("*.metrics.tsv") , emit: metrics + tuple val(meta), path("*.k_sweep.csv") , emit: k_sweep + tuple val(meta), path("*.selected.json"), emit: selected + tuple val(meta), path("*.png") , emit: plots, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + template 'cluster_metrics.py' + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.metrics.tsv + touch ${prefix}.k_sweep.csv + touch ${prefix}.selected.json + touch ${prefix}.elbow.png + touch ${prefix}.silhouette.png + touch ${prefix}.davies_bouldin.png + touch ${prefix}.calinski_harabasz.png + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python3 --version | sed 's/Python //') + matplotlib: \$(python3 -c "from importlib.metadata import version; print(version('matplotlib'))") + numpy: \$(python3 -c "from importlib.metadata import version; print(version('numpy'))") + pandas: \$(python3 -c "from importlib.metadata import version; print(version('pandas'))") + scikit-learn: \$(python3 -c "from importlib.metadata import version; print(version('scikit-learn'))") + END_VERSIONS + """ +} diff --git a/modules/nf-core/custom/clustermetrics/meta.yml b/modules/nf-core/custom/clustermetrics/meta.yml new file mode 100644 index 000000000000..e364f55b6f75 --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/meta.yml @@ -0,0 +1,96 @@ +name: "CUSTOM_CLUSTERMETRICS" +description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, + Davies-Bouldin) and performs k-sweep analysis" +keywords: + - clustering + - metrics + - silhouette + - calinski-harabasz + - davies-bouldin + - evaluation +tools: + - "scikit-learn": + description: "Machine learning library for clustering metrics" + homepage: "https://scikit-learn.org/" + documentation: "https://scikit-learn.org/stable/modules/clustering.html" + licence: + - "BSD-3-Clause" + identifier: "" +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1' ]` + - features: + type: file + description: | + Tab-separated feature matrix with a `sample_id` column and one + column per numeric feature (e.g. PCA scores). + pattern: "*.tsv" + ontologies: + - edam: http://edamontology.org/format_3475 + - clusters: + type: file + description: | + Comma-separated cluster assignments with `sample_id` and integer + `cluster` columns. Label -1 is treated as DBSCAN noise. + pattern: "*.csv" + ontologies: + - edam: http://edamontology.org/format_3752 +output: + metrics: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.metrics.tsv": + type: file + description: TSV with selected cluster quality metrics + pattern: "*.metrics.tsv" + ontologies: + - edam: http://edamontology.org/format_3475 + k_sweep: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.k_sweep.csv": + type: file + description: CSV with metrics for different values of k + pattern: "*.k_sweep.csv" + ontologies: + - edam: http://edamontology.org/format_3752 + selected: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.selected.json": + type: file + description: JSON with the selected/best metrics + pattern: "*.selected.json" + ontologies: + - edam: http://edamontology.org/format_3464 + plots: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.png": + type: file + description: Optional PNG plots (elbow, silhouette, etc.) + pattern: "*.png" + ontologies: [] + versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + ontologies: + - edam: http://edamontology.org/format_3750 +topics: + versions: + - versions.yml: + type: string + description: The name of the process +authors: + - "@dbaku42" +maintainers: + - "@dbaku42" diff --git a/modules/nf-core/custom/clustermetrics/templates/cluster_metrics.py b/modules/nf-core/custom/clustermetrics/templates/cluster_metrics.py new file mode 100644 index 000000000000..fd8856fd41cf --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/templates/cluster_metrics.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +# Copyright (c) nf-core +# This software is licensed under the MIT License. +# SPDX-License-Identifier: MIT + +import argparse +import json +import platform +import shlex + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import pandas as pd +import sklearn +import yaml +from sklearn.cluster import KMeans +from sklearn.metrics import ( + calinski_harabasz_score, + davies_bouldin_score, + silhouette_score, +) + + +def load_features(path): + """Read a TSV of `sample_id` + numeric feature columns, indexed by sample_id.""" + df = pd.read_csv(path, sep="\\t") + if "sample_id" not in df.columns: + raise ValueError(f"features file must have a 'sample_id' column. Found: {list(df.columns)}") + df["sample_id"] = df["sample_id"].astype(str) + return df.set_index("sample_id").apply(pd.to_numeric, errors="coerce").fillna(0.0) + + +def load_clusters(path): + """Read a CSV of `sample_id` + `cluster`, returning a Series of int labels.""" + df = pd.read_csv(path) + if "sample_id" not in df.columns or "cluster" not in df.columns: + raise ValueError(f"clusters file must have 'sample_id' and 'cluster' columns. Found: {list(df.columns)}") + df["sample_id"] = df["sample_id"].astype(str) + return df.set_index("sample_id")["cluster"].astype(int) + + +def cluster_quality(x, labels): + """Silhouette / Calinski-Harabasz / Davies-Bouldin for given (x, labels). + + Treats label -1 as DBSCAN noise and excludes those points. Returns None + for each score when fewer than 2 clusters of more than one point remain. + """ + mask = labels != -1 + x, labels = x[mask], labels[mask] + n = len(set(labels)) + valid = 2 <= n < len(x) + return { + "silhouette": float(silhouette_score(x, labels)) if valid else None, + "calinski_harabasz": float(calinski_harabasz_score(x, labels)) if valid else None, + "davies_bouldin": float(davies_bouldin_score(x, labels)) if valid else None, + } + + +def plot_curve(sweep_df, metric, title, ylabel, out_png): + plt.figure(figsize=(7, 4.5)) + vals = sweep_df[metric].dropna() + ks = sweep_df.loc[vals.index, "k"] + plt.plot(ks, vals, marker="o") + plt.xticks(sweep_df["k"].tolist()) + plt.title(title) + plt.xlabel("k") + plt.ylabel(ylabel) + plt.tight_layout() + plt.savefig(out_png, dpi=200) + plt.close() + + +def main(): + features = "$features" + clusters_path = "$clusters" + prefix = "${task.ext.prefix ?: meta.id}" + + # Optional configuration via task.ext.args (nf-core convention). + raw_args = "$task.ext.args" + parser = argparse.ArgumentParser() + parser.add_argument("--k-min", type=int, default=2) + parser.add_argument("--k-max", type=int, default=12) + opts = parser.parse_args(shlex.split(raw_args) if raw_args and raw_args != "null" else []) + + joined = load_features(features).join(load_clusters(clusters_path), how="inner") + if len(joined) < 2: + raise ValueError(f"Need at least 2 samples with matching sample_id in both inputs. Got {len(joined)}.") + + labels = joined["cluster"].values + x = joined.drop(columns=["cluster"]).to_numpy(dtype=float) + + # Quality metrics on the supplied labels. + selected = {"n_clusters": len(set(labels) - {-1}), **cluster_quality(x, labels)} + pd.DataFrame([selected]).to_csv(f"{prefix}.metrics.tsv", sep="\\t", index=False) + with open(f"{prefix}.selected.json", "w") as fh: + json.dump(selected, fh, indent=2) + + # KMeans k-sweep for downstream comparison. + rows = [] + for k in range(opts.k_min, min(opts.k_max, len(x)) + 1): + model = KMeans(n_clusters=k, n_init=10, random_state=42).fit(x) + rows.append({"k": k, "inertia": float(model.inertia_), **cluster_quality(x, model.labels_)}) + + sweep_df = pd.DataFrame(rows) + sweep_df.to_csv(f"{prefix}.k_sweep.csv", index=False, float_format="%.10g") + + if not sweep_df.empty: + plot_curve(sweep_df, "inertia", "Elbow method (KMeans inertia)", "inertia", f"{prefix}.elbow.png") + plot_curve( + sweep_df, "silhouette", "Silhouette score (higher is better)", "silhouette", f"{prefix}.silhouette.png" + ) + plot_curve( + sweep_df, + "davies_bouldin", + "Davies-Bouldin index (lower is better)", + "davies_bouldin", + f"{prefix}.davies_bouldin.png", + ) + plot_curve( + sweep_df, + "calinski_harabasz", + "Calinski-Harabasz index (higher is better)", + "calinski_harabasz", + f"{prefix}.calinski_harabasz.png", + ) + + versions = { + "${task.process}": { + "python": platform.python_version(), + "pandas": pd.__version__, + "scikit-learn": sklearn.__version__, + "matplotlib": matplotlib.__version__, + } + } + with open("versions.yml", "w") as fh: + yaml.dump(versions, fh, default_flow_style=False, sort_keys=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/custom/clustermetrics/tests/main.nf.test b/modules/nf-core/custom/clustermetrics/tests/main.nf.test new file mode 100644 index 000000000000..d5a42e682198 --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/tests/main.nf.test @@ -0,0 +1,70 @@ +nextflow_process { + + name "Test Process CUSTOM_CLUSTERMETRICS" + script "../main.nf" + process "CUSTOM_CLUSTERMETRICS" + + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "custom/clustermetrics" + + test("clustermetrics - features and clusters") { + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true), + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_clusters.csv", checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.metrics, + process.out.k_sweep, + process.out.selected, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() } + ) + } + } + + test("clustermetrics - features and clusters - stub") { + + options "-stub" + + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true), + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_clusters.csv", checkIfExists: true) + ] + """ + } + } + + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.metrics, + process.out.k_sweep, + process.out.selected, + process.out.plots, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() } + ) + } + } +} diff --git a/modules/nf-core/custom/clustermetrics/tests/main.nf.test.snap b/modules/nf-core/custom/clustermetrics/tests/main.nf.test.snap new file mode 100644 index 000000000000..0ec0df908bc5 --- /dev/null +++ b/modules/nf-core/custom/clustermetrics/tests/main.nf.test.snap @@ -0,0 +1,104 @@ +{ + "clustermetrics - features and clusters": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,15c36eab43e480e0311c4bcc3d511477" + ] + ], + [ + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,98635ea739c5e136ced833916ae5d931" + ] + ], + [ + [ + { + "id": "test" + }, + "test.selected.json:md5,d1d1b3788b7a111f38bf3b27d4bb1ab4" + ] + ], + [ + "versions.yml:md5,602aa5dfe6c0b807d758c4f5cf3fc5e4" + ], + { + "CUSTOM_CLUSTERMETRICS": { + "python": "3.12.13", + "pandas": "3.0.3", + "scikit-learn": "1.8.0", + "matplotlib": "3.10.9" + } + } + ], + "timestamp": "2026-05-19T14:11:28.630001071", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + }, + "clustermetrics - features and clusters - stub": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.selected.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + [ + "test.calinski_harabasz.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.davies_bouldin.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.elbow.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.silhouette.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] + ], + [ + "versions.yml:md5,90240f0a22302d1727455b6be6e96abb" + ], + { + "CUSTOM_CLUSTERMETRICS": { + "python": "3.12.13", + "matplotlib": "3.10.9", + "numpy": "2.4.4", + "pandas": "3.0.3", + "scikit-learn": "1.8.0" + } + } + ], + "timestamp": "2026-05-19T14:11:36.600888057", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/custom/clustervisualization/environment.yml b/modules/nf-core/custom/clustervisualization/environment.yml new file mode 100644 index 000000000000..befadae8f312 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/environment.yml @@ -0,0 +1,14 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::matplotlib=3.10.9 + - conda-forge::numpy=2.4.4 + - conda-forge::pandas=3.0.3 + - conda-forge::python=3.12.13 + - conda-forge::pyyaml=6.0.3 + - conda-forge::scikit-learn=1.8.0 + - conda-forge::seaborn=0.13.2 + - conda-forge::umap-learn=0.5.12 diff --git a/modules/nf-core/custom/clustervisualization/main.nf b/modules/nf-core/custom/clustervisualization/main.nf new file mode 100644 index 000000000000..0b1346f20de4 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/main.nf @@ -0,0 +1,45 @@ +process CUSTOM_CLUSTERVISUALIZATION { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/64/64297e13d9d4f05ce543656e943a023735e7cb252d7534ccc9134d8b40423083/data' : + 'community.wave.seqera.io/library/matplotlib_numpy_pandas_python_pruned:826e4ab1361ff931' }" + + input: + tuple val(meta), path(features), path(clusters) + + output: + tuple val(meta), path("*.umap.tsv"), emit: umap_tsv + tuple val(meta), path("*.tsne.tsv"), emit: tsne_tsv + tuple val(meta), path("*.umap.png"), emit: umap_png, optional: true + tuple val(meta), path("*.tsne.png"), emit: tsne_png, optional: true + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + template 'cluster_viz.py' + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.umap.tsv + touch ${prefix}.tsne.tsv + touch ${prefix}.umap.png + touch ${prefix}.tsne.png + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python3 --version | sed 's/Python //') + matplotlib: \$(python3 -c "from importlib.metadata import version; print(version('matplotlib'))") + numpy: \$(python3 -c "from importlib.metadata import version; print(version('numpy'))") + pandas: \$(python3 -c "from importlib.metadata import version; print(version('pandas'))") + scikit-learn: \$(python3 -c "from importlib.metadata import version; print(version('scikit-learn'))") + seaborn: \$(python3 -c "from importlib.metadata import version; print(version('seaborn'))") + umap-learn: \$(python3 -c "from importlib.metadata import version; print(version('umap-learn'))") + END_VERSIONS + """ +} diff --git a/modules/nf-core/custom/clustervisualization/meta.yml b/modules/nf-core/custom/clustervisualization/meta.yml new file mode 100644 index 000000000000..62bed07814e5 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/meta.yml @@ -0,0 +1,101 @@ +name: "CUSTOM_CLUSTERVISUALIZATION" +description: "Generates UMAP and t-SNE visualizations colored by cluster" +keywords: + - clustering + - visualization + - pca + - umap + - tsne + - dimension-reduction +tools: + - scikit-learn: + description: "Machine learning library for dimension reduction (PCA, t-SNE)" + homepage: "https://scikit-learn.org/" + documentation: "https://scikit-learn.org/stable/modules/clustering.html" + licence: + - "BSD-3-Clause" + identifier: "" + - umap-learn: + description: "Uniform Manifold Approximation and Projection for dimension reduction" + homepage: "https://umap-learn.readthedocs.io/" + documentation: "https://umap-learn.readthedocs.io/en/latest/" + licence: + - "BSD-3-Clause" + identifier: "" +input: + - - meta: + type: map + description: "Groovy Map containing sample information" + - features: + type: file + description: | + Tab-separated feature matrix with a `sample_id` column and one + column per numeric feature (e.g. PCA scores). + pattern: "*.tsv" + ontologies: + - edam: http://edamontology.org/format_3475 + - clusters: + type: file + description: | + Comma-separated cluster assignments with `sample_id` and integer + `cluster` columns. Label -1 is treated as DBSCAN noise. + pattern: "*.csv" + ontologies: + - edam: http://edamontology.org/format_3752 +output: + umap_tsv: + - - meta: + type: map + description: "Groovy Map containing sample information" + - "*.umap.tsv": + type: file + description: "UMAP coordinates per sample" + pattern: "*.umap.tsv" + ontologies: + - edam: "http://edamontology.org/operation_2432" + - edam: http://edamontology.org/format_3475 + tsne_tsv: + - - meta: + type: map + description: "Groovy Map containing sample information" + - "*.tsne.tsv": + type: file + description: "t-SNE coordinates per sample" + pattern: "*.tsne.tsv" + ontologies: + - edam: "http://edamontology.org/operation_2432" + - edam: http://edamontology.org/format_3475 + umap_png: + - - meta: + type: map + description: "Groovy Map containing sample information" + - "*.umap.png": + type: file + description: "UMAP visualization coloured by cluster" + pattern: "*.umap.png" + ontologies: [] + tsne_png: + - - meta: + type: map + description: "Groovy Map containing sample information" + - "*.tsne.png": + type: file + description: "t-SNE visualization coloured by cluster" + pattern: "*.tsne.png" + ontologies: [] + versions: + - versions.yml: + type: file + description: "Software versions used in the module" + pattern: "versions.yml" + ontologies: + - edam: http://edamontology.org/format_3750 +topics: + versions: + - versions.yml: + type: string + description: The name of the process +authors: + - "@dbaku42" +maintainers: + - "@dbaku42" diff --git a/modules/nf-core/custom/clustervisualization/templates/cluster_viz.py b/modules/nf-core/custom/clustervisualization/templates/cluster_viz.py new file mode 100644 index 000000000000..6f6a60d5ac37 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/templates/cluster_viz.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +# Copyright (c) nf-core +# This software is licensed under the MIT License. +# SPDX-License-Identifier: MIT + +import argparse +import os +import shlex + +# numba (UMAP) and matplotlib write caches; redirect to /tmp so the script works +# inside read-only container filesystems. +os.environ.setdefault("NUMBA_CACHE_DIR", "/tmp") +os.environ.setdefault("MPLCONFIGDIR", "/tmp") + +import platform + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import seaborn as sns +import sklearn +import umap +import yaml +from sklearn.manifold import TSNE + + +def load_features(path): + """Read a TSV of `sample_id` + numeric feature columns, indexed by sample_id.""" + df = pd.read_csv(path, sep="\\t") + if "sample_id" not in df.columns: + raise ValueError(f"features file must have a 'sample_id' column. Found: {list(df.columns)}") + df["sample_id"] = df["sample_id"].astype(str) + return df.set_index("sample_id").apply(pd.to_numeric, errors="coerce").fillna(0.0) + + +def load_clusters(path): + """Read a CSV of `sample_id` + `cluster`, returning a Series of int labels.""" + df = pd.read_csv(path) + if "sample_id" not in df.columns or "cluster" not in df.columns: + raise ValueError(f"clusters file must have 'sample_id' and 'cluster' columns. Found: {list(df.columns)}") + df["sample_id"] = df["sample_id"].astype(str) + return df.set_index("sample_id")["cluster"].astype(int) + + +def embed(x, method, umap_neighbors, tsne_perplexity): + """Project x to 2D using UMAP or t-SNE. + + n_neighbors / perplexity are clamped against sample count so tiny test + inputs (where the user-specified defaults exceed n_samples) still run. + """ + n = len(x) + if method == "umap": + reducer = umap.UMAP(n_components=2, n_neighbors=min(umap_neighbors, max(2, n - 1)), random_state=42) + elif method == "tsne": + reducer = TSNE(n_components=2, perplexity=min(tsne_perplexity, max(2, n - 1)), random_state=42) + else: + raise ValueError(f"Unknown method '{method}' (expected 'umap' or 'tsne')") + return reducer.fit_transform(x) + + +def plot_embedding(emb, labels, method, out_png): + plt.figure(figsize=(8, 6)) + palette = sns.color_palette("tab10", n_colors=max(1, len(np.unique(labels)))) + sns.scatterplot( + x=emb[:, 0], + y=emb[:, 1], + hue=labels.astype(str), + palette=palette, + alpha=0.8, + s=60, + edgecolor="k", + linewidth=0.3, + ) + plt.title(f"{method.upper()} projection colored by cluster") + plt.xlabel(f"{method.upper()} 1") + plt.ylabel(f"{method.upper()} 2") + plt.legend(title="Cluster", bbox_to_anchor=(1.05, 1), loc="upper left") + plt.tight_layout() + plt.savefig(out_png, dpi=200, bbox_inches="tight") + plt.close() + + +def main(): + features = "$features" + clusters_path = "$clusters" + prefix = "${task.ext.prefix ?: meta.id}" + + # Optional configuration via task.ext.args (nf-core convention). + raw_args = "$task.ext.args" + parser = argparse.ArgumentParser() + parser.add_argument("--umap-neighbors", type=int, default=15) + parser.add_argument("--tsne-perplexity", type=int, default=30) + opts = parser.parse_args(shlex.split(raw_args) if raw_args and raw_args != "null" else []) + + joined = load_features(features).join(load_clusters(clusters_path), how="inner") + if len(joined) < 2: + raise ValueError(f"Need at least 2 samples with matching sample_id in both inputs. Got {len(joined)}.") + + labels = joined["cluster"].values + x = joined.drop(columns=["cluster"]).to_numpy(dtype=float) + sample_ids = joined.index.to_numpy() + + for method in ("umap", "tsne"): + emb = embed(x, method, opts.umap_neighbors, opts.tsne_perplexity) + pd.DataFrame({"sample_id": sample_ids, "Dim1": emb[:, 0], "Dim2": emb[:, 1], "cluster": labels}).to_csv( + f"{prefix}.{method}.tsv", sep="\\t", index=False + ) + plot_embedding(emb, labels, method, f"{prefix}.{method}.png") + + versions = { + "${task.process}": { + "python": platform.python_version(), + "pandas": pd.__version__, + "matplotlib": matplotlib.__version__, + "seaborn": sns.__version__, + "umap-learn": umap.__version__, + "scikit-learn": sklearn.__version__, + } + } + with open("versions.yml", "w") as fh: + yaml.dump(versions, fh, default_flow_style=False, sort_keys=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/custom/clustervisualization/tests/main.nf.test b/modules/nf-core/custom/clustervisualization/tests/main.nf.test new file mode 100644 index 000000000000..0334f0d09434 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/tests/main.nf.test @@ -0,0 +1,60 @@ +nextflow_process { + name "Test Process CUSTOM_CLUSTERVISUALIZATION" + script "../main.nf" + process "CUSTOM_CLUSTERVISUALIZATION" + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "custom/clustervisualization" + test("clustervisualization - features and clusters") { + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true), + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_clusters.csv", checkIfExists: true) + ] + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.umap_tsv, + process.out.tsne_tsv, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() } + ) + } + } + test("clustervisualization - features and clusters - stub") { + options "-stub" + when { + process { + """ + input[0] = [ + [ id:'test' ], + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true), + file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_clusters.csv", checkIfExists: true) + ] + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.umap_tsv, + process.out.tsne_tsv, + process.out.umap_png, + process.out.tsne_png, + process.out.versions, + path(process.out.versions[0]).yaml + ).match() } + ) + } + } +} diff --git a/modules/nf-core/custom/clustervisualization/tests/main.nf.test.snap b/modules/nf-core/custom/clustervisualization/tests/main.nf.test.snap new file mode 100644 index 000000000000..f180dfa8b993 --- /dev/null +++ b/modules/nf-core/custom/clustervisualization/tests/main.nf.test.snap @@ -0,0 +1,95 @@ +{ + "clustervisualization - features and clusters - stub": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.umap.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.umap.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.tsne.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + [ + "versions.yml:md5,76a5042b303be74c24dee681187d6a1f" + ], + { + "CUSTOM_CLUSTERVISUALIZATION": { + "python": "3.12.13", + "matplotlib": "3.10.9", + "numpy": "2.4.4", + "pandas": "3.0.3", + "scikit-learn": "1.8.0", + "seaborn": "0.13.2", + "umap-learn": "0.5.12" + } + } + ], + "timestamp": "2026-05-19T16:36:24.854839682", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.09.0" + } + }, + "clustervisualization - features and clusters": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.umap.tsv:md5,d338c1ef0e979dbf653e21c3417e975e" + ] + ], + [ + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,eb00f27d82530f665552b158e8e3c8ff" + ] + ], + [ + "versions.yml:md5,a00bfbb9b1b4145177ec0e8a7406caf9" + ], + { + "CUSTOM_CLUSTERVISUALIZATION": { + "python": "3.12.13", + "pandas": "3.0.3", + "matplotlib": "3.10.9", + "seaborn": "0.13.2", + "umap-learn": "0.5.12", + "scikit-learn": "1.8.0" + } + } + ], + "timestamp": "2026-05-19T17:08:32.73077357", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.2" + } + } +} \ No newline at end of file diff --git a/modules/nf-core/custom/pcaclustering/environment.yml b/modules/nf-core/custom/pcaclustering/environment.yml new file mode 100644 index 000000000000..1fda7f0e4c10 --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/environment.yml @@ -0,0 +1,11 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/nf-core/modules/master/modules/environment-schema.json +channels: + - conda-forge + - bioconda +dependencies: + - conda-forge::numpy=2.4.4 + - conda-forge::pandas=3.0.3 + - conda-forge::python=3.12.13 + - conda-forge::pyyaml=6.0.3 + - conda-forge::scikit-learn=1.8.0 diff --git a/modules/nf-core/custom/pcaclustering/main.nf b/modules/nf-core/custom/pcaclustering/main.nf new file mode 100644 index 000000000000..4a4be41f876b --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/main.nf @@ -0,0 +1,42 @@ +process CUSTOM_PCACLUSTERING { + tag "$meta.id" + label 'process_medium' + + conda "${moduleDir}/environment.yml" + container "${ workflow.containerEngine in ['singularity', 'apptainer'] && !task.ext.singularity_pull_docker_container ? + 'https://community-cr-prod.seqera.io/docker/registry/v2/blobs/sha256/a3/a37807bdaf3edad30a2b212962b6af381bc10381a80c40efb2bb07f6ee43032f/data' : + 'community.wave.seqera.io/library/numpy_pandas_python_pyyaml_scikit-learn:c500ceb82d3d7606' }" + + input: + tuple val(meta), path(features) + val algorithm + val n_clusters + val dbscan_eps + val dbscan_min_samples + + output: + tuple val(meta), path("*.clusters.csv") , emit: clusters + tuple val(meta), path("*.clustering_info.json"), emit: info + path "versions.yml" , emit: versions + + when: + task.ext.when == null || task.ext.when + + script: + template 'clustering.py' + + stub: + def prefix = task.ext.prefix ?: "${meta.id}" + """ + touch ${prefix}.clusters.csv + touch ${prefix}.clustering_info.json + + cat <<-END_VERSIONS > versions.yml + "${task.process}": + python: \$(python3 --version | sed 's/Python //') + numpy: \$(python3 -c "from importlib.metadata import version; print(version('numpy'))") + pandas: \$(python3 -c "from importlib.metadata import version; print(version('pandas'))") + scikit-learn: \$(python3 -c "from importlib.metadata import version; print(version('scikit-learn'))") + END_VERSIONS + """ +} diff --git a/modules/nf-core/custom/pcaclustering/meta.yml b/modules/nf-core/custom/pcaclustering/meta.yml new file mode 100644 index 000000000000..72a1cefbaa11 --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/meta.yml @@ -0,0 +1,81 @@ +name: "CUSTOM_PCACLUSTERING" +description: "Performs KMeans or DBSCAN clustering on a sample-by-feature numeric + matrix (e.g. principal components, embeddings)" +keywords: + - clustering + - kmeans + - dbscan + - pca + - embeddings +tools: + - "scikit-learn": + description: "Machine learning library for clustering" + homepage: "https://scikit-learn.org/" + documentation: "https://scikit-learn.org/stable/modules/clustering.html" + licence: + - "BSD-3-Clause" + identifier: "" +input: + - - meta: + type: map + description: | + Groovy Map containing sample information + e.g. `[ id:'sample1' ]` + - features: + type: file + description: | + Tab-separated file whose first column is sample IDs (any column + name) and remaining columns are numeric features (e.g. PLINK2 PCA + components with `FID` dropped, scikit-learn embeddings, etc.). + pattern: "*.{tsv,txt}" + ontologies: + - edam: http://edamontology.org/format_3475 + - algorithm: + type: string + description: Clustering algorithm to use (kmeans or dbscan) + - n_clusters: + type: integer + description: Number of clusters for KMeans + - dbscan_eps: + type: float + description: Epsilon parameter for DBSCAN + - dbscan_min_samples: + type: integer + description: Minimum samples parameter for DBSCAN +output: + clusters: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.clusters.csv": + type: file + description: CSV file with sample_id and assigned cluster + pattern: "*.clusters.csv" + ontologies: + - edam: http://edamontology.org/format_3752 + info: + - - meta: + type: map + description: Groovy Map containing sample information + - "*.clustering_info.json": + type: file + description: JSON file with clustering parameters and statistics + pattern: "*.clustering_info.json" + ontologies: + - edam: http://edamontology.org/format_3464 + versions: + - "versions.yml": + type: file + description: File containing software versions + pattern: "versions.yml" + ontologies: + - edam: http://edamontology.org/format_3750 +topics: + versions: + - versions.yml: + type: string + description: The name of the process +authors: + - "@dbaku42" +maintainers: + - "@dbaku42" diff --git a/modules/nf-core/custom/pcaclustering/templates/clustering.py b/modules/nf-core/custom/pcaclustering/templates/clustering.py new file mode 100644 index 000000000000..0a133a21557d --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/templates/clustering.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +# Copyright (c) nf-core +# This software is licensed under the MIT License. +# SPDX-License-Identifier: MIT + +import json +import platform + +import numpy as np +import pandas as pd +import sklearn +import yaml +from sklearn.cluster import DBSCAN, KMeans + + +def load_features(path): + """Read a TSV where the first column is sample IDs and the remaining + columns are numeric features. + + Returns (sample_ids: pd.Series, features: np.ndarray). + """ + df = pd.read_csv(path, sep="\\t", dtype={0: str}) + if df.shape[1] < 2: + raise ValueError(f"features file must have at least one feature column. Found columns: {list(df.columns)}") + sample_ids = df.iloc[:, 0] + features = df.iloc[:, 1:].to_numpy(dtype=float) + return sample_ids, features + + +def main(): + features_path = "$features" + algorithm = "$algorithm" + n_clusters = int("$n_clusters") + dbscan_eps = float("$dbscan_eps") + dbscan_min_samples = int("$dbscan_min_samples") + prefix = "${task.ext.prefix ?: meta.id}" + + sample_ids, x = load_features(features_path) + + if algorithm == "kmeans": + model = KMeans(n_clusters=n_clusters, init="random", n_init=100, random_state=42) + labels = model.fit_predict(x) + info = { + "algorithm": "kmeans", + "k": n_clusters, + "inertia": float(model.inertia_), + } + elif algorithm == "dbscan": + model = DBSCAN(eps=dbscan_eps, min_samples=dbscan_min_samples) + labels = model.fit_predict(x) + info = { + "algorithm": "dbscan", + "eps": dbscan_eps, + "min_samples": dbscan_min_samples, + "n_clusters_found": len(set(labels) - {-1}), + "n_noise": int(np.sum(labels == -1)), + } + else: + raise ValueError(f"Unknown algorithm '{algorithm}' (expected 'kmeans' or 'dbscan')") + + info |= {"n_samples": int(x.shape[0]), "n_features": int(x.shape[1])} + + pd.DataFrame({"sample_id": sample_ids, "cluster": labels}).to_csv(f"{prefix}.clusters.csv", index=False) + with open(f"{prefix}.clustering_info.json", "w") as fh: + json.dump(info, fh, indent=2) + + versions = { + "${task.process}": { + "python": platform.python_version(), + "pandas": pd.__version__, + "numpy": np.__version__, + "scikit-learn": sklearn.__version__, + } + } + with open("versions.yml", "w") as fh: + yaml.dump(versions, fh, default_flow_style=False, sort_keys=False) + + +if __name__ == "__main__": + main() diff --git a/modules/nf-core/custom/pcaclustering/tests/main.nf.test b/modules/nf-core/custom/pcaclustering/tests/main.nf.test new file mode 100644 index 000000000000..2df4745f7679 --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/tests/main.nf.test @@ -0,0 +1,85 @@ +nextflow_process { + name "Test Process CUSTOM_PCACLUSTERING" + script "../main.nf" + process "CUSTOM_PCACLUSTERING" + + tag "modules" + tag "modules_nfcore" + tag "custom" + tag "custom/pcaclustering" + + test("clustering - eigenvec") { + when { + process { + """ + input[0] = [ [id:'test'], file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true) ] + input[1] = 'kmeans' + input[2] = 3 + input[3] = 0.5 + input[4] = 5 + """ + } + } + then { + // KMeans inertia varies by a few ULPs across CPU instruction sets + // (BLAS reduction order), so parse the JSON and round it for the + // snapshot. Production output keeps full precision. + def info_data = new groovy.json.JsonSlurper().parse(file(process.out.info[0][1])) + info_data.inertia = (info_data.inertia as Double).round(4) + + assertAll( + { assert process.success }, + { assert snapshot( + process.out.clusters, + info_data, + process.out.versions + ).match() } + ) + } + } + + test("clustering - eigenvec - dbscan") { + when { + process { + """ + input[0] = [ [id:'test'], file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true) ] + input[1] = 'dbscan' + input[2] = 3 + input[3] = 0.5 + input[4] = 2 + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot( + process.out.clusters, + process.out.info, + process.out.versions + ).match() } + ) + } + } + + test("clustering - eigenvec - stub") { + options "-stub" + when { + process { + """ + input[0] = [ [id:'test'], file(params.modules_testdata_base_path + "genomics/homo_sapiens/popgen/clustering/test_features.tsv", checkIfExists: true) ] + input[1] = 'kmeans' + input[2] = 3 + input[3] = 0.5 + input[4] = 5 + """ + } + } + then { + assertAll( + { assert process.success }, + { assert snapshot(process.out).match() } + ) + } + } +} diff --git a/modules/nf-core/custom/pcaclustering/tests/main.nf.test.snap b/modules/nf-core/custom/pcaclustering/tests/main.nf.test.snap new file mode 100644 index 000000000000..7e08b009bf70 --- /dev/null +++ b/modules/nf-core/custom/pcaclustering/tests/main.nf.test.snap @@ -0,0 +1,106 @@ +{ + "clustering - eigenvec - stub": { + "content": [ + { + "0": [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "1": [ + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "2": [ + "versions.yml:md5,0464456de4c7d72da565bc36330bb1a8" + ], + "clusters": [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "info": [ + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ], + "versions": [ + "versions.yml:md5,0464456de4c7d72da565bc36330bb1a8" + ] + } + ], + "timestamp": "2026-05-19T17:25:31.986165338", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.2" + } + }, + "clustering - eigenvec": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,cd4fbcba75a4ec9f809eb0f1c58d2f72" + ] + ], + { + "algorithm": "kmeans", + "k": 3, + "inertia": 8.6821, + "n_samples": 200, + "n_features": 10 + }, + [ + "versions.yml:md5,18e52941377b816e6e2b9e1a6b971fe6" + ] + ], + "timestamp": "2026-05-19T17:25:21.968960876", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.2" + } + }, + "clustering - eigenvec - dbscan": { + "content": [ + [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,92f56a552d32dce29af32327ab926a25" + ] + ], + [ + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,9e45308fc57961f23e547a1149ad0c26" + ] + ], + [ + "versions.yml:md5,18e52941377b816e6e2b9e1a6b971fe6" + ] + ], + "timestamp": "2026-05-19T17:25:27.683788344", + "meta": { + "nf-test": "0.9.5", + "nextflow": "25.10.2" + } + } +} \ No newline at end of file diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 221553311025..f23d4443365a 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,8 +1,8 @@ -include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle/main' -include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' -include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' -include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' -include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' workflow SNPCLUSTERING { @@ -26,9 +26,6 @@ workflow SNPCLUSTERING { * Build BEAGLE input tuple: * tuple val(meta), path(vcf), path(vcf_index), path(refpanel), path(refpanel_index), * path(genmap), path(exclsamples), path(exclmarkers), val(region) - * - * For now, refpanel/genmap/exclusion files are optional and passed as empty lists - * when not provided by the caller. */ ch_beagle_input = vcf_ch.map { meta, vcf, vcf_index -> tuple( @@ -74,7 +71,6 @@ workflow SNPCLUSTERING { /* * PCA clustering - * Signature inferred from your successful wiring so far: * CUSTOM_PCACLUSTERING(tsv, algorithm, n_clusters, dbscan_eps, dbscan_min_samples) */ CUSTOM_PCACLUSTERING( @@ -133,6 +129,7 @@ workflow SNPCLUSTERING { } process EIGENVEC_TO_TSV { + tag "${meta.id}" label 'process_single' @@ -145,8 +142,18 @@ process EIGENVEC_TO_TSV { script: """ - awk 'NR==1 { sub(/^#/, ""); \$1 = ""; sub(/^\\t/, ""); sub(/^IID\\t/, "sample_id\\t"); print; next } - { \$1 = ""; sub(/^\\t/, ""); print }' OFS='\\t' ${eigenvec} > ${meta.id}.tsv + awk ' + NR==1 { + sub(/^#/, "") + if (\$1 == "IID") { + \$1="sample_id" + } + print + next + } + { + print + }' OFS='\\t' ${eigenvec} > ${meta.id}.tsv cat <<-END_VERSIONS > versions.yml "${task.process}": @@ -154,3 +161,4 @@ process EIGENVEC_TO_TSV { END_VERSIONS """ } + diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 99eba676d04a..a3e8e398d109 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -8,37 +8,27 @@ params { } process { - withName: 'BEAGLE5_BEAGLE' { - ext.prefix = { "${meta.id}" } + withName: 'BEAGLE5_BEAGLE' { ext.when = false } - withName: 'PLINK2_VCF' { - ext.prefix = { "${meta.id}" } + withName: 'PLINK2_VCF' { ext.when = false } - withName: 'PLINK2_PCA' { - ext.prefix = { "${meta.id}" } + withName: 'PLINK2_PCA' { ext.when = false } - withName: 'EIGENVEC_TO_TSV' { - ext.prefix = { "${meta.id}" } - } - - withName: 'CUSTOM_PCACLUSTERING' { - ext.prefix = { "${meta.id}" } + withName: 'CUSTOM_PCACLUSTERING' { ext.when = false } - withName: 'CUSTOM_CLUSTERMETRICS' { - ext.prefix = { "${meta.id}" } + withName: 'CUSTOM_CLUSTERMETRICS' { ext.when = false } - withName: 'CUSTOM_CLUSTERVISUALIZATION' { - ext.prefix = { "${meta.id}" } + withName: 'CUSTOM_CLUSTERVISUALIZATION' { ext.when = false } } From a84585e425c3cd292a0cd4df03571fe38822afbe Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 19 Aug 2026 09:27:43 +0200 Subject: [PATCH 09/34] fix: adapt ch_versions.mix() to migrated topic-based emit names (plink2) --- subworkflows/nf-core/snpclustering/main.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index f23d4443365a..62eabbf5c7f9 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -48,7 +48,7 @@ workflow SNPCLUSTERING { * Convert imputed VCF to PLINK2 pfiles */ PLINK2_VCF(BEAGLE5_BEAGLE.out.vcf) - ch_versions = ch_versions.mix(PLINK2_VCF.out.versions) + ch_versions = ch_versions.mix(PLINK2_VCF.out.versions_plink2) /* * PLINK2_PCA expects: @@ -62,7 +62,7 @@ workflow SNPCLUSTERING { } PLINK2_PCA(ch_plink_pca_input) - ch_versions = ch_versions.mix(PLINK2_PCA.out.versions) + ch_versions = ch_versions.mix(PLINK2_PCA.out.versions_plink2) /* * Convert .eigenvec to TSV for downstream clustering modules From 4e37881c6b43ae44c24c0bd30f7293276743ae52 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 20 Aug 2026 16:23:44 +0200 Subject: [PATCH 10/34] test: regenerate snpclustering snapshots with real content, Root cause of empty snapshots: tests/nextflow.config had ext.when=false set on all main processes, skipping their execution entirely while workflow.success remained true. Removed that block and regenerated snapshots using --profile docker. --- .../snpclustering/tests/main.nf.test.snap | 410 +++++++++++++++--- .../snpclustering/tests/nextflow.config | 36 +- 2 files changed, 359 insertions(+), 87 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index f4d7988e57d5..7bb0440847d8 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -2,185 +2,483 @@ "homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters": { "content": [ [ - + [ + { + "id": "test" + }, + "test.clusters.csv" + ] ], [ - + [ + { + "id": "test" + }, + "test.clustering_info.json" + ] ], [ - + [ + { + "id": "test" + }, + "test.metrics.tsv" + ] ], [ - + [ + "SNPCLUSTERING:BEAGLE5_BEAGLE", + "beagle", + "5.5" + ], + [ + "SNPCLUSTERING:PLINK2_PCA", + "plink2", + "2.00a5.10LM" + ], + [ + "SNPCLUSTERING:PLINK2_VCF", + "plink2", + "2.00a2.3LM" + ], + "versions.yml:md5,088adb486c0aa9f234effd310576eaf7", + "versions.yml:md5,5e7ab136010cdcda51f91b12562b28c7", + "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" ] ], - "timestamp": "2026-06-04T01:16:07.326033555", + "timestamp": "2026-08-20T16:08:17.636962775", "meta": { "nf-test": "0.9.5", - "nextflow": "25.09.0" + "nextflow": "26.04.6" } }, "homo_sapiens - 1000GP.chr22.vcf.gz - dbscan": { "content": [ [ - + [ + { + "id": "test" + }, + "test.clusters.csv" + ] ], [ - + [ + { + "id": "test" + }, + "test.clustering_info.json" + ] ], [ - + [ + { + "id": "test" + }, + "test.metrics.tsv" + ] ], [ - + [ + "SNPCLUSTERING:BEAGLE5_BEAGLE", + "beagle", + "5.5" + ], + [ + "SNPCLUSTERING:PLINK2_PCA", + "plink2", + "2.00a5.10LM" + ], + [ + "SNPCLUSTERING:PLINK2_VCF", + "plink2", + "2.00a2.3LM" + ], + "versions.yml:md5,088adb486c0aa9f234effd310576eaf7", + "versions.yml:md5,5e7ab136010cdcda51f91b12562b28c7", + "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" ] ], - "timestamp": "2026-06-04T01:16:13.695098223", + "timestamp": "2026-08-20T16:08:55.94750336", "meta": { "nf-test": "0.9.5", - "nextflow": "25.09.0" + "nextflow": "26.04.6" } }, "homo_sapiens - 1000GP.chr22.vcf.gz - dbscan - stub": { "content": [ { "beagle_log": [ - + [ + { + "id": "test" + }, + "test.bglout.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "cluster_info": [ - + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "clusters": [ - + [ + { + "id": "test" + }, + "test.clusters.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "evecfile": [ - + [ + { + "id": "test" + }, + "test.eigenvec:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "evfile": [ - + [ + { + "id": "test" + }, + "test.eigenval:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "imputed_vcf": [ - + [ + { + "id": "test" + }, + "test.bglout.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] ], "k_sweep": [ - + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "metric_plots": [ - + [ + { + "id": "test" + }, + [ + "test.calinski_harabasz.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.davies_bouldin.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.elbow.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.silhouette.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] ], "metrics": [ - + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pca_log": [ - + [ + { + "id": "test" + }, + "test.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pgen": [ - + [ + { + "id": "test" + }, + "test.pgen:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "psam": [ - + [ + { + "id": "test" + }, + "test.psam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pvar": [ - + [ + { + "id": "test" + }, + "test.pvar:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "selected": [ - + [ + { + "id": "test" + }, + "test.selected.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsne_png": [ - + [ + { + "id": "test" + }, + "test.tsne.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsne_tsv": [ - + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsv": [ - + [ + { + "id": "test" + }, + "test.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "umap_png": [ - + [ + { + "id": "test" + }, + "test.umap.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "umap_tsv": [ - + [ + { + "id": "test" + }, + "test.umap.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "versions": [ - + [ + "SNPCLUSTERING:BEAGLE5_BEAGLE", + "beagle", + "5.5" + ], + [ + "SNPCLUSTERING:PLINK2_PCA", + "plink2", + "2.00a5.10LM" + ], + [ + "SNPCLUSTERING:PLINK2_VCF", + "plink2", + "2.00a2.3LM" + ], + "versions.yml:md5,a5331a41c30b50872e8f9dc902b1fbdb", + "versions.yml:md5,c3f41cb637b4e448e41a2451ac6ea796", + "versions.yml:md5,c9f69ed90f9787ff5d98fda995824a89" ] } ], - "timestamp": "2026-06-04T01:16:30.853529288", + "timestamp": "2026-08-20T16:09:17.538676351", "meta": { "nf-test": "0.9.5", - "nextflow": "25.09.0" + "nextflow": "26.04.6" } }, "homo_sapiens - 1000GP.chr22.vcf.gz - kmeans - stub": { "content": [ { "beagle_log": [ - + [ + { + "id": "test" + }, + "test.bglout.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "cluster_info": [ - + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "clusters": [ - + [ + { + "id": "test" + }, + "test.clusters.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "evecfile": [ - + [ + { + "id": "test" + }, + "test.eigenvec:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "evfile": [ - + [ + { + "id": "test" + }, + "test.eigenval:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "imputed_vcf": [ - + [ + { + "id": "test" + }, + "test.bglout.vcf.gz:md5,68b329da9893e34099c7d8ad5cb9c940" + ] ], "k_sweep": [ - + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "metric_plots": [ - + [ + { + "id": "test" + }, + [ + "test.calinski_harabasz.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.davies_bouldin.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.elbow.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.silhouette.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] + ] ], "metrics": [ - + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pca_log": [ - + [ + { + "id": "test" + }, + "test.log:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pgen": [ - + [ + { + "id": "test" + }, + "test.pgen:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "psam": [ - + [ + { + "id": "test" + }, + "test.psam:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "pvar": [ - + [ + { + "id": "test" + }, + "test.pvar:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "selected": [ - + [ + { + "id": "test" + }, + "test.selected.json:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsne_png": [ - + [ + { + "id": "test" + }, + "test.tsne.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsne_tsv": [ - + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "tsv": [ - + [ + { + "id": "test" + }, + "test.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "umap_png": [ - + [ + { + "id": "test" + }, + "test.umap.png:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "umap_tsv": [ - + [ + { + "id": "test" + }, + "test.umap.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" + ] ], "versions": [ - + [ + "SNPCLUSTERING:BEAGLE5_BEAGLE", + "beagle", + "5.5" + ], + [ + "SNPCLUSTERING:PLINK2_PCA", + "plink2", + "2.00a5.10LM" + ], + [ + "SNPCLUSTERING:PLINK2_VCF", + "plink2", + "2.00a2.3LM" + ], + "versions.yml:md5,a5331a41c30b50872e8f9dc902b1fbdb", + "versions.yml:md5,c3f41cb637b4e448e41a2451ac6ea796", + "versions.yml:md5,c9f69ed90f9787ff5d98fda995824a89" ] } ], - "timestamp": "2026-06-04T01:16:16.700560685", + "timestamp": "2026-08-20T16:09:07.718191139", "meta": { "nf-test": "0.9.5", - "nextflow": "25.09.0" + "nextflow": "26.04.6" } } } \ No newline at end of file diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index a3e8e398d109..9a0e1b2279b2 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -1,34 +1,8 @@ params { - npcs = 8 - use_approx = true - algorithm = "kmeans" - n_clusters = 3 - dbscan_eps = 0.5 + npcs = 8 + use_approx = true + algorithm = "kmeans" + n_clusters = 3 + dbscan_eps = 0.5 dbscan_min_samples = 5 } - -process { - withName: 'BEAGLE5_BEAGLE' { - ext.when = false - } - - withName: 'PLINK2_VCF' { - ext.when = false - } - - withName: 'PLINK2_PCA' { - ext.when = false - } - - withName: 'CUSTOM_PCACLUSTERING' { - ext.when = false - } - - withName: 'CUSTOM_CLUSTERMETRICS' { - ext.when = false - } - - withName: 'CUSTOM_CLUSTERVISUALIZATION' { - ext.when = false - } -} From 6400b7bbf8451e4f41010463ae874fd6de1c45df Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 20 Aug 2026 16:57:42 +0200 Subject: [PATCH 11/34] fix: update versions emit pattern in custom clustering modules --- modules/nf-core/custom/clustermetrics/main.nf | 2 +- modules/nf-core/custom/clustermetrics/meta.yml | 15 +++++++-------- .../nf-core/custom/clustervisualization/main.nf | 2 +- .../nf-core/custom/clustervisualization/meta.yml | 10 +++++----- modules/nf-core/custom/pcaclustering/main.nf | 2 +- modules/nf-core/custom/pcaclustering/meta.yml | 11 +++++------ 6 files changed, 20 insertions(+), 22 deletions(-) diff --git a/modules/nf-core/custom/clustermetrics/main.nf b/modules/nf-core/custom/clustermetrics/main.nf index de01489438ad..2eb096b08f40 100644 --- a/modules/nf-core/custom/clustermetrics/main.nf +++ b/modules/nf-core/custom/clustermetrics/main.nf @@ -15,7 +15,7 @@ process CUSTOM_CLUSTERMETRICS { tuple val(meta), path("*.k_sweep.csv") , emit: k_sweep tuple val(meta), path("*.selected.json"), emit: selected tuple val(meta), path("*.png") , emit: plots, optional: true - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions, topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/custom/clustermetrics/meta.yml b/modules/nf-core/custom/clustermetrics/meta.yml index e364f55b6f75..92a1bea0d37b 100644 --- a/modules/nf-core/custom/clustermetrics/meta.yml +++ b/modules/nf-core/custom/clustermetrics/meta.yml @@ -1,6 +1,5 @@ name: "CUSTOM_CLUSTERMETRICS" -description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, - Davies-Bouldin) and performs k-sweep analysis" +description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, Davies-Bouldin) and performs k-sweep analysis" keywords: - clustering - metrics @@ -29,7 +28,7 @@ input: column per numeric feature (e.g. PCA scores). pattern: "*.tsv" ontologies: - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV - clusters: type: file description: | @@ -37,7 +36,7 @@ input: `cluster` columns. Label -1 is treated as DBSCAN noise. pattern: "*.csv" ontologies: - - edam: http://edamontology.org/format_3752 + - edam: http://edamontology.org/format_3752 # CSV output: metrics: - - meta: @@ -48,7 +47,7 @@ output: description: TSV with selected cluster quality metrics pattern: "*.metrics.tsv" ontologies: - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV k_sweep: - - meta: type: map @@ -58,7 +57,7 @@ output: description: CSV with metrics for different values of k pattern: "*.k_sweep.csv" ontologies: - - edam: http://edamontology.org/format_3752 + - edam: http://edamontology.org/format_3752 # CSV selected: - - meta: type: map @@ -68,7 +67,7 @@ output: description: JSON with the selected/best metrics pattern: "*.selected.json" ontologies: - - edam: http://edamontology.org/format_3464 + - edam: http://edamontology.org/format_3464 # JSON plots: - - meta: type: map @@ -84,7 +83,7 @@ output: description: File containing software versions pattern: "versions.yml" ontologies: - - edam: http://edamontology.org/format_3750 + - edam: http://edamontology.org/format_3750 # YAML topics: versions: - versions.yml: diff --git a/modules/nf-core/custom/clustervisualization/main.nf b/modules/nf-core/custom/clustervisualization/main.nf index 0b1346f20de4..393c5345c636 100644 --- a/modules/nf-core/custom/clustervisualization/main.nf +++ b/modules/nf-core/custom/clustervisualization/main.nf @@ -15,7 +15,7 @@ process CUSTOM_CLUSTERVISUALIZATION { tuple val(meta), path("*.tsne.tsv"), emit: tsne_tsv tuple val(meta), path("*.umap.png"), emit: umap_png, optional: true tuple val(meta), path("*.tsne.png"), emit: tsne_png, optional: true - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions, topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/custom/clustervisualization/meta.yml b/modules/nf-core/custom/clustervisualization/meta.yml index 62bed07814e5..2465d4be4fe1 100644 --- a/modules/nf-core/custom/clustervisualization/meta.yml +++ b/modules/nf-core/custom/clustervisualization/meta.yml @@ -33,7 +33,7 @@ input: column per numeric feature (e.g. PCA scores). pattern: "*.tsv" ontologies: - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV - clusters: type: file description: | @@ -41,7 +41,7 @@ input: `cluster` columns. Label -1 is treated as DBSCAN noise. pattern: "*.csv" ontologies: - - edam: http://edamontology.org/format_3752 + - edam: http://edamontology.org/format_3752 # CSV output: umap_tsv: - - meta: @@ -53,7 +53,7 @@ output: pattern: "*.umap.tsv" ontologies: - edam: "http://edamontology.org/operation_2432" - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV tsne_tsv: - - meta: type: map @@ -64,7 +64,7 @@ output: pattern: "*.tsne.tsv" ontologies: - edam: "http://edamontology.org/operation_2432" - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV umap_png: - - meta: type: map @@ -89,7 +89,7 @@ output: description: "Software versions used in the module" pattern: "versions.yml" ontologies: - - edam: http://edamontology.org/format_3750 + - edam: http://edamontology.org/format_3750 # YAML topics: versions: - versions.yml: diff --git a/modules/nf-core/custom/pcaclustering/main.nf b/modules/nf-core/custom/pcaclustering/main.nf index 4a4be41f876b..b14414f28f75 100644 --- a/modules/nf-core/custom/pcaclustering/main.nf +++ b/modules/nf-core/custom/pcaclustering/main.nf @@ -17,7 +17,7 @@ process CUSTOM_PCACLUSTERING { output: tuple val(meta), path("*.clusters.csv") , emit: clusters tuple val(meta), path("*.clustering_info.json"), emit: info - path "versions.yml" , emit: versions + path "versions.yml" , emit: versions, topic: versions when: task.ext.when == null || task.ext.when diff --git a/modules/nf-core/custom/pcaclustering/meta.yml b/modules/nf-core/custom/pcaclustering/meta.yml index 72a1cefbaa11..13f612c8eb9e 100644 --- a/modules/nf-core/custom/pcaclustering/meta.yml +++ b/modules/nf-core/custom/pcaclustering/meta.yml @@ -1,6 +1,5 @@ name: "CUSTOM_PCACLUSTERING" -description: "Performs KMeans or DBSCAN clustering on a sample-by-feature numeric - matrix (e.g. principal components, embeddings)" +description: "Performs KMeans or DBSCAN clustering on a sample-by-feature numeric matrix (e.g. principal components, embeddings)" keywords: - clustering - kmeans @@ -29,7 +28,7 @@ input: components with `FID` dropped, scikit-learn embeddings, etc.). pattern: "*.{tsv,txt}" ontologies: - - edam: http://edamontology.org/format_3475 + - edam: http://edamontology.org/format_3475 # TSV - algorithm: type: string description: Clustering algorithm to use (kmeans or dbscan) @@ -52,7 +51,7 @@ output: description: CSV file with sample_id and assigned cluster pattern: "*.clusters.csv" ontologies: - - edam: http://edamontology.org/format_3752 + - edam: http://edamontology.org/format_3752 # CSV info: - - meta: type: map @@ -62,14 +61,14 @@ output: description: JSON file with clustering parameters and statistics pattern: "*.clustering_info.json" ontologies: - - edam: http://edamontology.org/format_3464 + - edam: http://edamontology.org/format_3464 # JSON versions: - "versions.yml": type: file description: File containing software versions pattern: "versions.yml" ontologies: - - edam: http://edamontology.org/format_3750 + - edam: http://edamontology.org/format_3750 # YAML topics: versions: - versions.yml: From 1fba4a3457120ffdf6d1b36d0d9d6f9237ff4851 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 20 Aug 2026 17:16:02 +0200 Subject: [PATCH 12/34] fix: ensure main.nf ends with single newline --- subworkflows/nf-core/snpclustering/main.nf | 1 - 1 file changed, 1 deletion(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 62eabbf5c7f9..0640bfaf386e 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -161,4 +161,3 @@ process EIGENVEC_TO_TSV { END_VERSIONS """ } - From 1f98f397512ff6d54b1acf71a21eb017b9030d54 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Tue, 25 Aug 2026 11:56:38 +0200 Subject: [PATCH 13/34] Replace inline EIGENVEC_TO_TSV process with GAWK module in snpclustering --- .../snpclustering/awk/eigenvec_to_tsv.awk | 9 +++ subworkflows/nf-core/snpclustering/main.nf | 71 +++++++------------ .../snpclustering/tests/main.nf.test.snap | 28 ++++++-- .../snpclustering/tests/nextflow.config | 7 ++ 4 files changed, 66 insertions(+), 49 deletions(-) create mode 100644 subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk diff --git a/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk b/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk new file mode 100644 index 000000000000..c84a0c3e9160 --- /dev/null +++ b/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk @@ -0,0 +1,9 @@ +NR==1 { + sub(/^#/, "") + if ($1 == "IID") { + $1 = "sample_id" + } + print + next +} +{ print } diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 0640bfaf386e..ae7c89b321b7 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,9 +1,10 @@ -include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' -include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' -include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' -include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' -include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' -include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' +include { GAWK as GAWK_EIGENVEC_TO_TSV } from '../../../modules/nf-core/gawk/main' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' +include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' workflow SNPCLUSTERING { @@ -22,6 +23,12 @@ workflow SNPCLUSTERING { main: ch_versions = Channel.empty() + // Bundled awk program (see ./awk/eigenvec_to_tsv.awk); shipped alongside + // the subworkflow and passed to GAWK_EIGENVEC_TO_TSV as `program_file` so + // the awk source stays readable as awk rather than as an escaped Groovy + // string. + ch_eigenvec_to_tsv_awk = file("${moduleDir}/awk/eigenvec_to_tsv.awk", checkIfExists: true) + /* * Build BEAGLE input tuple: * tuple val(meta), path(vcf), path(vcf_index), path(refpanel), path(refpanel_index), @@ -65,16 +72,24 @@ workflow SNPCLUSTERING { ch_versions = ch_versions.mix(PLINK2_PCA.out.versions_plink2) /* - * Convert .eigenvec to TSV for downstream clustering modules + * Convert .eigenvec to a tab-separated file for downstream clustering + * modules: strips the leading '#' from the header and renames the IID + * column to sample_id. Output keeps its default naming (.eigenvec) + * — only the tab-separated content matters downstream, not the extension. */ - EIGENVEC_TO_TSV(PLINK2_PCA.out.evecfile) + GAWK_EIGENVEC_TO_TSV( + PLINK2_PCA.out.evecfile, + ch_eigenvec_to_tsv_awk, + false + ) + ch_versions = ch_versions.mix(GAWK_EIGENVEC_TO_TSV.out.versions_gawk) /* * PCA clustering * CUSTOM_PCACLUSTERING(tsv, algorithm, n_clusters, dbscan_eps, dbscan_min_samples) */ CUSTOM_PCACLUSTERING( - EIGENVEC_TO_TSV.out.tsv, + GAWK_EIGENVEC_TO_TSV.out.output, algorithm, n_clusters, dbscan_eps, @@ -86,7 +101,7 @@ workflow SNPCLUSTERING { * Metrics and visualization both expect one tuple input channel * built from: meta + tsv + cluster assignments */ - ch_cluster_analysis_input = EIGENVEC_TO_TSV.out.tsv + ch_cluster_analysis_input = GAWK_EIGENVEC_TO_TSV.out.output .join(CUSTOM_PCACLUSTERING.out.clusters) .map { meta, tsv, clusters -> tuple(meta, tsv, clusters) @@ -110,7 +125,7 @@ workflow SNPCLUSTERING { evfile = PLINK2_PCA.out.evfile pca_log = PLINK2_PCA.out.logfile - tsv = EIGENVEC_TO_TSV.out.tsv + tsv = GAWK_EIGENVEC_TO_TSV.out.output clusters = CUSTOM_PCACLUSTERING.out.clusters cluster_info = CUSTOM_PCACLUSTERING.out.info @@ -127,37 +142,3 @@ workflow SNPCLUSTERING { versions = ch_versions } - -process EIGENVEC_TO_TSV { - - tag "${meta.id}" - label 'process_single' - - input: - tuple val(meta), path(eigenvec) - - output: - tuple val(meta), path("${meta.id}.tsv"), emit: tsv - path "versions.yml", emit: versions - - script: - """ - awk ' - NR==1 { - sub(/^#/, "") - if (\$1 == "IID") { - \$1="sample_id" - } - print - next - } - { - print - }' OFS='\\t' ${eigenvec} > ${meta.id}.tsv - - cat <<-END_VERSIONS > versions.yml - "${task.process}": - gawk: \$(awk --version | head -1 | sed 's/GNU Awk //;s/,.*//') - END_VERSIONS - """ -} diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 7bb0440847d8..4bd2b72cd485 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -31,6 +31,11 @@ "beagle", "5.5" ], + [ + "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", + "gawk", + "5.3.1" + ], [ "SNPCLUSTERING:PLINK2_PCA", "plink2", @@ -46,7 +51,7 @@ "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" ] ], - "timestamp": "2026-08-20T16:08:17.636962775", + "timestamp": "2026-08-25T11:53:17.934676935", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -84,6 +89,11 @@ "beagle", "5.5" ], + [ + "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", + "gawk", + "5.3.1" + ], [ "SNPCLUSTERING:PLINK2_PCA", "plink2", @@ -99,7 +109,7 @@ "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" ] ], - "timestamp": "2026-08-20T16:08:55.94750336", + "timestamp": "2026-08-25T11:53:57.926394897", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -271,6 +281,11 @@ "beagle", "5.5" ], + [ + "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", + "gawk", + "5.3.1" + ], [ "SNPCLUSTERING:PLINK2_PCA", "plink2", @@ -287,7 +302,7 @@ ] } ], - "timestamp": "2026-08-20T16:09:17.538676351", + "timestamp": "2026-08-25T11:54:24.140132513", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -459,6 +474,11 @@ "beagle", "5.5" ], + [ + "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", + "gawk", + "5.3.1" + ], [ "SNPCLUSTERING:PLINK2_PCA", "plink2", @@ -475,7 +495,7 @@ ] } ], - "timestamp": "2026-08-20T16:09:07.718191139", + "timestamp": "2026-08-25T11:54:10.985447937", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 9a0e1b2279b2..93f8ed077be8 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -6,3 +6,10 @@ params { dbscan_eps = 0.5 dbscan_min_samples = 5 } + +process { + withName: GAWK_EIGENVEC_TO_TSV { + ext.args = "-v OFS='\\t'" + ext.suffix = "tsv" + } +} From 6a7677fcb50a4bd5005451af856b4aab88575ba9 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 15:33:08 +0200 Subject: [PATCH 14/34] fix(snpclustering): use topic versions and snapshot stable md5s --- subworkflows/nf-core/snpclustering/main.nf | 19 - subworkflows/nf-core/snpclustering/meta.yml | 321 +++++++----- .../nf-core/snpclustering/tests/main.nf.test | 71 ++- .../snpclustering/tests/main.nf.test.snap | 470 ++++++++++++------ 4 files changed, 540 insertions(+), 341 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index ae7c89b321b7..3b8fb56bf670 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -7,7 +7,6 @@ include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/ include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' workflow SNPCLUSTERING { - take: vcf_ch refpanel_ch @@ -21,8 +20,6 @@ workflow SNPCLUSTERING { dbscan_min_samples main: - ch_versions = Channel.empty() - // Bundled awk program (see ./awk/eigenvec_to_tsv.awk); shipped alongside // the subworkflow and passed to GAWK_EIGENVEC_TO_TSV as `program_file` so // the awk source stays readable as awk rather than as an escaped Groovy @@ -49,13 +46,11 @@ workflow SNPCLUSTERING { } BEAGLE5_BEAGLE(ch_beagle_input) - ch_versions = ch_versions.mix(BEAGLE5_BEAGLE.out.versions_beagle) /* * Convert imputed VCF to PLINK2 pfiles */ PLINK2_VCF(BEAGLE5_BEAGLE.out.vcf) - ch_versions = ch_versions.mix(PLINK2_VCF.out.versions_plink2) /* * PLINK2_PCA expects: @@ -69,7 +64,6 @@ workflow SNPCLUSTERING { } PLINK2_PCA(ch_plink_pca_input) - ch_versions = ch_versions.mix(PLINK2_PCA.out.versions_plink2) /* * Convert .eigenvec to a tab-separated file for downstream clustering @@ -82,7 +76,6 @@ workflow SNPCLUSTERING { ch_eigenvec_to_tsv_awk, false ) - ch_versions = ch_versions.mix(GAWK_EIGENVEC_TO_TSV.out.versions_gawk) /* * PCA clustering @@ -95,7 +88,6 @@ workflow SNPCLUSTERING { dbscan_eps, dbscan_min_samples ) - ch_versions = ch_versions.mix(CUSTOM_PCACLUSTERING.out.versions) /* * Metrics and visualization both expect one tuple input channel @@ -108,37 +100,26 @@ workflow SNPCLUSTERING { } CUSTOM_CLUSTERMETRICS(ch_cluster_analysis_input) - ch_versions = ch_versions.mix(CUSTOM_CLUSTERMETRICS.out.versions) - CUSTOM_CLUSTERVISUALIZATION(ch_cluster_analysis_input) - ch_versions = ch_versions.mix(CUSTOM_CLUSTERVISUALIZATION.out.versions) emit: imputed_vcf = BEAGLE5_BEAGLE.out.vcf beagle_log = BEAGLE5_BEAGLE.out.log - pgen = PLINK2_VCF.out.pgen pvar = PLINK2_VCF.out.pvar psam = PLINK2_VCF.out.psam - evecfile = PLINK2_PCA.out.evecfile evfile = PLINK2_PCA.out.evfile pca_log = PLINK2_PCA.out.logfile - tsv = GAWK_EIGENVEC_TO_TSV.out.output - clusters = CUSTOM_PCACLUSTERING.out.clusters cluster_info = CUSTOM_PCACLUSTERING.out.info - metrics = CUSTOM_CLUSTERMETRICS.out.metrics k_sweep = CUSTOM_CLUSTERMETRICS.out.k_sweep selected = CUSTOM_CLUSTERMETRICS.out.selected metric_plots = CUSTOM_CLUSTERMETRICS.out.plots - umap_tsv = CUSTOM_CLUSTERVISUALIZATION.out.umap_tsv tsne_tsv = CUSTOM_CLUSTERVISUALIZATION.out.tsne_tsv umap_png = CUSTOM_CLUSTERVISUALIZATION.out.umap_png tsne_png = CUSTOM_CLUSTERVISUALIZATION.out.tsne_png - - versions = ch_versions } diff --git a/subworkflows/nf-core/snpclustering/meta.yml b/subworkflows/nf-core/snpclustering/meta.yml index 25098093fb72..171a9f1e6e12 100644 --- a/subworkflows/nf-core/snpclustering/meta.yml +++ b/subworkflows/nf-core/snpclustering/meta.yml @@ -13,158 +13,207 @@ keywords: - beagle - imputation - population genetics - components: - beagle5/beagle - plink2/vcf - plink2/pca + - gawk - custom/pcaclustering - custom/clustermetrics - custom/clustervisualization - input: - - meta: - type: map - description: Groovy map containing sample metadata. - vcf: - type: file - description: Genotyped VCF file. - pattern: "*.{vcf,vcf.gz}" - vcf_index: - type: file - description: Index for the VCF file. - pattern: "*.{tbi,csi}" - - - meta: - type: map - description: Groovy map for the optional reference panel. - refpanel: - type: file - description: Optional BEAGLE reference panel VCF. - pattern: "*.{vcf,vcf.gz}" - refpanel_index: - type: file - description: Optional index for the reference panel. - pattern: "*.{tbi,csi}" - - - meta: - type: map - description: Groovy map for the optional genetic map. - genmap: - type: file - description: Optional genetic map file for BEAGLE. - pattern: "*.map" - + - vcf_ch: + description: Channel with genotyped VCF and index + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - vcf: + type: file + description: Genotyped VCF file + pattern: "*.{vcf,vcf.gz}" + - vcf_index: + type: file + description: Index for the VCF file + pattern: "*.{tbi,csi}" + - refpanel_ch: + description: Optional BEAGLE reference panel channel + - genmap_ch: + description: Optional BEAGLE genetic map channel - region: - type: val - description: Optional genomic region passed to BEAGLE. - + type: string + description: Optional genomic region passed to BEAGLE - npcs: - type: val - description: Number of principal components to compute. - + type: integer + description: Number of principal components to compute - use_approx: - type: val - description: Whether to use approximate PCA. - + type: boolean + description: Whether to use approximate PCA - algorithm: - type: val - description: Clustering algorithm to use. - + type: string + description: Clustering algorithm to use (kmeans or dbscan) - n_clusters: - type: val - description: Number of clusters for kmeans or similar methods. - + type: integer + description: Number of clusters for k-means - dbscan_eps: - type: val - description: DBSCAN eps parameter. - + type: float + description: DBSCAN eps parameter - dbscan_min_samples: - type: val - description: DBSCAN minimum samples parameter. - + type: integer + description: DBSCAN minimum samples parameter output: - - imputed_vcf: - type: map - description: Groovy map of sample metadata. - vcf: - type: file - description: Imputed VCF from BEAGLE5. - pattern: "*.{vcf,vcf.gz}" - - - pfiles: - type: map - description: Groovy map of sample metadata. - pgen: - type: file - description: PLINK2 pgen file. - pattern: "*.pgen" - psam: - type: file - description: PLINK2 psam file. - pattern: "*.psam" - pvar: - type: file - description: PLINK2 pvar file. - pattern: "*.pvar" - - - pca: - type: map - description: Groovy map of sample metadata. - eigenvec: - type: file - description: PCA eigenvectors from PLINK2. - pattern: "*.eigenvec" - eigenval: - type: file - description: PCA eigenvalues from PLINK2. - pattern: "*.eigenval" - - - eigenvec_tsv: - type: map - description: Groovy map of sample metadata. - tsv: - type: file - description: Tidy TSV converted from eigenvec by awk. - pattern: "*.tsv" - - - cluster_labels: - type: map - description: Groovy map of sample metadata. - clusters: - type: file - description: Cluster labels output from PCA clustering. - pattern: "*.tsv" - - - cluster_info: - type: map - description: Groovy map of sample metadata. - info: - type: file - description: Additional cluster info output. - pattern: "*.tsv" - - - cluster_metrics: - type: map - description: Groovy map of sample metadata. - metrics: - type: file - description: Cluster quality metrics output. - pattern: "*.tsv" - - - cluster_plots: - type: map - description: Groovy map of sample metadata. - plots: - type: file - description: Cluster visualization outputs. - pattern: "*.{png,pdf,html}" - - - versions: - type: file - description: Software versions collected from the subworkflow. - pattern: "versions.yml" - + imputed_vcf: + - meta: + type: map + description: Groovy map containing sample metadata + - vcf: + type: file + description: Imputed VCF from BEAGLE5 + pattern: "*.{vcf,vcf.gz}" + beagle_log: + - meta: + type: map + description: Groovy map containing sample metadata + - log: + type: file + description: BEAGLE5 log file + pattern: "*.log" + pgen: + - meta: + type: map + description: Groovy map containing sample metadata + - pgen: + type: file + description: PLINK2 pgen file + pattern: "*.pgen" + pvar: + - meta: + type: map + description: Groovy map containing sample metadata + - pvar: + type: file + description: PLINK2 pvar file + pattern: "*.pvar" + psam: + - meta: + type: map + description: Groovy map containing sample metadata + - psam: + type: file + description: PLINK2 psam file + pattern: "*.psam" + evecfile: + - meta: + type: map + description: Groovy map containing sample metadata + - eigenvec: + type: file + description: PCA eigenvectors from PLINK2 + pattern: "*.eigenvec" + evfile: + - meta: + type: map + description: Groovy map containing sample metadata + - eigenval: + type: file + description: PCA eigenvalues from PLINK2 + pattern: "*.eigenval" + pca_log: + - meta: + type: map + description: Groovy map containing sample metadata + - log: + type: file + description: PLINK2 PCA log file + pattern: "*.log" + tsv: + - meta: + type: map + description: Groovy map containing sample metadata + - tsv: + type: file + description: TSV converted from eigenvec by gawk + pattern: "*.eigenvec" + clusters: + - meta: + type: map + description: Groovy map containing sample metadata + - clusters: + type: file + description: Cluster assignments + pattern: "*.clusters.csv" + cluster_info: + - meta: + type: map + description: Groovy map containing sample metadata + - info: + type: file + description: Clustering parameters and statistics + pattern: "*.clustering_info.json" + metrics: + - meta: + type: map + description: Groovy map containing sample metadata + - metrics: + type: file + description: Cluster quality metrics + pattern: "*.metrics.tsv" + k_sweep: + - meta: + type: map + description: Groovy map containing sample metadata + - k_sweep: + type: file + description: K sweep scores + pattern: "*.k_sweep.csv" + selected: + - meta: + type: map + description: Groovy map containing sample metadata + - selected: + type: file + description: Selected clustering parameters + pattern: "*.selected.json" + metric_plots: + - meta: + type: map + description: Groovy map containing sample metadata + - plots: + type: file + description: Cluster metric plots + pattern: "*.png" + umap_tsv: + - meta: + type: map + description: Groovy map containing sample metadata + - umap_tsv: + type: file + description: UMAP coordinates per sample + pattern: "*.umap.tsv" + tsne_tsv: + - meta: + type: map + description: Groovy map containing sample metadata + - tsne_tsv: + type: file + description: t-SNE coordinates per sample + pattern: "*.tsne.tsv" + umap_png: + - meta: + type: map + description: Groovy map containing sample metadata + - umap_png: + type: file + description: UMAP plot coloured by cluster + pattern: "*.umap.png" + tsne_png: + - meta: + type: map + description: Groovy map containing sample metadata + - tsne_png: + type: file + description: t-SNE plot coloured by cluster + pattern: "*.tsne.png" authors: - "@dbaku42" maintainers: diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 8a2af307932e..b1cb5388ec29 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -12,6 +12,7 @@ nextflow_workflow { tag "plink2/vcf" tag "plink2/pca" tag "beagle5/beagle" + tag "gawk" tag "custom/pcaclustering" tag "custom/clustermetrics" tag "custom/clustervisualization" @@ -41,22 +42,19 @@ nextflow_workflow { } then { - assert workflow.success - assert snapshot( - workflow.out.clusters.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.cluster_info.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.metrics.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.versions - ).match() + assertAll( + { assert workflow.success }, + { assert snapshot( + sanitizeOutput( + workflow.out, + unstableKeys: [ + "umap_png", + "tsne_png", + "metric_plots" + ] + ) + ).match() } + ) } } @@ -83,22 +81,19 @@ nextflow_workflow { } then { - assert workflow.success - assert snapshot( - workflow.out.clusters.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.cluster_info.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.metrics.collect { [ - it[0], - path(it[1]).getFileName().toString() - ]}, - workflow.out.versions - ).match() + assertAll( + { assert workflow.success }, + { assert snapshot( + sanitizeOutput( + workflow.out, + unstableKeys: [ + "umap_png", + "tsne_png", + "metric_plots" + ] + ) + ).match() } + ) } } @@ -126,8 +121,10 @@ nextflow_workflow { } then { - assert workflow.success - assert snapshot(sanitizeOutput(workflow.out)).match() + assertAll( + { assert workflow.success }, + { assert snapshot(sanitizeOutput(workflow.out)).match() } + ) } } @@ -155,8 +152,10 @@ nextflow_workflow { } then { - assert workflow.success - assert snapshot(sanitizeOutput(workflow.out)).match() + assertAll( + { assert workflow.success }, + { assert snapshot(sanitizeOutput(workflow.out)).match() } + ) } } } diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 4bd2b72cd485..f6d1862a39ad 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -1,57 +1,167 @@ { "homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters": { "content": [ - [ - [ - { - "id": "test" - }, - "test.clusters.csv" - ] - ], - [ - [ - { - "id": "test" - }, - "test.clustering_info.json" - ] - ], - [ - [ - { - "id": "test" - }, - "test.metrics.tsv" + { + "beagle_log": [ + [ + { + "id": "test" + }, + "test.bglout.log:md5,00f899a6db795f7d4bd79b5961ec7d10" + ] + ], + "cluster_info": [ + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,f4f73c989de9e52384b786ff6ac040e5" + ] + ], + "clusters": [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,be0a1b1d684974d5eff60be2c46ceedf" + ] + ], + "evecfile": [ + [ + { + "id": "test" + }, + "test.eigenvec:md5,655e0a4283a1f9d9f13c7dbe0e6da1d9" + ] + ], + "evfile": [ + [ + { + "id": "test" + }, + "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" + ] + ], + "imputed_vcf": [ + [ + { + "id": "test" + }, + "test.bglout.vcf.gz:md5,ef94995e50ba034aeb8119ad15fb6ccc" + ] + ], + "k_sweep": [ + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,113fa7245a810786ec404769f5ba5698" + ] + ], + "metric_plots": [ + [ + { + "id": "test" + }, + [ + "test.calinski_harabasz.png", + "test.davies_bouldin.png", + "test.elbow.png", + "test.silhouette.png" + ] + ] + ], + "metrics": [ + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,9f203e3c741435f1317dcb4ab44c43c3" + ] + ], + "pca_log": [ + [ + { + "id": "test" + }, + "test.log:md5,3d794ffb37ae49b03dbc8a06a408825f" + ] + ], + "pgen": [ + [ + { + "id": "test" + }, + "test.pgen:md5,b183c72cbf75758521bd42a21a02e882" + ] + ], + "psam": [ + [ + { + "id": "test" + }, + "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" + ] + ], + "pvar": [ + [ + { + "id": "test" + }, + "test.pvar:md5,9c89b734017db356ccaa0190ab88c199" + ] + ], + "selected": [ + [ + { + "id": "test" + }, + "test.selected.json:md5,ddbef02b869e51b8eaadceebd9ebd064" + ] + ], + "tsne_png": [ + [ + { + "id": "test" + }, + "test.tsne.png" + ] + ], + "tsne_tsv": [ + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,20f545d08f1175bfe1825b3932833b8d" + ] + ], + "tsv": [ + [ + { + "id": "test" + }, + "test.tsv:md5,94681fa5806fbf33fa5c5041c3c29f29" + ] + ], + "umap_png": [ + [ + { + "id": "test" + }, + "test.umap.png" + ] + ], + "umap_tsv": [ + [ + { + "id": "test" + }, + "test.umap.tsv:md5,a7d209592da80adf142c5f1c2bce9ea2" + ] ] - ], - [ - [ - "SNPCLUSTERING:BEAGLE5_BEAGLE", - "beagle", - "5.5" - ], - [ - "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", - "gawk", - "5.3.1" - ], - [ - "SNPCLUSTERING:PLINK2_PCA", - "plink2", - "2.00a5.10LM" - ], - [ - "SNPCLUSTERING:PLINK2_VCF", - "plink2", - "2.00a2.3LM" - ], - "versions.yml:md5,088adb486c0aa9f234effd310576eaf7", - "versions.yml:md5,5e7ab136010cdcda51f91b12562b28c7", - "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" - ] + } ], - "timestamp": "2026-08-25T11:53:17.934676935", + "timestamp": "2026-08-26T15:28:38.960190054", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -59,57 +169,167 @@ }, "homo_sapiens - 1000GP.chr22.vcf.gz - dbscan": { "content": [ - [ - [ - { - "id": "test" - }, - "test.clusters.csv" - ] - ], - [ - [ - { - "id": "test" - }, - "test.clustering_info.json" - ] - ], - [ - [ - { - "id": "test" - }, - "test.metrics.tsv" + { + "beagle_log": [ + [ + { + "id": "test" + }, + "test.bglout.log:md5,00f899a6db795f7d4bd79b5961ec7d10" + ] + ], + "cluster_info": [ + [ + { + "id": "test" + }, + "test.clustering_info.json:md5,fa9a2eff36d06fe360a59f4c3cd63393" + ] + ], + "clusters": [ + [ + { + "id": "test" + }, + "test.clusters.csv:md5,c119dc1c716d66ceebc220219b02a535" + ] + ], + "evecfile": [ + [ + { + "id": "test" + }, + "test.eigenvec:md5,cb7d3501265fb6e681f9a66552faff42" + ] + ], + "evfile": [ + [ + { + "id": "test" + }, + "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" + ] + ], + "imputed_vcf": [ + [ + { + "id": "test" + }, + "test.bglout.vcf.gz:md5,ef94995e50ba034aeb8119ad15fb6ccc" + ] + ], + "k_sweep": [ + [ + { + "id": "test" + }, + "test.k_sweep.csv:md5,d75662c7c614f39e294601653c02de8d" + ] + ], + "metric_plots": [ + [ + { + "id": "test" + }, + [ + "test.calinski_harabasz.png", + "test.davies_bouldin.png", + "test.elbow.png", + "test.silhouette.png" + ] + ] + ], + "metrics": [ + [ + { + "id": "test" + }, + "test.metrics.tsv:md5,d3f1a0d467c21875176c6f8c691b8d71" + ] + ], + "pca_log": [ + [ + { + "id": "test" + }, + "test.log:md5,65d20c952985adb5fa7229a5d0d4fc0c" + ] + ], + "pgen": [ + [ + { + "id": "test" + }, + "test.pgen:md5,b183c72cbf75758521bd42a21a02e882" + ] + ], + "psam": [ + [ + { + "id": "test" + }, + "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" + ] + ], + "pvar": [ + [ + { + "id": "test" + }, + "test.pvar:md5,9c89b734017db356ccaa0190ab88c199" + ] + ], + "selected": [ + [ + { + "id": "test" + }, + "test.selected.json:md5,138b2bae0f67f913dcead80f8202d9af" + ] + ], + "tsne_png": [ + [ + { + "id": "test" + }, + "test.tsne.png" + ] + ], + "tsne_tsv": [ + [ + { + "id": "test" + }, + "test.tsne.tsv:md5,f4952012bc87993ca78429d4a823edb1" + ] + ], + "tsv": [ + [ + { + "id": "test" + }, + "test.tsv:md5,8009b5a5210c1dae667f40c40b450035" + ] + ], + "umap_png": [ + [ + { + "id": "test" + }, + "test.umap.png" + ] + ], + "umap_tsv": [ + [ + { + "id": "test" + }, + "test.umap.tsv:md5,26c791bc4d38dff6f979582611dbc27d" + ] ] - ], - [ - [ - "SNPCLUSTERING:BEAGLE5_BEAGLE", - "beagle", - "5.5" - ], - [ - "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", - "gawk", - "5.3.1" - ], - [ - "SNPCLUSTERING:PLINK2_PCA", - "plink2", - "2.00a5.10LM" - ], - [ - "SNPCLUSTERING:PLINK2_VCF", - "plink2", - "2.00a2.3LM" - ], - "versions.yml:md5,088adb486c0aa9f234effd310576eaf7", - "versions.yml:md5,5e7ab136010cdcda51f91b12562b28c7", - "versions.yml:md5,7406d1551e405376f2e1be27dd888f0f" - ] + } ], - "timestamp": "2026-08-25T11:53:57.926394897", + "timestamp": "2026-08-26T15:29:18.988004598", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -274,35 +494,10 @@ }, "test.umap.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "versions": [ - [ - "SNPCLUSTERING:BEAGLE5_BEAGLE", - "beagle", - "5.5" - ], - [ - "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", - "gawk", - "5.3.1" - ], - [ - "SNPCLUSTERING:PLINK2_PCA", - "plink2", - "2.00a5.10LM" - ], - [ - "SNPCLUSTERING:PLINK2_VCF", - "plink2", - "2.00a2.3LM" - ], - "versions.yml:md5,a5331a41c30b50872e8f9dc902b1fbdb", - "versions.yml:md5,c3f41cb637b4e448e41a2451ac6ea796", - "versions.yml:md5,c9f69ed90f9787ff5d98fda995824a89" ] } ], - "timestamp": "2026-08-25T11:54:24.140132513", + "timestamp": "2026-08-26T15:29:43.011468684", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -467,35 +662,10 @@ }, "test.umap.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" ] - ], - "versions": [ - [ - "SNPCLUSTERING:BEAGLE5_BEAGLE", - "beagle", - "5.5" - ], - [ - "SNPCLUSTERING:GAWK_EIGENVEC_TO_TSV", - "gawk", - "5.3.1" - ], - [ - "SNPCLUSTERING:PLINK2_PCA", - "plink2", - "2.00a5.10LM" - ], - [ - "SNPCLUSTERING:PLINK2_VCF", - "plink2", - "2.00a2.3LM" - ], - "versions.yml:md5,a5331a41c30b50872e8f9dc902b1fbdb", - "versions.yml:md5,c3f41cb637b4e448e41a2451ac6ea796", - "versions.yml:md5,c9f69ed90f9787ff5d98fda995824a89" ] } ], - "timestamp": "2026-08-25T11:54:10.985447937", + "timestamp": "2026-08-26T15:29:31.961527536", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From e29c96618404a05d30bb9e3277477424bef3eed9 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 15:57:26 +0200 Subject: [PATCH 15/34] Fixing linting and unstable md5 --- subworkflows/nf-core/snpclustering/meta.yml | 342 ++++++++++-------- .../nf-core/snpclustering/tests/main.nf.test | 39 +- .../snpclustering/tests/main.nf.test.snap | 58 +-- 3 files changed, 238 insertions(+), 201 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/meta.yml b/subworkflows/nf-core/snpclustering/meta.yml index 171a9f1e6e12..1d57e69d8915 100644 --- a/subworkflows/nf-core/snpclustering/meta.yml +++ b/subworkflows/nf-core/snpclustering/meta.yml @@ -62,158 +62,196 @@ input: type: integer description: DBSCAN minimum samples parameter output: - imputed_vcf: - - meta: - type: map - description: Groovy map containing sample metadata - - vcf: - type: file - description: Imputed VCF from BEAGLE5 - pattern: "*.{vcf,vcf.gz}" - beagle_log: - - meta: - type: map - description: Groovy map containing sample metadata - - log: - type: file - description: BEAGLE5 log file - pattern: "*.log" - pgen: - - meta: - type: map - description: Groovy map containing sample metadata - - pgen: - type: file - description: PLINK2 pgen file - pattern: "*.pgen" - pvar: - - meta: - type: map - description: Groovy map containing sample metadata - - pvar: - type: file - description: PLINK2 pvar file - pattern: "*.pvar" - psam: - - meta: - type: map - description: Groovy map containing sample metadata - - psam: - type: file - description: PLINK2 psam file - pattern: "*.psam" - evecfile: - - meta: - type: map - description: Groovy map containing sample metadata - - eigenvec: - type: file - description: PCA eigenvectors from PLINK2 - pattern: "*.eigenvec" - evfile: - - meta: - type: map - description: Groovy map containing sample metadata - - eigenval: - type: file - description: PCA eigenvalues from PLINK2 - pattern: "*.eigenval" - pca_log: - - meta: - type: map - description: Groovy map containing sample metadata - - log: - type: file - description: PLINK2 PCA log file - pattern: "*.log" - tsv: - - meta: - type: map - description: Groovy map containing sample metadata - - tsv: - type: file - description: TSV converted from eigenvec by gawk - pattern: "*.eigenvec" - clusters: - - meta: - type: map - description: Groovy map containing sample metadata - - clusters: - type: file - description: Cluster assignments - pattern: "*.clusters.csv" - cluster_info: - - meta: - type: map - description: Groovy map containing sample metadata - - info: - type: file - description: Clustering parameters and statistics - pattern: "*.clustering_info.json" - metrics: - - meta: - type: map - description: Groovy map containing sample metadata - - metrics: - type: file - description: Cluster quality metrics - pattern: "*.metrics.tsv" - k_sweep: - - meta: - type: map - description: Groovy map containing sample metadata - - k_sweep: - type: file - description: K sweep scores - pattern: "*.k_sweep.csv" - selected: - - meta: - type: map - description: Groovy map containing sample metadata - - selected: - type: file - description: Selected clustering parameters - pattern: "*.selected.json" - metric_plots: - - meta: - type: map - description: Groovy map containing sample metadata - - plots: - type: file - description: Cluster metric plots - pattern: "*.png" - umap_tsv: - - meta: - type: map - description: Groovy map containing sample metadata - - umap_tsv: - type: file - description: UMAP coordinates per sample - pattern: "*.umap.tsv" - tsne_tsv: - - meta: - type: map - description: Groovy map containing sample metadata - - tsne_tsv: - type: file - description: t-SNE coordinates per sample - pattern: "*.tsne.tsv" - umap_png: - - meta: - type: map - description: Groovy map containing sample metadata - - umap_png: - type: file - description: UMAP plot coloured by cluster - pattern: "*.umap.png" - tsne_png: - - meta: - type: map - description: Groovy map containing sample metadata - - tsne_png: - type: file - description: t-SNE plot coloured by cluster - pattern: "*.tsne.png" + - imputed_vcf: + description: Imputed VCF from BEAGLE5 + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - vcf: + type: file + description: Imputed VCF from BEAGLE5 + pattern: "*.{vcf,vcf.gz}" + - beagle_log: + description: BEAGLE5 log file + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - log: + type: file + description: BEAGLE5 log file + pattern: "*.log" + - pgen: + description: PLINK2 pgen file + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - pgen: + type: file + description: PLINK2 pgen file + pattern: "*.pgen" + - pvar: + description: PLINK2 pvar file + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - pvar: + type: file + description: PLINK2 pvar file + pattern: "*.pvar" + - psam: + description: PLINK2 psam file + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - psam: + type: file + description: PLINK2 psam file + pattern: "*.psam" + - evecfile: + description: PCA eigenvectors from PLINK2 + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - eigenvec: + type: file + description: PCA eigenvectors from PLINK2 + pattern: "*.eigenvec" + - evfile: + description: PCA eigenvalues from PLINK2 + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - eigenval: + type: file + description: PCA eigenvalues from PLINK2 + pattern: "*.eigenval" + - pca_log: + description: PLINK2 PCA log file + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - log: + type: file + description: PLINK2 PCA log file + pattern: "*.log" + - tsv: + description: TSV converted from eigenvec by gawk + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - tsv: + type: file + description: TSV converted from eigenvec by gawk + pattern: "*.eigenvec" + - clusters: + description: Cluster assignments + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - clusters: + type: file + description: Cluster assignments + pattern: "*.clusters.csv" + - cluster_info: + description: Clustering parameters and statistics + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - info: + type: file + description: Clustering parameters and statistics + pattern: "*.clustering_info.json" + - metrics: + description: Cluster quality metrics + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - metrics: + type: file + description: Cluster quality metrics + pattern: "*.metrics.tsv" + - k_sweep: + description: K sweep scores + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - k_sweep: + type: file + description: K sweep scores + pattern: "*.k_sweep.csv" + - selected: + description: Selected clustering parameters + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - selected: + type: file + description: Selected clustering parameters + pattern: "*.selected.json" + - metric_plots: + description: Cluster metric plots + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - plots: + type: file + description: Cluster metric plots + pattern: "*.png" + - umap_tsv: + description: UMAP coordinates per sample + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - umap_tsv: + type: file + description: UMAP coordinates per sample + pattern: "*.umap.tsv" + - tsne_tsv: + description: t-SNE coordinates per sample + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - tsne_tsv: + type: file + description: t-SNE coordinates per sample + pattern: "*.tsne.tsv" + - umap_png: + description: UMAP plot coloured by cluster + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - umap_png: + type: file + description: UMAP plot coloured by cluster + pattern: "*.umap.png" + - tsne_png: + description: t-SNE plot coloured by cluster + structure: + - meta: + type: map + description: Groovy map containing sample metadata + - tsne_png: + type: file + description: t-SNE plot coloured by cluster + pattern: "*.tsne.png" authors: - "@dbaku42" maintainers: diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index b1cb5388ec29..01db045b0976 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -19,6 +19,23 @@ nextflow_workflow { config "./nextflow.config" + def unstable = [ + "umap_png", + "tsne_png", + "metric_plots", + "beagle_log", + "pca_log", + "imputed_vcf", + "pgen", + "pvar", + "psam", + "evecfile", + "evfile", + "tsv", + "umap_tsv", + "tsne_tsv" + ] + test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { when { workflow { @@ -44,16 +61,7 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot( - sanitizeOutput( - workflow.out, - unstableKeys: [ - "umap_png", - "tsne_png", - "metric_plots" - ] - ) - ).match() } + { assert snapshot(sanitizeOutput(workflow.out, unstableKeys: unstable)).match() } ) } } @@ -83,16 +91,7 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot( - sanitizeOutput( - workflow.out, - unstableKeys: [ - "umap_png", - "tsne_png", - "metric_plots" - ] - ) - ).match() } + { assert snapshot(sanitizeOutput(workflow.out, unstableKeys: unstable)).match() } ) } } diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index f6d1862a39ad..91ca7a3b6258 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -7,7 +7,7 @@ { "id": "test" }, - "test.bglout.log:md5,00f899a6db795f7d4bd79b5961ec7d10" + "test.bglout.log" ] ], "cluster_info": [ @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,f4f73c989de9e52384b786ff6ac040e5" + "test.clustering_info.json:md5,fe931a66180b920fb76958dc4026eae2" ] ], "clusters": [ @@ -31,7 +31,7 @@ { "id": "test" }, - "test.eigenvec:md5,655e0a4283a1f9d9f13c7dbe0e6da1d9" + "test.eigenvec" ] ], "evfile": [ @@ -39,7 +39,7 @@ { "id": "test" }, - "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" + "test.eigenval" ] ], "imputed_vcf": [ @@ -47,7 +47,7 @@ { "id": "test" }, - "test.bglout.vcf.gz:md5,ef94995e50ba034aeb8119ad15fb6ccc" + "test.bglout.vcf.gz" ] ], "k_sweep": [ @@ -55,7 +55,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,113fa7245a810786ec404769f5ba5698" + "test.k_sweep.csv:md5,c45d6918834e00a28498ca5ca8c0813e" ] ], "metric_plots": [ @@ -76,7 +76,7 @@ { "id": "test" }, - "test.metrics.tsv:md5,9f203e3c741435f1317dcb4ab44c43c3" + "test.metrics.tsv:md5,554b253a6508b4aa3d5396c94b41e829" ] ], "pca_log": [ @@ -84,7 +84,7 @@ { "id": "test" }, - "test.log:md5,3d794ffb37ae49b03dbc8a06a408825f" + "test.log" ] ], "pgen": [ @@ -92,7 +92,7 @@ { "id": "test" }, - "test.pgen:md5,b183c72cbf75758521bd42a21a02e882" + "test.pgen" ] ], "psam": [ @@ -100,7 +100,7 @@ { "id": "test" }, - "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" + "test.psam" ] ], "pvar": [ @@ -108,7 +108,7 @@ { "id": "test" }, - "test.pvar:md5,9c89b734017db356ccaa0190ab88c199" + "test.pvar" ] ], "selected": [ @@ -116,7 +116,7 @@ { "id": "test" }, - "test.selected.json:md5,ddbef02b869e51b8eaadceebd9ebd064" + "test.selected.json:md5,40dcca755ed1cc16c4ecd73a8f71ddcc" ] ], "tsne_png": [ @@ -132,7 +132,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,20f545d08f1175bfe1825b3932833b8d" + "test.tsne.tsv" ] ], "tsv": [ @@ -140,7 +140,7 @@ { "id": "test" }, - "test.tsv:md5,94681fa5806fbf33fa5c5041c3c29f29" + "test.tsv" ] ], "umap_png": [ @@ -156,12 +156,12 @@ { "id": "test" }, - "test.umap.tsv:md5,a7d209592da80adf142c5f1c2bce9ea2" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-26T15:28:38.960190054", + "timestamp": "2026-08-26T15:54:35.227818603", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -175,7 +175,7 @@ { "id": "test" }, - "test.bglout.log:md5,00f899a6db795f7d4bd79b5961ec7d10" + "test.bglout.log" ] ], "cluster_info": [ @@ -199,7 +199,7 @@ { "id": "test" }, - "test.eigenvec:md5,cb7d3501265fb6e681f9a66552faff42" + "test.eigenvec" ] ], "evfile": [ @@ -207,7 +207,7 @@ { "id": "test" }, - "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" + "test.eigenval" ] ], "imputed_vcf": [ @@ -215,7 +215,7 @@ { "id": "test" }, - "test.bglout.vcf.gz:md5,ef94995e50ba034aeb8119ad15fb6ccc" + "test.bglout.vcf.gz" ] ], "k_sweep": [ @@ -223,7 +223,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,d75662c7c614f39e294601653c02de8d" + "test.k_sweep.csv:md5,f3d4ba46963ec779099d616d23cf8156" ] ], "metric_plots": [ @@ -252,7 +252,7 @@ { "id": "test" }, - "test.log:md5,65d20c952985adb5fa7229a5d0d4fc0c" + "test.log" ] ], "pgen": [ @@ -260,7 +260,7 @@ { "id": "test" }, - "test.pgen:md5,b183c72cbf75758521bd42a21a02e882" + "test.pgen" ] ], "psam": [ @@ -268,7 +268,7 @@ { "id": "test" }, - "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" + "test.psam" ] ], "pvar": [ @@ -276,7 +276,7 @@ { "id": "test" }, - "test.pvar:md5,9c89b734017db356ccaa0190ab88c199" + "test.pvar" ] ], "selected": [ @@ -300,7 +300,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,f4952012bc87993ca78429d4a823edb1" + "test.tsne.tsv" ] ], "tsv": [ @@ -308,7 +308,7 @@ { "id": "test" }, - "test.tsv:md5,8009b5a5210c1dae667f40c40b450035" + "test.tsv" ] ], "umap_png": [ @@ -324,12 +324,12 @@ { "id": "test" }, - "test.umap.tsv:md5,26c791bc4d38dff6f979582611dbc27d" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-26T15:29:18.988004598", + "timestamp": "2026-08-26T15:55:13.730658349", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 59e09ba60ebba387c66bf90a3374fb81cfb41fc2 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:09:30 +0200 Subject: [PATCH 16/34] Update import paths in main.nf for modules --- subworkflows/nf-core/snpclustering/main.nf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 3b8fb56bf670..edb439e8f14a 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,10 +1,10 @@ include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' -include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf/main' -include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca/main' -include { GAWK as GAWK_EIGENVEC_TO_TSV } from '../../../modules/nf-core/gawk/main' -include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering/main' -include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics/main' -include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization/main' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca' +include { GAWK as GAWK_EIGENVEC_TO_TSV } from '../../../modules/nf-core/gawk' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics' +include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization' workflow SNPCLUSTERING { take: From cc0a0528a2287b90f7b31b2a182b1567a2222c20 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 19:14:27 +0200 Subject: [PATCH 17/34] Set seed 1 plink2_pca --- .../nf-core/snpclustering/tests/main.nf.test | 8 +-- .../snpclustering/tests/main.nf.test.snap | 54 +++++++++---------- .../snpclustering/tests/nextflow.config | 3 ++ 3 files changed, 31 insertions(+), 34 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 01db045b0976..ab54d600b2ab 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -20,9 +20,6 @@ nextflow_workflow { config "./nextflow.config" def unstable = [ - "umap_png", - "tsne_png", - "metric_plots", "beagle_log", "pca_log", "imputed_vcf", @@ -30,10 +27,7 @@ nextflow_workflow { "pvar", "psam", "evecfile", - "evfile", - "tsv", - "umap_tsv", - "tsne_tsv" + "evfile" ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 91ca7a3b6258..fca6b4aa8b84 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,fe931a66180b920fb76958dc4026eae2" + "test.clustering_info.json:md5,0fb1dbe8182e71725eb0d910c400d49f" ] ], "clusters": [ @@ -55,7 +55,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,c45d6918834e00a28498ca5ca8c0813e" + "test.k_sweep.csv:md5,d05da63474596a4a20cac7160907228d" ] ], "metric_plots": [ @@ -64,10 +64,10 @@ "id": "test" }, [ - "test.calinski_harabasz.png", - "test.davies_bouldin.png", - "test.elbow.png", - "test.silhouette.png" + "test.calinski_harabasz.png:md5,126ae1fd131c3d312200dede14daf921", + "test.davies_bouldin.png:md5,e30697feedfe4eef4256f1a29cd5a174", + "test.elbow.png:md5,58642a6c45ca085b73f65062a094ed94", + "test.silhouette.png:md5,4036cc8163eafff1bfbc45648e9bfa03" ] ] ], @@ -76,7 +76,7 @@ { "id": "test" }, - "test.metrics.tsv:md5,554b253a6508b4aa3d5396c94b41e829" + "test.metrics.tsv:md5,82d8ff162f7ae167e5613afc9ca41311" ] ], "pca_log": [ @@ -116,7 +116,7 @@ { "id": "test" }, - "test.selected.json:md5,40dcca755ed1cc16c4ecd73a8f71ddcc" + "test.selected.json:md5,b0da8d7265531d757724e3ec8b358167" ] ], "tsne_png": [ @@ -124,7 +124,7 @@ { "id": "test" }, - "test.tsne.png" + "test.tsne.png:md5,d298f009fc6ae29b3e902069c2c19116" ] ], "tsne_tsv": [ @@ -132,7 +132,7 @@ { "id": "test" }, - "test.tsne.tsv" + "test.tsne.tsv:md5,f3be272b77fdc209611d7f552443c54d" ] ], "tsv": [ @@ -140,7 +140,7 @@ { "id": "test" }, - "test.tsv" + "test.tsv:md5,402ff5581c491c03227f66e59d55e189" ] ], "umap_png": [ @@ -148,7 +148,7 @@ { "id": "test" }, - "test.umap.png" + "test.umap.png:md5,42641c121e5040434e8b365dfd4ad18a" ] ], "umap_tsv": [ @@ -156,12 +156,12 @@ { "id": "test" }, - "test.umap.tsv" + "test.umap.tsv:md5,cd124eb5e14ed6d02f3cce45a2ae2559" ] ] } ], - "timestamp": "2026-08-26T15:54:35.227818603", + "timestamp": "2026-08-26T19:12:30.318638023", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -223,7 +223,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,f3d4ba46963ec779099d616d23cf8156" + "test.k_sweep.csv:md5,d05da63474596a4a20cac7160907228d" ] ], "metric_plots": [ @@ -232,10 +232,10 @@ "id": "test" }, [ - "test.calinski_harabasz.png", - "test.davies_bouldin.png", - "test.elbow.png", - "test.silhouette.png" + "test.calinski_harabasz.png:md5,126ae1fd131c3d312200dede14daf921", + "test.davies_bouldin.png:md5,e30697feedfe4eef4256f1a29cd5a174", + "test.elbow.png:md5,58642a6c45ca085b73f65062a094ed94", + "test.silhouette.png:md5,4036cc8163eafff1bfbc45648e9bfa03" ] ] ], @@ -292,7 +292,7 @@ { "id": "test" }, - "test.tsne.png" + "test.tsne.png:md5,23fc4ac667d9ef86bfd56652a8d79eab" ] ], "tsne_tsv": [ @@ -300,7 +300,7 @@ { "id": "test" }, - "test.tsne.tsv" + "test.tsne.tsv:md5,971871b2a4befda6baa9a9521cc1084d" ] ], "tsv": [ @@ -308,7 +308,7 @@ { "id": "test" }, - "test.tsv" + "test.tsv:md5,402ff5581c491c03227f66e59d55e189" ] ], "umap_png": [ @@ -316,7 +316,7 @@ { "id": "test" }, - "test.umap.png" + "test.umap.png:md5,3df8ccc41ea0cd080224879353d3f5bf" ] ], "umap_tsv": [ @@ -324,12 +324,12 @@ { "id": "test" }, - "test.umap.tsv" + "test.umap.tsv:md5,0b660d0fbd3df1f6cd130eb50339afe3" ] ] } ], - "timestamp": "2026-08-26T15:55:13.730658349", + "timestamp": "2026-08-26T19:13:10.66966192", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -497,7 +497,7 @@ ] } ], - "timestamp": "2026-08-26T15:29:43.011468684", + "timestamp": "2026-08-26T18:02:19.483018891", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -665,7 +665,7 @@ ] } ], - "timestamp": "2026-08-26T15:29:31.961527536", + "timestamp": "2026-08-26T18:01:09.390553721", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 93f8ed077be8..0d5b9ffa89f2 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -8,6 +8,9 @@ params { } process { + withName: 'PLINK2_PCA' { + ext.args = '--seed 1' + } withName: GAWK_EIGENVEC_TO_TSV { ext.args = "-v OFS='\\t'" ext.suffix = "tsv" From 44bb942437ff70e1e18baaabb48444003c4ae935 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 19:55:59 +0200 Subject: [PATCH 18/34] Added tsv in unstable md5 --- .../nf-core/snpclustering/tests/main.nf.test | 6 ++++- .../snpclustering/tests/main.nf.test.snap | 22 +++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index ab54d600b2ab..ad0a394907d2 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -27,7 +27,11 @@ nextflow_workflow { "pvar", "psam", "evecfile", - "evfile" + "evfile", + "umap_png", + "tsne_png", + "umap_tsv", + "tsne_tsv" ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index fca6b4aa8b84..d1108974dd9e 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,0fb1dbe8182e71725eb0d910c400d49f" + "test.clustering_info.json:md5,23e8966d37219b6ebe614eaaebd8636d" ] ], "clusters": [ @@ -124,7 +124,7 @@ { "id": "test" }, - "test.tsne.png:md5,d298f009fc6ae29b3e902069c2c19116" + "test.tsne.png" ] ], "tsne_tsv": [ @@ -132,7 +132,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,f3be272b77fdc209611d7f552443c54d" + "test.tsne.tsv" ] ], "tsv": [ @@ -148,7 +148,7 @@ { "id": "test" }, - "test.umap.png:md5,42641c121e5040434e8b365dfd4ad18a" + "test.umap.png" ] ], "umap_tsv": [ @@ -156,12 +156,12 @@ { "id": "test" }, - "test.umap.tsv:md5,cd124eb5e14ed6d02f3cce45a2ae2559" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-26T19:12:30.318638023", + "timestamp": "2026-08-26T19:54:05.005883762", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -292,7 +292,7 @@ { "id": "test" }, - "test.tsne.png:md5,23fc4ac667d9ef86bfd56652a8d79eab" + "test.tsne.png" ] ], "tsne_tsv": [ @@ -300,7 +300,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,971871b2a4befda6baa9a9521cc1084d" + "test.tsne.tsv" ] ], "tsv": [ @@ -316,7 +316,7 @@ { "id": "test" }, - "test.umap.png:md5,3df8ccc41ea0cd080224879353d3f5bf" + "test.umap.png" ] ], "umap_tsv": [ @@ -324,12 +324,12 @@ { "id": "test" }, - "test.umap.tsv:md5,0b660d0fbd3df1f6cd130eb50339afe3" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-26T19:13:10.66966192", + "timestamp": "2026-08-26T19:54:38.94631929", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 44baf0f0b1ab7028199dd5dc6b8eabc2ec349969 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:09:57 +0200 Subject: [PATCH 19/34] Add UMAP and t-SNE output file references Added new output files for UMAP and t-SNE visualizations. --- subworkflows/nf-core/snpclustering/tests/main.nf.test | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index ab54d600b2ab..4f09fc2098ef 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -26,8 +26,10 @@ nextflow_workflow { "pgen", "pvar", "psam", - "evecfile", - "evfile" + "umap_png", + "tsne_png", + "umap_tsv", + "tsne_tsv ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { From 6017365f2958ba3dd24650b30e1f406f38dce119 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Wed, 26 Aug 2026 20:13:58 +0200 Subject: [PATCH 20/34] Fix typo in tsne_tsv variable declaration --- subworkflows/nf-core/snpclustering/tests/main.nf.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 4f09fc2098ef..48ac5c9b7650 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -29,7 +29,7 @@ nextflow_workflow { "umap_png", "tsne_png", "umap_tsv", - "tsne_tsv + "tsne_tsv" ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { From 90425b64e574c18eb43fb6a742d7a8e17e5da805 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 20:59:30 +0200 Subject: [PATCH 21/34] Update snapshot --- subworkflows/nf-core/snpclustering/tests/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index d1108974dd9e..d7928a65d525 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,23e8966d37219b6ebe614eaaebd8636d" + "test.clustering_info.json:md5,32112858ff769d83d1558ec56c212dd2" ] ], "clusters": [ @@ -161,7 +161,7 @@ ] } ], - "timestamp": "2026-08-26T19:54:05.005883762", + "timestamp": "2026-08-26T20:57:46.611731056", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 94751cce4aeb1a83dec9285b1941d4c9be0432c8 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 26 Aug 2026 21:14:54 +0200 Subject: [PATCH 22/34] Update snapshot --- subworkflows/nf-core/snpclustering/tests/main.nf.test.snap | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index d7928a65d525..6b8b53861054 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,32112858ff769d83d1558ec56c212dd2" + "test.clustering_info.json:md5,0fb1dbe8182e71725eb0d910c400d49f" ] ], "clusters": [ @@ -161,7 +161,7 @@ ] } ], - "timestamp": "2026-08-26T20:57:46.611731056", + "timestamp": "2026-08-26T21:09:37.53551214", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 2e0c680c375d9aba10b9c42ad490047ca996244f Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 27 Aug 2026 11:47:59 +0200 Subject: [PATCH 23/34] stabilizing through config file --- .../nf-core/snpclustering/tests/main.nf.test | 7 ++- .../snpclustering/tests/main.nf.test.snap | 36 ++++++------- .../snpclustering/tests/nextflow.config | 53 +++++++++++++++++-- 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 152cdcf616ad..9e87e34e4c1c 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -19,7 +19,7 @@ nextflow_workflow { config "./nextflow.config" - def unstable = [ + def unstable = [ "beagle_log", "pca_log", "imputed_vcf", @@ -28,11 +28,14 @@ nextflow_workflow { "psam", "evecfile", "evfile", + "k_sweep", + "metric_plots", + "selected", "umap_png", "tsne_png", "umap_tsv", "tsne_tsv" - ] + ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { when { diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 148ccbaeb915..3c325fd8eea2 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,32112858ff769d83d1558ec56c212dd2" + "test.clustering_info.json:md5,e3cfaf9b4966910425fe730fe80299fa" ] ], "clusters": [ @@ -55,7 +55,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,d05da63474596a4a20cac7160907228d" + "test.k_sweep.csv" ] ], "metric_plots": [ @@ -64,10 +64,10 @@ "id": "test" }, [ - "test.calinski_harabasz.png:md5,126ae1fd131c3d312200dede14daf921", - "test.davies_bouldin.png:md5,e30697feedfe4eef4256f1a29cd5a174", - "test.elbow.png:md5,58642a6c45ca085b73f65062a094ed94", - "test.silhouette.png:md5,4036cc8163eafff1bfbc45648e9bfa03" + "test.calinski_harabasz.png", + "test.davies_bouldin.png", + "test.elbow.png", + "test.silhouette.png" ] ] ], @@ -76,7 +76,7 @@ { "id": "test" }, - "test.metrics.tsv:md5,82d8ff162f7ae167e5613afc9ca41311" + "test.metrics.tsv:md5,82e1eecd3e94e64de5c1a2f5d44eb9f5" ] ], "pca_log": [ @@ -116,7 +116,7 @@ { "id": "test" }, - "test.selected.json:md5,b0da8d7265531d757724e3ec8b358167" + "test.selected.json" ] ], "tsne_png": [ @@ -140,7 +140,7 @@ { "id": "test" }, - "test.tsv:md5,402ff5581c491c03227f66e59d55e189" + "test.tsv:md5,f5ec5abd246d89cf50d333d432e4366a" ] ], "umap_png": [ @@ -161,7 +161,7 @@ ] } ], - "timestamp": "2026-08-26T21:30:27.835190347", + "timestamp": "2026-08-27T11:45:42.400185807", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -223,7 +223,7 @@ { "id": "test" }, - "test.k_sweep.csv:md5,d05da63474596a4a20cac7160907228d" + "test.k_sweep.csv" ] ], "metric_plots": [ @@ -232,10 +232,10 @@ "id": "test" }, [ - "test.calinski_harabasz.png:md5,126ae1fd131c3d312200dede14daf921", - "test.davies_bouldin.png:md5,e30697feedfe4eef4256f1a29cd5a174", - "test.elbow.png:md5,58642a6c45ca085b73f65062a094ed94", - "test.silhouette.png:md5,4036cc8163eafff1bfbc45648e9bfa03" + "test.calinski_harabasz.png", + "test.davies_bouldin.png", + "test.elbow.png", + "test.silhouette.png" ] ] ], @@ -284,7 +284,7 @@ { "id": "test" }, - "test.selected.json:md5,138b2bae0f67f913dcead80f8202d9af" + "test.selected.json" ] ], "tsne_png": [ @@ -308,7 +308,7 @@ { "id": "test" }, - "test.tsv:md5,402ff5581c491c03227f66e59d55e189" + "test.tsv:md5,f5ec5abd246d89cf50d333d432e4366a" ] ], "umap_png": [ @@ -329,7 +329,7 @@ ] } ], - "timestamp": "2026-08-26T19:54:38.94631929", + "timestamp": "2026-08-27T11:46:23.671901862", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 0d5b9ffa89f2..f3c12ec19a78 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -1,6 +1,6 @@ params { npcs = 8 - use_approx = true + use_approx = false algorithm = "kmeans" n_clusters = 3 dbscan_eps = 0.5 @@ -8,11 +8,58 @@ params { } process { + withName: 'BEAGLE5_BEAGLE' { + cpus = 4 + memory = '8 GB' + ext.args = 'seed=1 nthreads=4' + } + + withName: 'PLINK2_VCF' { + ext.args = '--seed 1' + } + withName: 'PLINK2_PCA' { ext.args = '--seed 1' } - withName: GAWK_EIGENVEC_TO_TSV { + + withName: 'GAWK_EIGENVEC_TO_TSV' { ext.args = "-v OFS='\\t'" - ext.suffix = "tsv" + ext.suffix = 'tsv' + } + + withName: 'CUSTOM_PCACLUSTERING' { + cpus = 1 + env { + OMP_NUM_THREADS = '1' + OPENBLAS_NUM_THREADS = '1' + MKL_NUM_THREADS = '1' + NUMEXPR_NUM_THREADS = '1' + } + } + + withName: 'CUSTOM_CLUSTERMETRICS' { + cpus = 1 + ext.args = '--k-min 2 --k-max 12' + env { + OMP_NUM_THREADS = '1' + OPENBLAS_NUM_THREADS = '1' + MKL_NUM_THREADS = '1' + NUMEXPR_NUM_THREADS = '1' + NUMBA_NUM_THREADS = '1' + } + } + + withName: 'CUSTOM_CLUSTERVISUALIZATION' { + cpus = 1 + ext.args = '--umap-neighbors 15 --tsne-perplexity 30' + env { + OMP_NUM_THREADS = '1' + OPENBLAS_NUM_THREADS = '1' + MKL_NUM_THREADS = '1' + NUMEXPR_NUM_THREADS = '1' + NUMBA_NUM_THREADS = '1' + NUMBA_CACHE_DIR = '/tmp' + MPLCONFIGDIR = '/tmp' + } } } From 78122f0b9ad6230369b64a7410313e888ae3869e Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 27 Aug 2026 12:36:40 +0200 Subject: [PATCH 24/34] Fixing env block --- .../snpclustering/tests/main.nf.test.snap | 4 +-- .../snpclustering/tests/nextflow.config | 34 ++++++------------- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 3c325fd8eea2..b5fb67b176cc 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,e3cfaf9b4966910425fe730fe80299fa" + "test.clustering_info.json:md5,ffab1c4d2fa185211fe024365e7bc432" ] ], "clusters": [ @@ -161,7 +161,7 @@ ] } ], - "timestamp": "2026-08-27T11:45:42.400185807", + "timestamp": "2026-08-27T12:32:09.448470818", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index f3c12ec19a78..29057c7a404f 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -7,10 +7,20 @@ params { dbscan_min_samples = 5 } +env { + OMP_NUM_THREADS = '1' + OPENBLAS_NUM_THREADS = '1' + MKL_NUM_THREADS = '1' + NUMEXPR_NUM_THREADS = '1' + NUMBA_NUM_THREADS = '1' + NUMBA_CACHE_DIR = '/tmp' + MPLCONFIGDIR = '/tmp' +} + process { withName: 'BEAGLE5_BEAGLE' { cpus = 4 - memory = '8 GB' + memory = 8.GB ext.args = 'seed=1 nthreads=4' } @@ -29,37 +39,15 @@ process { withName: 'CUSTOM_PCACLUSTERING' { cpus = 1 - env { - OMP_NUM_THREADS = '1' - OPENBLAS_NUM_THREADS = '1' - MKL_NUM_THREADS = '1' - NUMEXPR_NUM_THREADS = '1' - } } withName: 'CUSTOM_CLUSTERMETRICS' { cpus = 1 ext.args = '--k-min 2 --k-max 12' - env { - OMP_NUM_THREADS = '1' - OPENBLAS_NUM_THREADS = '1' - MKL_NUM_THREADS = '1' - NUMEXPR_NUM_THREADS = '1' - NUMBA_NUM_THREADS = '1' - } } withName: 'CUSTOM_CLUSTERVISUALIZATION' { cpus = 1 ext.args = '--umap-neighbors 15 --tsne-perplexity 30' - env { - OMP_NUM_THREADS = '1' - OPENBLAS_NUM_THREADS = '1' - MKL_NUM_THREADS = '1' - NUMEXPR_NUM_THREADS = '1' - NUMBA_NUM_THREADS = '1' - NUMBA_CACHE_DIR = '/tmp' - MPLCONFIGDIR = '/tmp' - } } } From 2d428b297ec677ca9b8cf7f11bac83d74519c8a1 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Thu, 27 Aug 2026 15:23:11 +0200 Subject: [PATCH 25/34] Remove unused output variables from tests to check md5 stabilty --- subworkflows/nf-core/snpclustering/tests/main.nf.test | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 9e87e34e4c1c..5a17f1fd3607 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -25,16 +25,10 @@ nextflow_workflow { "imputed_vcf", "pgen", "pvar", - "psam", - "evecfile", - "evfile", "k_sweep", "metric_plots", - "selected", "umap_png", - "tsne_png", - "umap_tsv", - "tsne_tsv" + "tsne_png" ] test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { From da18b2fcc45c286f1c80f8a17ed7922e6bc6540f Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 27 Aug 2026 15:36:26 +0200 Subject: [PATCH 26/34] Update snapshot --- .../snpclustering/tests/main.nf.test.snap | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index b5fb67b176cc..b04ec8ffebd4 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -31,7 +31,7 @@ { "id": "test" }, - "test.eigenvec" + "test.eigenvec:md5,0e5d88169892d522252827b7897c0369" ] ], "evfile": [ @@ -39,7 +39,7 @@ { "id": "test" }, - "test.eigenval" + "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" ] ], "imputed_vcf": [ @@ -100,7 +100,7 @@ { "id": "test" }, - "test.psam" + "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" ] ], "pvar": [ @@ -116,7 +116,7 @@ { "id": "test" }, - "test.selected.json" + "test.selected.json:md5,d085933630a002310508c3ef0879f1c8" ] ], "tsne_png": [ @@ -132,7 +132,7 @@ { "id": "test" }, - "test.tsne.tsv" + "test.tsne.tsv:md5,1a3bbc0cd0c62160252b8bd6c23b5934" ] ], "tsv": [ @@ -156,12 +156,12 @@ { "id": "test" }, - "test.umap.tsv" + "test.umap.tsv:md5,9cd5ff6fd3a036d721882eb02ce6617a" ] ] } ], - "timestamp": "2026-08-27T12:32:09.448470818", + "timestamp": "2026-08-27T15:34:32.894346349", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -199,7 +199,7 @@ { "id": "test" }, - "test.eigenvec" + "test.eigenvec:md5,0e5d88169892d522252827b7897c0369" ] ], "evfile": [ @@ -207,7 +207,7 @@ { "id": "test" }, - "test.eigenval" + "test.eigenval:md5,a5d4576064133a95bbf629b99b43a0c3" ] ], "imputed_vcf": [ @@ -268,7 +268,7 @@ { "id": "test" }, - "test.psam" + "test.psam:md5,31604b4a3b20fb02ef1c59e9ead65f00" ] ], "pvar": [ @@ -284,7 +284,7 @@ { "id": "test" }, - "test.selected.json" + "test.selected.json:md5,138b2bae0f67f913dcead80f8202d9af" ] ], "tsne_png": [ @@ -300,7 +300,7 @@ { "id": "test" }, - "test.tsne.tsv" + "test.tsne.tsv:md5,f58cb946289e5c96f7a3c2a7908ef020" ] ], "tsv": [ @@ -324,12 +324,12 @@ { "id": "test" }, - "test.umap.tsv" + "test.umap.tsv:md5,29d8fa6c7cd768a652d208628f704cc5" ] ] } ], - "timestamp": "2026-08-27T11:46:23.671901862", + "timestamp": "2026-08-27T15:35:20.040760668", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From f788b1ae89a8b606d642956cf5ac790a382e45cb Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Thu, 27 Aug 2026 15:58:47 +0200 Subject: [PATCH 27/34] TSNE and UMAP tsv unstable md5 --- .../nf-core/snpclustering/tests/main.nf.test | 2 ++ .../nf-core/snpclustering/tests/main.nf.test.snap | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 5a17f1fd3607..368e45c16240 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -28,6 +28,8 @@ nextflow_workflow { "k_sweep", "metric_plots", "umap_png", + "tsne_tsv", + "umap_tsv", "tsne_png" ] diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index b04ec8ffebd4..ba1ae9179c91 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -132,7 +132,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,1a3bbc0cd0c62160252b8bd6c23b5934" + "test.tsne.tsv" ] ], "tsv": [ @@ -156,12 +156,12 @@ { "id": "test" }, - "test.umap.tsv:md5,9cd5ff6fd3a036d721882eb02ce6617a" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-27T15:34:32.894346349", + "timestamp": "2026-08-27T15:56:45.881810208", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -300,7 +300,7 @@ { "id": "test" }, - "test.tsne.tsv:md5,f58cb946289e5c96f7a3c2a7908ef020" + "test.tsne.tsv" ] ], "tsv": [ @@ -324,12 +324,12 @@ { "id": "test" }, - "test.umap.tsv:md5,29d8fa6c7cd768a652d208628f704cc5" + "test.umap.tsv" ] ] } ], - "timestamp": "2026-08-27T15:35:20.040760668", + "timestamp": "2026-08-27T15:57:35.308865685", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 7c590bde1c6c58e50bd9753cbafdb834eed83594 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Mon, 31 Aug 2026 12:25:26 +0200 Subject: [PATCH 28/34] fix(snpclustering): keep test config to seeds and gawk suffix Drop env/cpu overrides and default ext.args. Pin BEAGLE/PLINK2 seeds for stable hashes and set GAWK_EIGENVEC_TO_TSV ext.suffix=tsv so the output does not collide with PLINK2 ${meta.id}.eigenvec. --- .../snpclustering/awk/eigenvec_to_tsv.awk | 18 +++++++--- .../nf-core/snpclustering/tests/main.nf.test | 36 +++++++++---------- .../snpclustering/tests/main.nf.test.snap | 10 +++--- .../snpclustering/tests/nextflow.config | 29 +-------------- 4 files changed, 38 insertions(+), 55 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk b/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk index c84a0c3e9160..be40fe450b82 100644 --- a/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk +++ b/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk @@ -1,9 +1,19 @@ -NR==1 { +BEGIN { + FS = OFS = "\t" +} + +NR == 1 { sub(/^#/, "") - if ($1 == "IID") { - $1 = "sample_id" + $0 = $0 + for (i = 1; i <= NF; i++) { + if ($i == "IID") { + $i = "sample_id" + } } print next } -{ print } + +{ + print +} diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 368e45c16240..f80b487bdb32 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -45,12 +45,12 @@ nextflow_workflow { input[1] = Channel.empty() input[2] = Channel.empty() input[3] = "" - input[4] = params.npcs - input[5] = params.use_approx + input[4] = 8 + input[5] = false input[6] = "kmeans" input[7] = 3 - input[8] = params.dbscan_eps - input[9] = params.dbscan_min_samples + input[8] = 0.5 + input[9] = 5 """ } } @@ -75,12 +75,12 @@ nextflow_workflow { input[1] = Channel.empty() input[2] = Channel.empty() input[3] = "" - input[4] = params.npcs - input[5] = params.use_approx + input[4] = 8 + input[5] = false input[6] = "dbscan" - input[7] = params.n_clusters - input[8] = params.dbscan_eps - input[9] = params.dbscan_min_samples + input[7] = 3 + input[8] = 0.5 + input[9] = 5 """ } } @@ -106,12 +106,12 @@ nextflow_workflow { input[1] = Channel.empty() input[2] = Channel.empty() input[3] = "" - input[4] = params.npcs - input[5] = params.use_approx + input[4] = 8 + input[5] = false input[6] = "kmeans" input[7] = 3 - input[8] = params.dbscan_eps - input[9] = params.dbscan_min_samples + input[8] = 0.5 + input[9] = 5 """ } } @@ -137,12 +137,12 @@ nextflow_workflow { input[1] = Channel.empty() input[2] = Channel.empty() input[3] = "" - input[4] = params.npcs - input[5] = params.use_approx + input[4] = 8 + input[5] = false input[6] = "dbscan" - input[7] = params.n_clusters - input[8] = params.dbscan_eps - input[9] = params.dbscan_min_samples + input[7] = 3 + input[8] = 0.5 + input[9] = 5 """ } } diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index ba1ae9179c91..8aa645c8a1a2 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,ffab1c4d2fa185211fe024365e7bc432" + "test.clustering_info.json:md5,5bfc98c30e4e6d7d4bd730e55fa90ca7" ] ], "clusters": [ @@ -161,7 +161,7 @@ ] } ], - "timestamp": "2026-08-27T15:56:45.881810208", + "timestamp": "2026-08-31T12:13:13.95854724", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -329,7 +329,7 @@ ] } ], - "timestamp": "2026-08-27T15:57:35.308865685", + "timestamp": "2026-08-31T12:13:46.467083247", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -497,7 +497,7 @@ ] } ], - "timestamp": "2026-08-26T18:02:19.483018891", + "timestamp": "2026-08-31T12:14:11.172911335", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -665,7 +665,7 @@ ] } ], - "timestamp": "2026-08-26T18:01:09.390553721", + "timestamp": "2026-08-31T12:13:58.254898619", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index 29057c7a404f..c68232c7a163 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -7,21 +7,9 @@ params { dbscan_min_samples = 5 } -env { - OMP_NUM_THREADS = '1' - OPENBLAS_NUM_THREADS = '1' - MKL_NUM_THREADS = '1' - NUMEXPR_NUM_THREADS = '1' - NUMBA_NUM_THREADS = '1' - NUMBA_CACHE_DIR = '/tmp' - MPLCONFIGDIR = '/tmp' -} - process { withName: 'BEAGLE5_BEAGLE' { - cpus = 4 - memory = 8.GB - ext.args = 'seed=1 nthreads=4' + ext.args = 'seed=1' } withName: 'PLINK2_VCF' { @@ -33,21 +21,6 @@ process { } withName: 'GAWK_EIGENVEC_TO_TSV' { - ext.args = "-v OFS='\\t'" ext.suffix = 'tsv' } - - withName: 'CUSTOM_PCACLUSTERING' { - cpus = 1 - } - - withName: 'CUSTOM_CLUSTERMETRICS' { - cpus = 1 - ext.args = '--k-min 2 --k-max 12' - } - - withName: 'CUSTOM_CLUSTERVISUALIZATION' { - cpus = 1 - ext.args = '--umap-neighbors 15 --tsne-perplexity 30' - } } From 50fe64c418106bd786b62c639ad2640218498d63 Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Mon, 31 Aug 2026 12:56:20 +0200 Subject: [PATCH 29/34] Fix unstable md5sum --- .../nf-core/snpclustering/tests/main.nf.test | 33 +++++++++- .../snpclustering/tests/main.nf.test.snap | 64 +++++++++++++++++-- 2 files changed, 91 insertions(+), 6 deletions(-) diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index f80b487bdb32..77c8398c2bfa 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -27,12 +27,31 @@ nextflow_workflow { "pvar", "k_sweep", "metric_plots", + "cluster_info", "umap_png", "tsne_tsv", "umap_tsv", "tsne_png" ] + def embedStats = { path -> + def rows = file(path).readLines().findAll { it } + def header = rows.head().split('\t') as List + [ + header : header, + n_rows : rows.size() - 1, + n_cols : header.size() + ] + } + + def clusterInfo = { path -> + def info_data = new groovy.json.JsonSlurper().parse(file(path)) + if (info_data.containsKey('inertia')) { + info_data.inertia = (info_data.inertia as Double).round(4) + } + info_data + } + test("homo_sapiens - 1000GP.chr22.vcf.gz - kmeans 3 clusters") { when { workflow { @@ -58,7 +77,12 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(sanitizeOutput(workflow.out, unstableKeys: unstable)).match() } + { assert snapshot( + sanitizeOutput(workflow.out, unstableKeys: unstable), + clusterInfo(workflow.out.cluster_info[0][1]), + embedStats(workflow.out.umap_tsv[0][1]), + embedStats(workflow.out.tsne_tsv[0][1]) + ).match() } ) } } @@ -88,7 +112,12 @@ nextflow_workflow { then { assertAll( { assert workflow.success }, - { assert snapshot(sanitizeOutput(workflow.out, unstableKeys: unstable)).match() } + { assert snapshot( + sanitizeOutput(workflow.out, unstableKeys: unstable), + clusterInfo(workflow.out.cluster_info[0][1]), + embedStats(workflow.out.umap_tsv[0][1]), + embedStats(workflow.out.tsne_tsv[0][1]) + ).match() } ) } } diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index 8aa645c8a1a2..eb11e4cc6b4c 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -15,7 +15,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,5bfc98c30e4e6d7d4bd730e55fa90ca7" + "test.clustering_info.json" ] ], "clusters": [ @@ -159,9 +159,36 @@ "test.umap.tsv" ] ] + }, + { + "algorithm": "kmeans", + "k": 3, + "inertia": 6.2255, + "n_samples": 3202, + "n_features": 8 + }, + { + "header": [ + "sample_id", + "Dim1", + "Dim2", + "cluster" + ], + "n_rows": 3202, + "n_cols": 4 + }, + { + "header": [ + "sample_id", + "Dim1", + "Dim2", + "cluster" + ], + "n_rows": 3202, + "n_cols": 4 } ], - "timestamp": "2026-08-31T12:13:13.95854724", + "timestamp": "2026-08-31T12:49:22.352660673", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -183,7 +210,7 @@ { "id": "test" }, - "test.clustering_info.json:md5,fa9a2eff36d06fe360a59f4c3cd63393" + "test.clustering_info.json" ] ], "clusters": [ @@ -327,9 +354,38 @@ "test.umap.tsv" ] ] + }, + { + "algorithm": "dbscan", + "eps": 0.5, + "min_samples": 5, + "n_clusters_found": 1, + "n_noise": 0, + "n_samples": 3202, + "n_features": 8 + }, + { + "header": [ + "sample_id", + "Dim1", + "Dim2", + "cluster" + ], + "n_rows": 3202, + "n_cols": 4 + }, + { + "header": [ + "sample_id", + "Dim1", + "Dim2", + "cluster" + ], + "n_rows": 3202, + "n_cols": 4 } ], - "timestamp": "2026-08-31T12:13:46.467083247", + "timestamp": "2026-08-31T12:49:54.909445446", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" From 029813808f71ea684887f6c4a6bca63dd2b441ea Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:49:27 +0200 Subject: [PATCH 30/34] Update subworkflows/nf-core/snpclustering/main.nf Co-authored-by: Niklas Schandry --- subworkflows/nf-core/snpclustering/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index edb439e8f14a..fb7afd77570c 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -12,7 +12,7 @@ workflow SNPCLUSTERING { refpanel_ch genmap_ch region - npcs + n_pcs use_approx algorithm n_clusters From bbaa6d42663ef340406ea8db48044898fecd9f92 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Thu, 3 Sep 2026 08:49:47 +0200 Subject: [PATCH 31/34] Update subworkflows/nf-core/snpclustering/main.nf Co-authored-by: Niklas Schandry --- subworkflows/nf-core/snpclustering/main.nf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index fb7afd77570c..963e33dcba1c 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -60,7 +60,7 @@ workflow SNPCLUSTERING { .join(PLINK2_VCF.out.pvar) .join(PLINK2_VCF.out.psam) .map { meta, pgen, pvar, psam -> - tuple(meta, npcs, use_approx, pgen, psam, pvar) + tuple(meta, n_pcs, use_approx, pgen, psam, pvar) } PLINK2_PCA(ch_plink_pca_input) From bd4bd3cb3cadd3ce257b1feec42ef4d226123f6c Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:21:09 +0200 Subject: [PATCH 32/34] Enhance description for clustering metrics Updated the description to include k-distance analysis. --- modules/nf-core/custom/clustermetrics/meta.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/nf-core/custom/clustermetrics/meta.yml b/modules/nf-core/custom/clustermetrics/meta.yml index 92a1bea0d37b..2cc8800d81a4 100644 --- a/modules/nf-core/custom/clustermetrics/meta.yml +++ b/modules/nf-core/custom/clustermetrics/meta.yml @@ -1,5 +1,5 @@ name: "CUSTOM_CLUSTERMETRICS" -description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, Davies-Bouldin) and performs k-sweep analysis" +description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, Davies-Bouldin), performs k-sweep and k-distance analysis" keywords: - clustering - metrics From 2897267696d35fd23826103cc05fa16e939b78d6 Mon Sep 17 00:00:00 2001 From: Donald Baku <141358602+dbaku42@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:22:26 +0200 Subject: [PATCH 33/34] Format description in meta.yml for readability Split the description into multiple lines for better readability. --- modules/nf-core/custom/clustermetrics/meta.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/nf-core/custom/clustermetrics/meta.yml b/modules/nf-core/custom/clustermetrics/meta.yml index 2cc8800d81a4..4b53608ebae1 100644 --- a/modules/nf-core/custom/clustermetrics/meta.yml +++ b/modules/nf-core/custom/clustermetrics/meta.yml @@ -1,5 +1,6 @@ name: "CUSTOM_CLUSTERMETRICS" -description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, Davies-Bouldin), performs k-sweep and k-distance analysis" +description: "Computes clustering quality metrics (silhouette, Calinski-Harabasz, + Davies-Bouldin), performs k-sweep and k-distance analysis" keywords: - clustering - metrics From 1e7ac0f50b54f6f3fa552807cb653deea2d0f0fd Mon Sep 17 00:00:00 2001 From: dbaku42 Date: Wed, 9 Sep 2026 10:59:44 +0200 Subject: [PATCH 34/34] fix(snpclustering): drop gawk; consume PLINK2 eigenvec directly --- .../custom/clustervisualization/meta.yml | 10 +-- modules/nf-core/custom/pcaclustering/meta.yml | 11 ++-- .../snpclustering/awk/eigenvec_to_tsv.awk | 19 ------ subworkflows/nf-core/snpclustering/main.nf | 61 +++---------------- subworkflows/nf-core/snpclustering/meta.yml | 14 +---- .../nf-core/snpclustering/tests/main.nf.test | 1 - .../snpclustering/tests/main.nf.test.snap | 44 +++---------- .../snpclustering/tests/nextflow.config | 3 - 8 files changed, 30 insertions(+), 133 deletions(-) delete mode 100644 subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk diff --git a/modules/nf-core/custom/clustervisualization/meta.yml b/modules/nf-core/custom/clustervisualization/meta.yml index 0b71dedc9490..7f3abf907165 100644 --- a/modules/nf-core/custom/clustervisualization/meta.yml +++ b/modules/nf-core/custom/clustervisualization/meta.yml @@ -35,7 +35,7 @@ input: numeric features (e.g. PCA scores). pattern: "*.{tsv,txt,eigenvec}" ontologies: - - edam: http://edamontology.org/format_3475 # TSV + - edam: http://edamontology.org/format_3475 - clusters: type: file description: | @@ -43,7 +43,7 @@ input: `cluster` columns. Label -1 is treated as DBSCAN noise. pattern: "*.csv" ontologies: - - edam: http://edamontology.org/format_3752 # CSV + - edam: http://edamontology.org/format_3752 output: umap_tsv: - - meta: @@ -55,7 +55,7 @@ output: pattern: "*.umap.tsv" ontologies: - edam: "http://edamontology.org/operation_2432" - - edam: http://edamontology.org/format_3475 # TSV + - edam: http://edamontology.org/format_3475 tsne_tsv: - - meta: type: map @@ -66,7 +66,7 @@ output: pattern: "*.tsne.tsv" ontologies: - edam: "http://edamontology.org/operation_2432" - - edam: http://edamontology.org/format_3475 # TSV + - edam: http://edamontology.org/format_3475 umap_png: - - meta: type: map @@ -91,7 +91,7 @@ output: description: "Software versions used in the module" pattern: "versions.yml" ontologies: - - edam: http://edamontology.org/format_3750 # YAML + - edam: http://edamontology.org/format_3750 topics: versions: - versions.yml: diff --git a/modules/nf-core/custom/pcaclustering/meta.yml b/modules/nf-core/custom/pcaclustering/meta.yml index 76ed03367b9d..4b867e771edc 100644 --- a/modules/nf-core/custom/pcaclustering/meta.yml +++ b/modules/nf-core/custom/pcaclustering/meta.yml @@ -1,5 +1,6 @@ name: "CUSTOM_PCACLUSTERING" -description: "Performs KMeans or DBSCAN clustering on a sample-by-feature numeric matrix (e.g. principal components, embeddings)" +description: "Performs KMeans or DBSCAN clustering on a sample-by-feature numeric + matrix (e.g. principal components, embeddings)" keywords: - clustering - kmeans @@ -32,7 +33,7 @@ input: `sample_id`. pattern: "*.{tsv,txt,eigenvec}" ontologies: - - edam: http://edamontology.org/format_3475 # TSV + - edam: http://edamontology.org/format_3475 - algorithm: type: string description: Clustering algorithm to use (kmeans or dbscan) @@ -55,7 +56,7 @@ output: description: CSV file with sample_id and assigned cluster pattern: "*.clusters.csv" ontologies: - - edam: http://edamontology.org/format_3752 # CSV + - edam: http://edamontology.org/format_3752 info: - - meta: type: map @@ -65,14 +66,14 @@ output: description: JSON file with clustering parameters and statistics pattern: "*.clustering_info.json" ontologies: - - edam: http://edamontology.org/format_3464 # JSON + - edam: http://edamontology.org/format_3464 versions: - "versions.yml": type: file description: File containing software versions pattern: "versions.yml" ontologies: - - edam: http://edamontology.org/format_3750 # YAML + - edam: http://edamontology.org/format_3750 topics: versions: - versions.yml: diff --git a/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk b/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk deleted file mode 100644 index be40fe450b82..000000000000 --- a/subworkflows/nf-core/snpclustering/awk/eigenvec_to_tsv.awk +++ /dev/null @@ -1,19 +0,0 @@ -BEGIN { - FS = OFS = "\t" -} - -NR == 1 { - sub(/^#/, "") - $0 = $0 - for (i = 1; i <= NF; i++) { - if ($i == "IID") { - $i = "sample_id" - } - } - print - next -} - -{ - print -} diff --git a/subworkflows/nf-core/snpclustering/main.nf b/subworkflows/nf-core/snpclustering/main.nf index 963e33dcba1c..dc312450f280 100644 --- a/subworkflows/nf-core/snpclustering/main.nf +++ b/subworkflows/nf-core/snpclustering/main.nf @@ -1,10 +1,9 @@ -include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' -include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf' -include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca' -include { GAWK as GAWK_EIGENVEC_TO_TSV } from '../../../modules/nf-core/gawk' -include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering' -include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics' -include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization' +include { BEAGLE5_BEAGLE } from '../../../modules/nf-core/beagle5/beagle' +include { PLINK2_VCF } from '../../../modules/nf-core/plink2/vcf' +include { PLINK2_PCA } from '../../../modules/nf-core/plink2/pca' +include { CUSTOM_PCACLUSTERING } from '../../../modules/nf-core/custom/pcaclustering' +include { CUSTOM_CLUSTERMETRICS } from '../../../modules/nf-core/custom/clustermetrics' +include { CUSTOM_CLUSTERVISUALIZATION } from '../../../modules/nf-core/custom/clustervisualization' workflow SNPCLUSTERING { take: @@ -20,17 +19,6 @@ workflow SNPCLUSTERING { dbscan_min_samples main: - // Bundled awk program (see ./awk/eigenvec_to_tsv.awk); shipped alongside - // the subworkflow and passed to GAWK_EIGENVEC_TO_TSV as `program_file` so - // the awk source stays readable as awk rather than as an escaped Groovy - // string. - ch_eigenvec_to_tsv_awk = file("${moduleDir}/awk/eigenvec_to_tsv.awk", checkIfExists: true) - - /* - * Build BEAGLE input tuple: - * tuple val(meta), path(vcf), path(vcf_index), path(refpanel), path(refpanel_index), - * path(genmap), path(exclsamples), path(exclmarkers), val(region) - */ ch_beagle_input = vcf_ch.map { meta, vcf, vcf_index -> tuple( meta, @@ -46,16 +34,8 @@ workflow SNPCLUSTERING { } BEAGLE5_BEAGLE(ch_beagle_input) - - /* - * Convert imputed VCF to PLINK2 pfiles - */ PLINK2_VCF(BEAGLE5_BEAGLE.out.vcf) - /* - * PLINK2_PCA expects: - * tuple val(meta), val(npcs), val(use_approx), path(pgen), path(psam), path(pvar) - */ ch_plink_pca_input = PLINK2_VCF.out.pgen .join(PLINK2_VCF.out.pvar) .join(PLINK2_VCF.out.psam) @@ -65,38 +45,18 @@ workflow SNPCLUSTERING { PLINK2_PCA(ch_plink_pca_input) - /* - * Convert .eigenvec to a tab-separated file for downstream clustering - * modules: strips the leading '#' from the header and renames the IID - * column to sample_id. Output keeps its default naming (.eigenvec) - * — only the tab-separated content matters downstream, not the extension. - */ - GAWK_EIGENVEC_TO_TSV( - PLINK2_PCA.out.evecfile, - ch_eigenvec_to_tsv_awk, - false - ) - - /* - * PCA clustering - * CUSTOM_PCACLUSTERING(tsv, algorithm, n_clusters, dbscan_eps, dbscan_min_samples) - */ CUSTOM_PCACLUSTERING( - GAWK_EIGENVEC_TO_TSV.out.output, + PLINK2_PCA.out.evecfile, algorithm, n_clusters, dbscan_eps, dbscan_min_samples ) - /* - * Metrics and visualization both expect one tuple input channel - * built from: meta + tsv + cluster assignments - */ - ch_cluster_analysis_input = GAWK_EIGENVEC_TO_TSV.out.output + ch_cluster_analysis_input = PLINK2_PCA.out.evecfile .join(CUSTOM_PCACLUSTERING.out.clusters) - .map { meta, tsv, clusters -> - tuple(meta, tsv, clusters) + .map { meta, eigenvec, clusters -> + tuple(meta, eigenvec, clusters) } CUSTOM_CLUSTERMETRICS(ch_cluster_analysis_input) @@ -111,7 +71,6 @@ workflow SNPCLUSTERING { evecfile = PLINK2_PCA.out.evecfile evfile = PLINK2_PCA.out.evfile pca_log = PLINK2_PCA.out.logfile - tsv = GAWK_EIGENVEC_TO_TSV.out.output clusters = CUSTOM_PCACLUSTERING.out.clusters cluster_info = CUSTOM_PCACLUSTERING.out.info metrics = CUSTOM_CLUSTERMETRICS.out.metrics diff --git a/subworkflows/nf-core/snpclustering/meta.yml b/subworkflows/nf-core/snpclustering/meta.yml index 1d57e69d8915..6ba3f0622b02 100644 --- a/subworkflows/nf-core/snpclustering/meta.yml +++ b/subworkflows/nf-core/snpclustering/meta.yml @@ -2,8 +2,7 @@ name: "snpclustering" description: > Subworkflow for SNP clustering. Takes genotyped VCF files, imputes missing - genotypes with BEAGLE5, converts to PLINK2 pgen format, performs PCA, - converts eigenvec to TSV with awk, then runs PCA-based clustering, + genotypes with BEAGLE5, converts to PLINK2 pgen format, performs PCA,then runs PCA-based clustering, cluster metrics, and cluster visualization. keywords: - snp @@ -17,7 +16,6 @@ components: - beagle5/beagle - plink2/vcf - plink2/pca - - gawk - custom/pcaclustering - custom/clustermetrics - custom/clustervisualization @@ -142,16 +140,6 @@ output: type: file description: PLINK2 PCA log file pattern: "*.log" - - tsv: - description: TSV converted from eigenvec by gawk - structure: - - meta: - type: map - description: Groovy map containing sample metadata - - tsv: - type: file - description: TSV converted from eigenvec by gawk - pattern: "*.eigenvec" - clusters: description: Cluster assignments structure: diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test b/subworkflows/nf-core/snpclustering/tests/main.nf.test index 77c8398c2bfa..29a3e0e2d9d3 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test @@ -12,7 +12,6 @@ nextflow_workflow { tag "plink2/vcf" tag "plink2/pca" tag "beagle5/beagle" - tag "gawk" tag "custom/pcaclustering" tag "custom/clustermetrics" tag "custom/clustervisualization" diff --git a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap index eb11e4cc6b4c..48255cca9687 100644 --- a/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap +++ b/subworkflows/nf-core/snpclustering/tests/main.nf.test.snap @@ -67,6 +67,7 @@ "test.calinski_harabasz.png", "test.davies_bouldin.png", "test.elbow.png", + "test.k_distance.png", "test.silhouette.png" ] ] @@ -135,14 +136,6 @@ "test.tsne.tsv" ] ], - "tsv": [ - [ - { - "id": "test" - }, - "test.tsv:md5,f5ec5abd246d89cf50d333d432e4366a" - ] - ], "umap_png": [ [ { @@ -188,7 +181,7 @@ "n_cols": 4 } ], - "timestamp": "2026-08-31T12:49:22.352660673", + "timestamp": "2026-09-09T10:30:31.743191089", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -262,6 +255,7 @@ "test.calinski_harabasz.png", "test.davies_bouldin.png", "test.elbow.png", + "test.k_distance.png", "test.silhouette.png" ] ] @@ -330,14 +324,6 @@ "test.tsne.tsv" ] ], - "tsv": [ - [ - { - "id": "test" - }, - "test.tsv:md5,f5ec5abd246d89cf50d333d432e4366a" - ] - ], "umap_png": [ [ { @@ -385,7 +371,7 @@ "n_cols": 4 } ], - "timestamp": "2026-08-31T12:49:54.909445446", + "timestamp": "2026-09-09T10:31:04.29645672", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -459,6 +445,7 @@ "test.calinski_harabasz.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.davies_bouldin.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.elbow.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.k_distance.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.silhouette.png:md5,d41d8cd98f00b204e9800998ecf8427e" ] ] @@ -527,14 +514,6 @@ "test.tsne.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "tsv": [ - [ - { - "id": "test" - }, - "test.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], "umap_png": [ [ { @@ -553,7 +532,7 @@ ] } ], - "timestamp": "2026-08-31T12:14:11.172911335", + "timestamp": "2026-09-09T10:31:26.336556716", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" @@ -627,6 +606,7 @@ "test.calinski_harabasz.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.davies_bouldin.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.elbow.png:md5,d41d8cd98f00b204e9800998ecf8427e", + "test.k_distance.png:md5,d41d8cd98f00b204e9800998ecf8427e", "test.silhouette.png:md5,d41d8cd98f00b204e9800998ecf8427e" ] ] @@ -695,14 +675,6 @@ "test.tsne.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" ] ], - "tsv": [ - [ - { - "id": "test" - }, - "test.tsv:md5,d41d8cd98f00b204e9800998ecf8427e" - ] - ], "umap_png": [ [ { @@ -721,7 +693,7 @@ ] } ], - "timestamp": "2026-08-31T12:13:58.254898619", + "timestamp": "2026-09-09T10:31:16.00969678", "meta": { "nf-test": "0.9.5", "nextflow": "26.04.6" diff --git a/subworkflows/nf-core/snpclustering/tests/nextflow.config b/subworkflows/nf-core/snpclustering/tests/nextflow.config index c68232c7a163..b7b0b81220e5 100644 --- a/subworkflows/nf-core/snpclustering/tests/nextflow.config +++ b/subworkflows/nf-core/snpclustering/tests/nextflow.config @@ -20,7 +20,4 @@ process { ext.args = '--seed 1' } - withName: 'GAWK_EIGENVEC_TO_TSV' { - ext.suffix = 'tsv' - } }