Skip to content

optimizing resources, fix stagefiles for large datasets #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CaroAMN merged 4 commits into from
Jan 8, 2026
Merged

optimizing resources, fix stagefiles for large datasets #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
9a6aea9
optimizing resources, fix stagefiles for large datasets
CaroAMN Jan 2, 2026
1cdb825
update resource usage, fix input for large datasets
CaroAMN Jan 7, 2026
7addc4b
back to unzip
CaroAMN Jan 8, 2026
0fce8e5
update snapshot
CaroAMN Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions conf/base.config
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,19 @@ process {
ext.use_gpu = true
accelerator = 1
}
withName: 'NUMORPHINTENSITY' {
cpus = { 2 * task.attempt }
memory = { 30.GB * task.attempt }
time = { 2.h * task.attempt }
}
withName: 'NUMORPHALIGN' {
cpus = { 7 * task.attempt }
memory = { 6.GB * task.attempt }
time = { 11.h * task.attempt }
}
withName: 'NUMORPHSTITCH' {
cpus = { 2 * task.attempt }
memory = { 9.GB * task.attempt }
time = { 15.h * task.attempt }
}
}
4 changes: 1 addition & 3 deletions modules/local/numorphalign/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,15 +32,13 @@ process NUMORPHALIGN {
"""
mkdir -p results/samples/
mkdir -p results/variables/
mkdir -p ./images

ln -sr ${img_directory} ./images
ln -sr ${adj_params_mat} results/variables
ln -sr ${path_table_mat} results/variables
ln -sr ${thresholds_mat} results/variables

# resolve symlinks and paths
img_dir=\$(readlink -f ./images)
img_dir=\$(readlink -f ${img_directory})
parameter_file=\$(readlink -f ${parameter_file})
results_dir=\$(readlink -f ./results)
NM_variables=\$(readlink -f ${NM_variables})
Expand Down
5 changes: 1 addition & 4 deletions modules/local/numorphintensity/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,9 @@ process NUMORPHINTENSITY {

"""
mkdir -p ./results
mkdir -p ./images

ln -sr ${img_directory} ./images

# resolve symlinks and paths
img_dir=\$(readlink -f ./images)
img_dir=\$(readlink -f ${img_directory})
parameter_file=\$(readlink -f ${parameter_file})
results_dir=\$(readlink -f ./results)

Expand Down
4 changes: 1 addition & 3 deletions modules/local/numorphstitch/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,15 @@ process NUMORPHSTITCH {
"""

mkdir -p results/variables/
mkdir -p ./images

ln -sr ${img_directory} ./images
ln -sr ${alignment_table_mat} results/variables
ln -sr ${z_displacement_align_mat} results/variables
ln -sr ${thresholds_mat} results/variables
ln -sr ${adj_params_mat} results/variables
ln -sr ${path_table_mat} results/variables

# resolve symlinks and paths
img_dir=\$(readlink -f ./images)
img_dir=\$(readlink -f ${img_directory})
parameter_file=\$(readlink -f ${parameter_file})
results_dir=\$(readlink -f ./results)
NM_variables=\$(readlink -f ${NM_variables})
Expand Down
4 changes: 2 additions & 2 deletions modules/local/stagefiles/main.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ process STAGEFILES {
tuple val(meta), path(file_directory) // directory containing files

output:
tuple val(meta), path("${meta.id}_raw/**") , emit: raw_files
tuple val(meta), path("${meta.id}_raw/*") , emit: raw_files
path "versions.yml" , emit: versions

when:
Expand All @@ -29,7 +29,7 @@ process STAGEFILES {
# This process will stage the data into the workdir and then it will be symlinked

mkdir -p ${prefix}_raw
ln -sr ${file_directory}/* ${prefix}_raw/
ln -sr ${file_directory} ${prefix}_raw/

cat <<-END_VERSIONS > versions.yml
"${task.process}":
Expand Down
358 changes: 178 additions & 180 deletions tests/default.nf.test.snap
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 4 additions & 4 deletions workflows/lsmquant.nf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ include { methodsDescriptionText } from '../subworkflows/local/utils_nfcore_lsmq
include { MAT2JSON } from '../modules/local/mat2json'
include { NUMORPH3DUNET } from '../modules/local/numorph3dunet'
include { UNZIPFILES } from '../modules/nf-core/unzipfiles'
include { UNZIP } from '../modules/nf-core/unzip'
include { STAGEFILES } from '../modules/local/stagefiles'
include { MULTIQC } from '../modules/nf-core/multiqc'

Expand All @@ -37,17 +38,16 @@ workflow LSMQUANT {
// if test profile then first data needs to be unzipped
if ( workflow.profile.contains('test') ) {
params.stage = 'preprocessing'

samplesheet
.map { meta, img_directory, parameter_file ->
tuple(meta, img_directory)
}
.set { img_archive }

UNZIPFILES (img_archive)
ch_versions = ch_versions.mix(UNZIPFILES.out.versions)
UNZIP (img_archive)
ch_versions = ch_versions.mix(UNZIP.out.versions)

def unzipped_output = UNZIPFILES.out.files
def unzipped_output = UNZIP.out.unzipped_archive

unzipped_output
.join(samplesheet)
Expand Down
Loading