From c5c48461147521f1001170b353f953e38557d516 Mon Sep 17 00:00:00 2001 From: Jon Clucas Date: Tue, 1 Nov 2022 09:58:11 -0400 Subject: [PATCH 1/2] :wrench: Add `fail_fast` to expose Nipype's `stop_on_first_crash` Co-authored-by: Greg Kiar --- CHANGELOG.md | 1 + CPAC/pipeline/cpac_pipeline.py | 6 +++--- CPAC/pipeline/schema.py | 1 + CPAC/resources/configs/pipeline_config_blank.yml | 2 ++ CPAC/resources/configs/pipeline_config_default.yml | 3 ++- CPAC/resources/configs/pipeline_config_rbc-options.yml | 2 ++ dev/docker_data/run.py | 7 +++++++ 7 files changed, 18 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f84bde5b..f5db8f85b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the ability to use TotalReadoutTime of epi field maps in the calculation of FSL topup distortion correction. - Difference method (``-``) for ``CPAC.utils.configuration.Configuration`` instances - Calculate reho and alff when timeseries in template space +- Added ``fail_fast`` configuration setting and CLI flag ### Changed - Added a level of depth to `working` directories to match `log` and `output` directory structure diff --git a/CPAC/pipeline/cpac_pipeline.py b/CPAC/pipeline/cpac_pipeline.py index 14ce239602..2131baf14a 100644 --- a/CPAC/pipeline/cpac_pipeline.py +++ b/CPAC/pipeline/cpac_pipeline.py @@ -281,9 +281,9 @@ def run_workflow(sub_dict, c, run, pipeline_timing_info=None, p_name=None, }, 'execution': { 'crashfile_format': 'txt', - 'resource_monitor_frequency': 0.2 - } - }) + 'resource_monitor_frequency': 0.2, + 'stop_on_first_crash': c['pipeline_setup', 'system_config', + 'fail_fast']}}) config.enable_resource_monitor() logging.update_logging(config) diff --git a/CPAC/pipeline/schema.py b/CPAC/pipeline/schema.py index 19764b6cfb..2cf1ce94bc 100644 --- a/CPAC/pipeline/schema.py +++ b/CPAC/pipeline/schema.py @@ -265,6 +265,7 @@ def sanitize(filename): 'path': Maybe(str), }, 'system_config': { + 'fail_fast': bool1_1, 'FSLDIR': Maybe(str), 'on_grid': { 'run': bool1_1, diff --git a/CPAC/resources/configs/pipeline_config_blank.yml b/CPAC/resources/configs/pipeline_config_blank.yml index 09ff784f42..50988c43b6 100644 --- a/CPAC/resources/configs/pipeline_config_blank.yml +++ b/CPAC/resources/configs/pipeline_config_blank.yml @@ -57,6 +57,8 @@ pipeline_setup: system_config: + fail_fast: Off + # Random seed used to fix the state of execution. # If unset, each process uses its own default. # If set, a `random.log` file will be generated logging the random seed and each node to which that seed was applied. diff --git a/CPAC/resources/configs/pipeline_config_default.yml b/CPAC/resources/configs/pipeline_config_default.yml index be84b46008..3b8660f1f6 100644 --- a/CPAC/resources/configs/pipeline_config_default.yml +++ b/CPAC/resources/configs/pipeline_config_default.yml @@ -85,7 +85,8 @@ pipeline_setup: path: /outputs/crash system_config: - + # Stop worklow execution on first crash? + fail_fast: Off # Random seed used to fix the state of execution. # If unset, each process uses its own default. # If set, a `random.log` file will be generated logging the random seed and each node to which that seed was applied. diff --git a/CPAC/resources/configs/pipeline_config_rbc-options.yml b/CPAC/resources/configs/pipeline_config_rbc-options.yml index b79a016de3..8953103261 100644 --- a/CPAC/resources/configs/pipeline_config_rbc-options.yml +++ b/CPAC/resources/configs/pipeline_config_rbc-options.yml @@ -23,6 +23,8 @@ pipeline_setup: system_config: + fail_fast: On + # Random seed used to fix the state of execution. # If unset, each process uses its own default. # If set, a `random.log` file will be generated logging the random seed and each node to which that seed was applied. diff --git a/dev/docker_data/run.py b/dev/docker_data/run.py index f4814a8c5e..ac805464df 100755 --- a/dev/docker_data/run.py +++ b/dev/docker_data/run.py @@ -28,6 +28,7 @@ from CPAC import license_notice, __version__ from CPAC.pipeline import AVAILABLE_PIPELINE_CONFIGS from CPAC.pipeline.random_state import set_up_random_state +from CPAC.pipeline.schema import str_to_bool1_1 from CPAC.utils.bids_utils import create_cpac_data_config, \ load_cpac_data_config, \ load_yaml_config, \ @@ -248,6 +249,8 @@ def run_main(): help='Save the contents of the working directory.', default=False) + parser.add_argument('--fail_fast', type=str.title, + help='Stop worklow execution on first crash?') parser.add_argument('--participant_label', help='The label of the participant that should be ' 'analyzed. The label corresponds to ' @@ -604,6 +607,10 @@ def run_main(): 'Either change the output directory to something ' 'local or turn off the --save_working_dir flag') + if args.fail_fast is not None: + c['pipeline_setup', 'system_config', + 'fail_fast'] = str_to_bool1_1(args.fail_fast) + if c['pipeline_setup']['output_directory']['quality_control'][ 'generate_xcpqc_files']: c['functional_preproc']['motion_estimates_and_correction'][ From e0ca85b184c95e4fede91c1d0e1e22c1b6923ce2 Mon Sep 17 00:00:00 2001 From: "Theodore (Machine User)" Date: Tue, 1 Nov 2022 13:59:42 +0000 Subject: [PATCH 2/2] :bulb: Update comments based on default preconfig --- CPAC/resources/configs/pipeline_config_blank.yml | 4 ++-- CPAC/resources/configs/pipeline_config_rbc-options.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CPAC/resources/configs/pipeline_config_blank.yml b/CPAC/resources/configs/pipeline_config_blank.yml index 50988c43b6..cd326f63b0 100644 --- a/CPAC/resources/configs/pipeline_config_blank.yml +++ b/CPAC/resources/configs/pipeline_config_blank.yml @@ -57,6 +57,7 @@ pipeline_setup: system_config: + # Stop worklow execution on first crash? fail_fast: Off # Random seed used to fix the state of execution. @@ -1193,8 +1194,7 @@ functional_preproc: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [Off] - - space: 'native' + space: native nuisance_corrections: 2-nuisance_regression: diff --git a/CPAC/resources/configs/pipeline_config_rbc-options.yml b/CPAC/resources/configs/pipeline_config_rbc-options.yml index 8953103261..a404a5e4ae 100644 --- a/CPAC/resources/configs/pipeline_config_rbc-options.yml +++ b/CPAC/resources/configs/pipeline_config_rbc-options.yml @@ -23,6 +23,7 @@ pipeline_setup: system_config: + # Stop worklow execution on first crash? fail_fast: On # Random seed used to fix the state of execution. @@ -112,7 +113,7 @@ functional_preproc: # this is a fork point # run: [On, Off] - this will run both and fork the pipeline run: [On] - space: 'template' + space: template nuisance_corrections: 2-nuisance_regression: