diff --git a/.travis.yml b/.travis.yml index fdd912d..098ebe5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,10 +2,10 @@ sudo: false language: python python: - - "3.6" - "3.7" - "3.8" - - "3.9-dev" + - "3.9" + - "3.10-dev" install: - pip install codecov diff --git a/README.md b/README.md index c687c6d..5ce0e96 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ yet another layer of complexity. Gromax is here to help. ### Via pip To install the most recent release: ``` -pip install git+https://github.com/scal444/gromax@v0.1.0 +pip install git+https://github.com/scal444/gromax@v0.2.0 ``` To install the current master version: @@ -25,10 +25,8 @@ To install the current master version: pip install git+https://github.com/scal444/gromax ``` -TODO: Add branches as they come online - ### Requirements -- python 3.6 or greater +- python 3.7 or greater ## Capabilities - Given a Gromacs TPR file and a description of the hardware (CPU count and GPU IDs), generate a series of Gromacs run @@ -56,6 +54,9 @@ in [the examples doc](docs/examples.md)! - Feel free to file an issue bug/feature request, or create a PR. There is a [known issues doc](docs/known_issues.md) for problems that are known but can't yet be addressed. +## Release notes +- Release notes can be found in [docs/releaes_notes](docs/release_notes) + ## Other awesome resources - Want to see how well your system scales to various clusters? Check out [MDBenchmark](https://github.com/bio-phys/mdbenchmark)! diff --git a/admin/regenerate_reference_data.sh b/admin/regenerate_reference_data.sh index d61c7b6..ff7e27d 100644 --- a/admin/regenerate_reference_data.sh +++ b/admin/regenerate_reference_data.sh @@ -14,37 +14,50 @@ RUNBASE="generate --cpu_ids=0-3 --gpu_ids=0,1" OUTDIR="$topdir/gromax/tests/integration/testdata" # Basic 2016 test -$PYTHON "$exe" ${RUNBASE} --run_file="${OUTDIR}"/generate_test_default_2016.sh --gmx_version=2016 +$PYTHON "$exe" ${RUNBASE} --run_file="${OUTDIR}"/generate_test_default_2016.sh --gmx_version=2016 \ + --generate_exhaustive_combinations # Basic 2018 test -$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2018.sh --gmx_version=2018 +$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2018.sh --gmx_version=2018 \ + --generate_exhaustive_combinations # basic 2019 test -$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2019.sh --gmx_version=2019 +$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2019.sh --gmx_version=2019 \ + --generate_exhaustive_combinations # basic 2020 test -$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2020.sh --gmx_version=2020 +$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2020.sh --gmx_version=2020 \ + --generate_exhaustive_combinations # custom executable test $PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_custom_exe.sh --gmx_version=2016 \ - --gmx_executable=/path/to/gmx_mpi + --generate_exhaustive_combinations --gmx_executable=/path/to/gmx_mpi # custom trials test $PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_custom_ntrials.sh --gmx_version=2016 \ - --trials_per_group=18 + --generate_exhaustive_combinations --trials_per_group=18 # custom directory is ignored for generate $PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_default_2016.sh --gmx_version=2016 \ - --directory=/path/to/nowhere + --generate_exhaustive_combinations --directory=/path/to/nowhere # custom tpr $PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_custom_tpr.sh --gmx_version=2016 \ - --tpr=custom_tpr.tpr + --generate_exhaustive_combinations --tpr=custom_tpr.tpr # Test with strange CPU/GPU count that still works -$PYTHON "$exe" generate --cpu_ids=0-6 --gpu_ids=0,1 --run_file="$OUTDIR"/generate_test_odd_cpu_count.sh --gmx_version=2016 +$PYTHON "$exe" generate --cpu_ids=0-6 --gpu_ids=0,1 --run_file="$OUTDIR"/generate_test_odd_cpu_count.sh \ + --gmx_version=2016 --generate_exhaustive_combinations # Test the single_sim_only argument -$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_single_sim_only.sh --gmx_version=2016 --single_sim_only +$PYTHON "$exe" ${RUNBASE} --run_file="$OUTDIR"/generate_test_single_sim_only.sh --gmx_version=2016 --single_sim_only \ + --generate_exhaustive_combinations +# Test minimal subset only +$PYTHON "$exe" generate --cpu_ids=0-3 --gpu_ids=0 --run_file="$OUTDIR"/generate_test_minimal_subset.sh \ + --gmx_version=2020 + +# Test minimal subset with single sim. +$PYTHON "$exe" generate --cpu_ids=0-3 --gpu_ids=0 --run_file="$OUTDIR"/generate_test_minimal_subset_single_sim.sh \ + --gmx_version=2020 --single_sim_only exit 0 diff --git a/docs/examples.md b/docs/examples.md index abe9fc4..af9f6a0 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -41,6 +41,20 @@ gromax generate --gmx_version=2020 --cpu_ids=0:2:7 --gpu_ids=0 gromax generate --gmx_version=2020 --cpu_ids=0:7 --gpu_ids=0 --single_sim_only ``` +#### Exhaustive vs minimal configurations +Since Gromax v1.1, the `--generate_exhaustive_combinations` flag can be used to tune the number of configs Gromax +creates. It set to false by default, which adds the following limits: + +* combinations of -pme and (where applicable to the Gromacs version) -bonded -update must be the same. E.g. a combination + such as -pme gpu -bonded cpu -update cpu is not created. +* A cap of 2 simultaneous simulations per GPU is set. + +This significantly cuts down on the number of combinations generated, while not sacrificing much in coverage. Exhaustive +configs can still be generated by adding in the flag: +```shell script +gromax generate --gmx_version=2020 --cpu_ids=0:7 --gpu_ids=0 --generate_exhaustive_combinations +``` + #### Some configuration options for the run script. *NOTE: Each of these options can also easily be set in the first few lines of the benchmark script.* ```shell script diff --git a/docs/release_notes/release_notes_0.2.md b/docs/release_notes/release_notes_0.2.md new file mode 100644 index 0000000..c147dfe --- /dev/null +++ b/docs/release_notes/release_notes_0.2.md @@ -0,0 +1,11 @@ +## Gromax v0.2.0 +### Features +* Gromacs 2021 is now supported. Note that support of advanced GPU options is not yet implemented, + but any features that worked for Gromacs 2020 should work for Gromacs 2021. +* Reduced the default number of combinations generated. The default combinations + should still come close to maximizing performance. Exhausive benchmark generation + can still be requested using the `--generate_exhaustive_combinations` flag. + See [the examples doc](../examples.md) for more details. +### Compatibility +* Python 3.7 is now required. + diff --git a/gromax/combination_generator.py b/gromax/combination_generator.py index c98e682..6fa700c 100644 --- a/gromax/combination_generator.py +++ b/gromax/combination_generator.py @@ -1,14 +1,11 @@ import math +from dataclasses import dataclass + +from gromax.constants import _SUPPORTED_GMX_VERSIONS from gromax.hardware_config import HardwareConfig from copy import deepcopy -from typing import List, Dict, Set, Any, Callable - -# Constants -_SUPPORTED_VERSIONS: Set[str] = {"2016", "2018", "2019", "2020", "2021"} - -# TODO: consolidate constants like this and potentially make configurable -_MAX_RANKS_PME_GPU = 8 +from typing import List, Dict, Any, Callable # Convenience definitions # A grouping of GPU ids. @@ -21,7 +18,17 @@ HardwareConfigBreakdown = List[HardwareConfig] -def genNtmpiOptions(total_procs: int, num_gpus: int, max_sims_per_gpu: int = 4) -> List[int]: +@dataclass +class GenerateOptions: + """ + Consolidated options for parameter combination generation. + """ + max_sims_per_gpu: int = 4 + max_ranks_for_pme_gpu: int = 8 + generate_exhaustive_options: bool = True + + +def genNtmpiOptions(total_procs: int, num_gpus: int, max_ranks_per_gpu: int = 4) -> List[int]: """ Breaks down the possible combinations of ntmpi for the given number of GPUs. For example, with 2 GPUs and 6 CPUs, the combinations are: @@ -31,8 +38,6 @@ def genNtmpiOptions(total_procs: int, num_gpus: int, max_sims_per_gpu: int = 4) Note that in this example ntmpi=3 does not work, because you can't eveny split 2 GPUs among 3 ranks. - Will not assign more than max_sims_per_gpu simulations to a single GPU - If passed with no GPUs, returns a size one list with the number of processors, as non-GPU simulations should maximize thread counts. """ @@ -42,7 +47,7 @@ def genNtmpiOptions(total_procs: int, num_gpus: int, max_sims_per_gpu: int = 4) return [total_procs] curroption: int = num_gpus options: List[int] = [] - while curroption <= total_procs and math.ceil(curroption / num_gpus) <= max_sims_per_gpu: + while curroption <= total_procs and math.ceil(curroption / num_gpus) <= max_ranks_per_gpu: if total_procs % curroption == 0 and curroption % num_gpus == 0: options.append(curroption) curroption += 1 @@ -135,7 +140,25 @@ def pruneOptionIf(parameters: List[ParameterSet], predicate: Callable[[Parameter return [option for option in parameters if not predicate(option)] -def _createVersionedOptions(base_opts: ParameterSet, hw_config: HardwareConfig, gmx_version: str) -> ParameterSetGroup: +def _pruneUnlikelyCombinations(parameters: List[ParameterSet]) -> List[ParameterSet]: + """ + Removes combinations where PME != bonded != update - such as: + + * PME GPU with bonded or update CPU + * PME CPU with bonded or update GPU + """ + def removalPredicate(params: ParameterSet) -> bool: + pme = params["pme"] + # For earlier versions without these options (eg. 2016), + # set the other options to PME so the inequality isn't triggered. + update = params.get("update", pme) + bonded = params.get("bonded", pme) + return pme != update or pme != bonded + return pruneOptionIf(parameters, removalPredicate) + + +def _createVersionedOptions(base_opts: ParameterSet, hw_config: HardwareConfig, gmx_version: str, + generate_options: GenerateOptions) -> ParameterSetGroup: """ Given a partial base parameter set, a hardware config, and a target Gromacs version, creates all of the parameter combination possibilities. @@ -161,12 +184,11 @@ def _createVersionedOptions(base_opts: ParameterSet, hw_config: HardwareConfig, # Cap max ranks for PME GPU def excessRanksPredicate(params: ParameterSet) -> bool: - return params["pme"] == "gpu" and params["ntmpi"] > _MAX_RANKS_PME_GPU + return params["pme"] == "gpu" and params["ntmpi"] > generate_options.max_ranks_for_pme_gpu options = pruneOptionIf(options, excessRanksPredicate) # set npme if more than one rank. def nPmePredicate(params: ParameterSet) -> bool: - # TODO verify that this is correct on all versions return params["pme"] == "gpu" and params["ntmpi"] > 1 options = applyOptionIf(options, "npme", 1, nPmePredicate) if gmx_version >= "2019": @@ -180,6 +202,8 @@ def nPmePredicate(params: ParameterSet) -> bool: def multiRankPredicate(params: ParameterSet): return params["pme"] == "cpu" and params["update"] == "gpu" and params["ntmpi"] > 1 options = pruneOptionIf(options, multiRankPredicate) + if not generate_options.generate_exhaustive_options: + options = _pruneUnlikelyCombinations(options) # Add gputasks for opt in options: if opt.get("nb") == "gpu": @@ -189,11 +213,11 @@ def multiRankPredicate(params: ParameterSet): def _versionIsValid(version: str): - # TODO move this more central when sanitizing user input - return version in _SUPPORTED_VERSIONS + return version in _SUPPORTED_GMX_VERSIONS -def createRunOptionsForSingleConfig(hw_config: HardwareConfig, gmx_version: str) -> ParameterSetGroup: +def createRunOptionsForSingleConfig(hw_config: HardwareConfig, gmx_version: str, + generate_options: GenerateOptions) -> ParameterSetGroup: """ Generate the set of possible run options for a specific hardware config. @@ -201,12 +225,13 @@ def createRunOptionsForSingleConfig(hw_config: HardwareConfig, gmx_version: str) (though the ordering can be arbitrary). """ - options: ParameterSet = _createBaseOptions() - addConfigDependentOptions(options, hw_config) - return _createVersionedOptions(options, hw_config, gmx_version) + base_options: ParameterSet = _createBaseOptions() + addConfigDependentOptions(base_options, hw_config) + return _createVersionedOptions(base_options, hw_config, gmx_version, generate_options) -def createRunOptionsForConfigGroup(configs: HardwareConfigBreakdown, gmx_version: str) -> List[ParameterSetGroup]: +def createRunOptionsForConfigGroup(configs: HardwareConfigBreakdown, gmx_version: str, + generate_options: GenerateOptions) -> List[ParameterSetGroup]: """ Generate all the parameter combinations for a given subconfiguration of the hardware. @@ -221,11 +246,12 @@ def createRunOptionsForConfigGroup(configs: HardwareConfigBreakdown, gmx_version if not _versionIsValid(gmx_version): raise ValueError("Invalid Gromacs version: {}. Supported options are {}".format(gmx_version, - _SUPPORTED_VERSIONS)) + _SUPPORTED_GMX_VERSIONS)) # This gets us all the combinations we want, but with the wrong structure. The top level is for each partial # hardware config, and the second level is over the options within each subconfig. - breakdowns_per_config: List[ParameterSetGroup] = [createRunOptionsForSingleConfig(config, gmx_version) + breakdowns_per_config: List[ParameterSetGroup] = [createRunOptionsForSingleConfig(config, gmx_version, + generate_options) for config in configs] # Now we reorder, inverting the organization such that # [[subconfig1_option1, subconfig1_option2], [subconfig2_option1, subconfig2_option2]] diff --git a/gromax/command_line.py b/gromax/command_line.py index e450a39..3f6ea25 100644 --- a/gromax/command_line.py +++ b/gromax/command_line.py @@ -1,7 +1,9 @@ import argparse -import logging from typing import List, Iterable +from gromax.constants import _SUPPORTED_GMX_VERSIONS, _GROMAX_VERSION +from gromax.utils import fatalError + # File constants. _DESCRIPTION = "Gromax is a tool to build benchmarking scripts for Gromax and analyze the results. \n" \ "Generate scripts with the 'gromax generate' command, and analyze results with 'gromax analyze'." @@ -51,12 +53,16 @@ def _buildParser() -> argparse.ArgumentParser: generate_group = parser.add_argument_group("generate", "arguments for 'gromax generate'") generate_group.add_argument('--gmx_version', type=str, metavar="", - help='Gromacs version - "2016", "2018", "2019", "2020", or "2021"', ) + help=f'Gromacs version - one of {sorted(set(_SUPPORTED_GMX_VERSIONS))}', ) generate_group.add_argument("--run_file", type=str, help="Path to bash benchmark script to create.", default="benchmark.sh", metavar="") generate_group.add_argument("--gmx_executable", type=str, default="gmx", metavar="", help=( "gmx or gmx_mpi executable path. Defaults to 'gmx', which works if the executable is in your path." )) + generate_group.add_argument("--generate_exhaustive_combinations", default=False, action="store_true", + help=("If set, Gromax will generate all possible combinations of PME/bonded/update " + "CPU/GPU options. If not set(by default), only options likely to have maximum " + "performance are generated.")) generate_group.add_argument("--trials_per_group", type=int, default=3, metavar="", help="Number of times to run each parameter set.") generate_group.add_argument("--tpr", type=str, help="Absolute path to the tpr file to benchmark.", metavar="") @@ -72,39 +78,34 @@ def _buildParser() -> argparse.ArgumentParser: help="If set, do not divide the hardware among multiple concurrent simulations") analyze_group = parser.add_argument_group("analyze", "arguments for 'gromax analyze'") analyze_group.add_argument("--directory", type=str, help="Path to execution/analysis directory.", metavar="") - parser.add_argument("--version", action="version", version="alpha") + parser.add_argument("--version", action="version", version=_GROMAX_VERSION) parser.add_argument("--log_level", type=str, default="info", metavar="", help="Set logging verbosity - 'silent', 'info'(default), or 'debug'") return parser -def _failWithError(err: str): - logging.getLogger("gromax").error(err) - raise SystemExit(1) - - def _checkGenerateArgs(args: argparse.Namespace) -> None: - good_versions: Iterable[str] = ("2016", "2018", "2019", "2020", "2021") - if args.gmx_version not in good_versions: - _failWithError("Invalid gmx version {}, must be one of {}".format(args.gmx_version, good_versions)) + if args.gmx_version not in _SUPPORTED_GMX_VERSIONS: + fatalError("Invalid gmx version {}, must be one of {}".format(args.gmx_version, + sorted(_SUPPORTED_GMX_VERSIONS))) if not args.cpu_ids and not args.num_cpus: - _failWithError("One of --cpu_ids or --num_cpus is required") + fatalError("One of --cpu_ids or --num_cpus is required") if args.num_cpus: if args.cpu_ids: - _failWithError("Cannot specify both --cpu_ids and --num_cpus") + fatalError("Cannot specify both --cpu_ids and --num_cpus") args.cpu_ids = ",".join([str(identifier) for identifier in range(args.num_cpus)]) if not args.gpu_ids and not args.num_gpus: - _failWithError("One of --gpu_ids or --num_gpus is required") + fatalError("One of --gpu_ids or --num_gpus is required") if args.num_gpus: if args.gpu_ids: - _failWithError("Cannot specify both --gpu_ids and --num_gpus") + fatalError("Cannot specify both --gpu_ids and --num_gpus") args.gpu_ids = ",".join([str(identifier) for identifier in range(args.num_gpus)]) def checkArgs(args: argparse.Namespace) -> None: good_modes: Iterable[str] = ("generate", "execute", "analyze") if args.mode not in good_modes: - _failWithError("'mode' is a required positional argument - options are 'generate', 'execute', 'analyze'") + fatalError("'mode' is a required positional argument - options are 'generate', 'execute', 'analyze'") if args.mode == "generate": _checkGenerateArgs(args) diff --git a/gromax/constants.py b/gromax/constants.py new file mode 100644 index 0000000..1eb6b5b --- /dev/null +++ b/gromax/constants.py @@ -0,0 +1,7 @@ +from typing import FrozenSet +""" + Contains program-level constants +""" +_GROMAX_VERSION = "1.2-dev" + +_SUPPORTED_GMX_VERSIONS: FrozenSet[str] = frozenset({"2016", "2018", "2019", "2020", "2021"}) diff --git a/gromax/file_io.py b/gromax/file_io.py index 680971c..0202571 100644 --- a/gromax/file_io.py +++ b/gromax/file_io.py @@ -140,7 +140,7 @@ def parseDirectoryStructure(directory: str) -> allDirectoryContent: return result -def _maxItems(content: allDirectoryContent): +def _maxItems(content: Dict): """ Given a nested dictionary, determine the max number of items in the first subdictionary. So given, { diff --git a/gromax/hardware_config.py b/gromax/hardware_config.py index 0901a44..69a47fb 100644 --- a/gromax/hardware_config.py +++ b/gromax/hardware_config.py @@ -56,12 +56,12 @@ def gpu_ids(self, gpu_ids: List[int]): try: for item in gpu_ids: if not isinstance(item, int): - utils.fatal_error("All gpu ids must be integers: {} is not an int".format(item)) + utils.fatalError("All gpu ids must be integers: {} is not an int".format(item)) if item < 0: - utils.fatal_error("Cannot have a negative GPU ID") + utils.fatalError("Cannot have a negative GPU ID") self._gpu_ids = gpu_ids except TypeError: - utils.fatal_error("gpu_ids paramater must be iterable") + utils.fatalError("gpu_ids paramater must be iterable") def __str__(self) -> str: return "\nHardware config:\n\tcpu IDs : {}\n\tgpu IDs : {}".format(self._cpu_ids, self._gpu_ids) @@ -124,8 +124,6 @@ def generateConfigSplitOptions(hw_config: HardwareConfig, max_sims_per_gpu: int configsplit1 should add up to the entire config. Will not create a breakdown such that more than max_sims_per_gpu simulations use the same GPU. - TODO: integration test for this - """ num_total_cpus: int = hw_config.num_cpus num_total_gpus: int = hw_config.num_gpus @@ -167,10 +165,10 @@ def checkProcessorIDContent(cpu_ids: List[int]): Exits the program if a condition is not met. """ if not isinstance(cpu_ids, list): - utils.fatal_error("Expected a list for parameter 'cpu_ids'") + utils.fatalError("Expected a list for parameter 'cpu_ids'") if not all([isinstance(i, int) and i >= 0 for i in cpu_ids]): - utils.fatal_error("Not all values in 'cpu_ids' are ints") + utils.fatalError("Not all values in 'cpu_ids' are ints") # case of empty list if not cpu_ids: @@ -181,4 +179,4 @@ def checkProcessorIDContent(cpu_ids: List[int]): for i, val in enumerate(cpu_ids[:-1]): if (cpu_ids[i+1] - val) != diff: - utils.fatal_error("Inconsistent stride between cpu ids") + utils.fatalError("Inconsistent stride between cpu ids") diff --git a/gromax/main.py b/gromax/main.py index bd498d5..5df50f7 100644 --- a/gromax/main.py +++ b/gromax/main.py @@ -5,7 +5,7 @@ import sys from gromax.analysis import GromaxData, constructGromaxData, reportStatistics from gromax.file_io import parseDirectoryStructure, allDirectoryContent, SanitizeDirectoryStructure -from gromax.combination_generator import createRunOptionsForConfigGroup +from gromax.combination_generator import createRunOptionsForConfigGroup, GenerateOptions from gromax.command_line import checkArgs, parseArgs, parseIDString from gromax.hardware_config import HardwareConfig, generateConfigSplitOptions from gromax.output import ParamsToString, WriteRunScript @@ -15,6 +15,12 @@ """ +def _populateGenerateOptions(args: argparse.Namespace) -> GenerateOptions: + max_sims_per_gpu: int = 4 if args.generate_exhaustive_combinations else 2 + return GenerateOptions(max_sims_per_gpu=max_sims_per_gpu, + generate_exhaustive_options=args.generate_exhaustive_combinations) + + def _executeGenerateWorkflow(args: argparse.Namespace) -> None: logger: logging.Logger = logging.getLogger("gromax") logger.info("Generating run options.") @@ -33,15 +39,17 @@ def _executeGenerateWorkflow(args: argparse.Namespace) -> None: num_cpus, num_gpus, num_cpus - modval)) cpu_ids = cpu_ids[:-modval] hw_config: HardwareConfig = HardwareConfig(cpu_ids=cpu_ids, gpu_ids=gpu_ids) - # generate options + + generate_options: GenerateOptions = _populateGenerateOptions(args) if args.single_sim_only: config_splits: List[List[HardwareConfig]] = [[hw_config]] else: - config_splits: List[List[HardwareConfig]] = generateConfigSplitOptions(hw_config) + config_splits: List[List[HardwareConfig]] = generateConfigSplitOptions( + hw_config, max_sims_per_gpu=generate_options.max_sims_per_gpu) logger.debug("Generated {} hardware config breakdowns".format(len(config_splits))) run_opts: List[List[Dict]] = [] for config_split in config_splits: - run_opts.extend(createRunOptionsForConfigGroup(config_split, args.gmx_version)) + run_opts.extend(createRunOptionsForConfigGroup(config_split, args.gmx_version, generate_options)) # Serialize options. out_file: str = args.run_file # TODO Make this configurable and robust @@ -70,7 +78,7 @@ def _executeAnalyzeWorkflow(args: argparse.Namespace) -> None: sys.stdout.write(reportStatistics(result_data.groupStatistics())) -def _executeExecuteWorkflow(args: argparse.Namespace) -> None: +def _executeExecuteWorkflow(_: argparse.Namespace) -> None: raise NotImplementedError("Direct execution not yet supported.") diff --git a/gromax/output.py b/gromax/output.py index 079528f..c126dd5 100644 --- a/gromax/output.py +++ b/gromax/output.py @@ -4,7 +4,6 @@ from gromax.combination_generator import ParameterSet, ParameterSetGroup from typing import Any, List -# TODO make ntrials a top-level parameter # TODO turn group_1, group_2... to group_${group} diff --git a/gromax/tests/combination_generator_test.py b/gromax/tests/combination_generator_test.py index cd76204..5df5eea 100644 --- a/gromax/tests/combination_generator_test.py +++ b/gromax/tests/combination_generator_test.py @@ -30,7 +30,7 @@ def testNoGpu(self): class DetermineGpuTasksTest(unittest.TestCase): def testOneRankWithPme(self): - self.assertEqual(cg.determineGpuTasks(1, [0], True), "00") + self.assertEqual(cg.determineGpuTasks(1, [0]), "00") def testOneRankNoPme(self): self.assertEqual(cg.determineGpuTasks(1, [0], False), "0") @@ -39,10 +39,10 @@ def testMultiRankNoPme(self): self.assertEqual(cg.determineGpuTasks(3, [0], False), "000") def testMultiRankOneGpu(self): - self.assertEqual(cg.determineGpuTasks(3, [0], True), "000") + self.assertEqual(cg.determineGpuTasks(3, [0]), "000") def testMultiRankMultiGpu(self): - self.assertEqual(cg.determineGpuTasks(6, [0, 4], True), "000444") + self.assertEqual(cg.determineGpuTasks(6, [0, 4]), "000444") class ApplyOptionToAllTest(unittest.TestCase): @@ -146,9 +146,11 @@ class CreateRunOptionsForSingleConfigTestv2016(unittest.TestCase): def setUp(self): self.config = HardwareConfig(cpu_ids=[0, 1, 2, 3]) + self._default_options: cg.GenerateOptions = cg.GenerateOptions(generate_exhaustive_options=True, + max_sims_per_gpu=4) def testNoGpu(self): - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -162,7 +164,7 @@ def testNoGpu(self): def testSingleGPU(self): self.maxDiff = None self.config.gpu_ids = [0] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -190,7 +192,7 @@ def testSingleGPU(self): def testMultiGPU(self): self.config.gpu_ids = [0, 1] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -221,9 +223,11 @@ class CreateRunOptionsForSingleConfigTestv2018(unittest.TestCase): def setUp(self): self.config = HardwareConfig(cpu_ids=[0, 1, 2, 3]) + self._default_options: cg.GenerateOptions = cg.GenerateOptions(generate_exhaustive_options=True, + max_sims_per_gpu=4) def testNoGpu(self): - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -237,7 +241,7 @@ def testNoGpu(self): def testSingleGPU(self): self.maxDiff = None self.config.gpu_ids = [0] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -296,7 +300,7 @@ def testSingleGPU(self): def testMultiGPU(self): self.config.gpu_ids = [0, 1] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -349,9 +353,11 @@ class CreateRunOptionsForSingleConfigTestv2019(unittest.TestCase): def setUp(self): self.config = HardwareConfig(cpu_ids=[0, 1, 2, 3]) + self._default_options: cg.GenerateOptions = cg.GenerateOptions(generate_exhaustive_options=True, + max_sims_per_gpu=4) def testNoGpu(self): - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -365,7 +371,7 @@ def testNoGpu(self): def testSingleGPU(self): self.maxDiff = None self.config.gpu_ids = [0] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ # bonded = cpu cases { @@ -491,7 +497,7 @@ def testSingleGPU(self): def testMultiGPU(self): self.config.gpu_ids = [0, 1] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ # bonded = cpu { @@ -592,9 +598,11 @@ class CreateRunOptionsForSingleConfigTestv2020(unittest.TestCase): def setUp(self): self.config = HardwareConfig(cpu_ids=[0, 1, 2, 3]) + self._default_options: cg.GenerateOptions = cg.GenerateOptions(generate_exhaustive_options=True, + max_sims_per_gpu=4) def testNoGpu(self): - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ { **self.expected_base, @@ -608,7 +616,7 @@ def testNoGpu(self): def testSingleGPU(self): self.maxDiff = None self.config.gpu_ids = [0] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ # bonded = cpu cases { @@ -837,7 +845,7 @@ def testSingleGPU(self): def testMultiGPU(self): self.config.gpu_ids = [0, 1] - result = cg.createRunOptionsForSingleConfig(self.config, self.version) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, self._default_options) expected = [ # bonded = cpu { @@ -978,6 +986,80 @@ def testMultiGPU(self): ] self.assertCountEqual(result, expected) + def testMinimalSubset(self): + self.maxDiff = None + self.config.gpu_ids = [0] + options = cg.GenerateOptions(max_sims_per_gpu=2, generate_exhaustive_options=False) + result = cg.createRunOptionsForSingleConfig(self.config, self.version, options) + expected = [ + # PME = gpu cases need npme=1 + { + **self.expected_base, + "ntmpi": 4, + "ntomp": 1, + "nb": "gpu", + "bonded": "gpu", + "pme": "gpu", + "npme": 1, + "update": "gpu", + "gputasks": "0000" + }, + { + **self.expected_base, + "ntmpi": 2, + "ntomp": 2, + "nb": "gpu", + "bonded": "gpu", + "pme": "gpu", + "npme": 1, + "update": "gpu", + "gputasks": "00" + }, + # Note the double gputask even though there's only one rank + { + **self.expected_base, + "ntmpi": 1, + "ntomp": 4, + "nb": "gpu", + "bonded": "gpu", + "pme": "gpu", + "update": "gpu", + "gputasks": "00" + }, + # update = cpu cases + { + **self.expected_base, + "ntmpi": 4, + "ntomp": 1, + "nb": "gpu", + "pme": "cpu", + "bonded": "cpu", + "update": "cpu", + "gputasks": "0000" + }, + { + **self.expected_base, + "ntmpi": 2, + "ntomp": 2, + "nb": "gpu", + "bonded": "cpu", + "pme": "cpu", + "update": "cpu", + "gputasks": "00" + }, + { + **self.expected_base, + "ntmpi": 1, + "ntomp": 4, + "nb": "gpu", + "bonded": "cpu", + "pme": "cpu", + "update": "cpu", + "gputasks": "0" + }, + ] + self.assertCountEqual(result, expected) + class CreateRunOptionsForConfigGroupTest(unittest.TestCase): common = { @@ -991,16 +1073,20 @@ class CreateRunOptionsForConfigGroupTest(unittest.TestCase): } + def setUp(self) -> None: + self._default_options: cg.GenerateOptions = cg.GenerateOptions(generate_exhaustive_options=True, + max_sims_per_gpu=4) + def testEmptyBreakdown(self): - self.assertEqual(cg.createRunOptionsForConfigGroup([], "2020"), []) + self.assertEqual(cg.createRunOptionsForConfigGroup([], "2020", self._default_options), []) def testFailsInvalidVersion(self): with self.assertRaises(ValueError): - cg.createRunOptionsForConfigGroup([HardwareConfig(cpu_ids=[0])], "2015") + cg.createRunOptionsForConfigGroup([HardwareConfig(cpu_ids=[0])], "2015", self._default_options) def testSinglebreakdown(self): configs = [HardwareConfig(cpu_ids=[0, 1, 2, 3], gpu_ids=[0])] - result = cg.createRunOptionsForConfigGroup(configs, "2016") + result = cg.createRunOptionsForConfigGroup(configs, "2016", self._default_options) expected = [ [ { @@ -1037,7 +1123,7 @@ def testSinglebreakdown(self): def testMultiBreakdown(self): configs = [HardwareConfig(cpu_ids=[0, 1], gpu_ids=[0]), HardwareConfig(cpu_ids=[2, 3], gpu_ids=[1])] - result = cg.createRunOptionsForConfigGroup(configs, "2016") + result = cg.createRunOptionsForConfigGroup(configs, "2016", self._default_options) expected = [ [ { diff --git a/gromax/tests/command_line_test.py b/gromax/tests/command_line_test.py index 5bea69a..c2bab38 100644 --- a/gromax/tests/command_line_test.py +++ b/gromax/tests/command_line_test.py @@ -1,5 +1,7 @@ import unittest + from gromax.command_line import checkArgs, parseArgs, parseIDString +from gromax.constants import _SUPPORTED_GMX_VERSIONS class CommandLineInputTest(unittest.TestCase): @@ -9,9 +11,7 @@ def testInvalidGromacsVersionCaught(self): checkArgs(parseArgs(args)) def testValidGromacsVersionsAccepted(self): - # TODO import this when centralized and loop over valid ones - valid_options = ["2016", "2018", "2019"] - for opt in valid_options: + for opt in _SUPPORTED_GMX_VERSIONS: checkArgs(parseArgs( ["generate", "--gmx_version", opt, "--cpu_ids", 0, "--gpu_ids", 0, "--run_file", "test.sh"])) diff --git a/gromax/tests/hardware_config_test.py b/gromax/tests/hardware_config_test.py index e0894b3..c6326ba 100644 --- a/gromax/tests/hardware_config_test.py +++ b/gromax/tests/hardware_config_test.py @@ -26,38 +26,38 @@ def testGoodLongStride(self): cpu_ids = [1, 4, 7] checkProcessorIDContent(cpu_ids) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfNotList(self, mock_fatal_error): cpu_ids = 1 with self.assertRaises(TypeError): checkProcessorIDContent(cpu_ids) self.assertTrue(mock_fatal_error.called) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfContentsString(self, mock_fatal_error): cpu_ids = ["1"] checkProcessorIDContent(cpu_ids) self.assertTrue(mock_fatal_error.called) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfContentsFloat(self, mock_fatal_error): cpu_ids = [1.5] checkProcessorIDContent(cpu_ids) self.assertTrue(mock_fatal_error.called) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfAnElementIsNonInt(self, mock_fatal_error): cpu_ids = [1, 2, 3.5, 4, 5] checkProcessorIDContent(cpu_ids) self.assertTrue(mock_fatal_error.called) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfNegativeIDs(self, mock_fatal_error): cpu_ids = [-1, 3] checkProcessorIDContent(cpu_ids) self.assertTrue(mock_fatal_error.called) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testFailIfInconsistentStride(self, mock_fatal_error): cpu_ids = [1, 3, 5, 8] checkProcessorIDContent(cpu_ids) @@ -90,19 +90,19 @@ def testRepr(self): config = HardwareConfig(cpu_ids=[1, 2], gpu_ids=[5]) self.assertEqual(config.__repr__(), "\nHardware config:\n\tcpu IDs : [1, 2]\n\tgpu IDs : [5]") - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testGpuIDFailureNotAList(self, mock_fatal): hw_config = HardwareConfig() hw_config.gpu_ids = 5 mock_fatal.assert_called_with("gpu_ids paramater must be iterable") - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testGpuIDFailureNotAnInt(self, mock_fatal): hw_config = HardwareConfig() hw_config.gpu_ids = [1, 1.7584, 3] mock_fatal.assert_called_with("All gpu ids must be integers: {} is not an int".format(1.7584)) - @patch('gromax.utils.fatal_error') + @patch('gromax.utils.fatalError') def testGpuIDFailureNotPositive(self, mock_fatal): hw_config = HardwareConfig() hw_config.gpu_ids = [1, 3, -1] diff --git a/gromax/tests/integration/generate_cli_test.py b/gromax/tests/integration/generate_cli_test.py index 062ccf4..621e722 100644 --- a/gromax/tests/integration/generate_cli_test.py +++ b/gromax/tests/integration/generate_cli_test.py @@ -67,7 +67,6 @@ def _runAndCompareOutput(self, reference_file: str): Executes the program, checks for correct execution, compares output file to reference. """ self._combineArgs() - print("args: " + " ".join(self.args)) self.assertEqual(self._run_and_get_rc(), 0) with open(self.kvs["--run_file"], 'r') as test_output_file: test_output: str = test_output_file.read() @@ -84,6 +83,7 @@ def setUp(self): "--gpu_ids": "0,1", "--gmx_version": "2016", "--run_file": tempfile.mkstemp()[1], + "--generate_exhaustive_combinations": None, } self.args = ["gromax", "generate"] @@ -142,3 +142,22 @@ def testWeirdCount(self): def testSingleSimOnly(self): self.kvs["--single_sim_only"] = None self._runAndCompareOutput("generate_test_single_sim_only.sh") + + def testNonExhaustive(self): + self.kvs = { + "--cpu_ids": "0-3", + "--gpu_ids": "0", + "--gmx_version": "2020", + "--run_file": tempfile.mkstemp()[1], + } + self._runAndCompareOutput("generate_test_minimal_subset.sh") + + def testNonExhaustiveWillSingleSim(self): + self.kvs = { + "--cpu_ids": "0-3", + "--gpu_ids": "0", + "--gmx_version": "2020", + "--run_file": tempfile.mkstemp()[1], + "--single_sim_only": None, + } + self._runAndCompareOutput("generate_test_minimal_subset_single_sim.sh") diff --git a/gromax/tests/integration/testdata/generate_test_minimal_subset.sh b/gromax/tests/integration/testdata/generate_test_minimal_subset.sh new file mode 100644 index 0000000..dabe78e --- /dev/null +++ b/gromax/tests/integration/testdata/generate_test_minimal_subset.sh @@ -0,0 +1,158 @@ +#!/bin/bash + +gmx='gmx mdrun' +tpr=None +nsteps=15000 +resetstep=10000 +ntrials=3 +workdir=`pwd` + +################################################################################ + +group=1 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 1 -ntomp 4 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=2 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 1 -ntomp 4 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=3 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 2 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=4 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 2 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=5 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0000 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 4 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=6 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0000 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 4 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=7 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 1 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu & + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_2 -gputasks 0 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 1 -ntomp 2 -pin on -pinoffset 2 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=8 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 1 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu & + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_2 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 1 -ntomp 2 -pin on -pinoffset 2 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=9 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 2 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu & + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_2 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 2 -ntomp 1 -pin on -pinoffset 2 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=10 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 2 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu & + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_2 -gputasks 00 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 2 -ntmpi 2 -ntomp 1 -pin on -pinoffset 2 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + + + +exit diff --git a/gromax/tests/integration/testdata/generate_test_minimal_subset_single_sim.sh b/gromax/tests/integration/testdata/generate_test_minimal_subset_single_sim.sh new file mode 100644 index 0000000..8ca12b5 --- /dev/null +++ b/gromax/tests/integration/testdata/generate_test_minimal_subset_single_sim.sh @@ -0,0 +1,98 @@ +#!/bin/bash + +gmx='gmx mdrun' +tpr=None +nsteps=15000 +resetstep=10000 +ntrials=3 +workdir=`pwd` + +################################################################################ + +group=1 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 1 -ntomp 4 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=2 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 1 -ntomp 4 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=3 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 2 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=4 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 00 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 2 -ntomp 2 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + +group=5 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded cpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0000 -nb gpu -noconfout -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 4 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme cpu -resetstep ${resetstep} -s ${tpr} -update cpu + wait + cd ${groupdir} +done + + +group=6 +groupdir=$workdir/group_${group} +mkdir $groupdir +cd $groupdir +for i in $(seq 1 ${ntrials}); do + trialdir=${groupdir}/trial_${i} + mkdir $trialdir + cd $trialdir + $gmx -bonded gpu -deffnm group_${group}_trial_${i}_component_1 -gputasks 0000 -nb gpu -noconfout -npme 1 -nsteps ${nsteps} -nstlist 80 -nt 4 -ntmpi 4 -ntomp 1 -pin on -pinoffset 0 -pinstride 1 -pme gpu -resetstep ${resetstep} -s ${tpr} -update gpu + wait + cd ${groupdir} +done + + + + +exit diff --git a/gromax/tests/output_test.py b/gromax/tests/output_test.py index 0d96a30..9d84b28 100644 --- a/gromax/tests/output_test.py +++ b/gromax/tests/output_test.py @@ -204,7 +204,7 @@ class ParamsToStringTest(unittest.TestCase): @mock.patch("gromax.output._ProcessAllGroups") def testParamsToString(self, mock_process): mock_process.return_value = "mock body" - result = ParamsToString([[], []], "mytpr.tpr", "gmx mdrun", 3, "i", 15000, 10000) + result = ParamsToString([[], []], "mytpr.tpr", "gmx mdrun", 3) expected = "#!/bin/bash\n\ngmx='gmx mdrun'\ntpr=mytpr.tpr\nnsteps=15000\nresetstep=10000\nntrials=3\nworkdir" \ "=`pwd`\n\n" + "#" * 80 + "\n\nmock body\n\nexit\n" self.assertEqual(result, expected) diff --git a/gromax/tests/utils_test.py b/gromax/tests/utils_test.py index f1a2535..3b5cee3 100644 --- a/gromax/tests/utils_test.py +++ b/gromax/tests/utils_test.py @@ -19,12 +19,11 @@ def tearDown(self): def testWritesMessage(self): with self.assertRaises(SystemExit): - utils.fatal_error("message") - self.assertEqual("message", str(self.stderr)) + utils.fatalError("message") def testExitsCleanly(self): with self.assertRaises(SystemExit) as cm: - utils.fatal_error("message") + utils.fatalError("message") self.assertEqual(cm.exception.code, 1) @@ -55,22 +54,18 @@ def testFileExistsError(self): mode = "xt" with self.assertRaises(SystemExit): utils.openOrDie(file, mode) - self.assertEqual(str(self.stderr), "File {} already exists, will not overwrite".format(file)) def testFileNotFoundError(self): file = get_relative_path("testdata/nonexistentfile.txt") mode = "rt" with self.assertRaises(SystemExit): utils.openOrDie(file, mode) - self.assertEqual(str(self.stderr), "File not found - could not find {}".format(file)) def testIsADirectoryError(self): file = get_relative_path("testdata") mode = "rt" with self.assertRaises(SystemExit): utils.openOrDie(file, mode) - expected_result = "Path specified is a directory, not a file. Path in question: {}".format(file) - self.assertEqual(str(self.stderr), expected_result) def testPermissionErrorRead(self): file = get_relative_path("testdata/existing_file.txt") @@ -79,7 +74,6 @@ def testPermissionErrorRead(self): mock_file.side_effect = PermissionError() with self.assertRaises(SystemExit): utils.openOrDie(file, mode) - self.assertEqual(str(self.stderr), "You lack read permissions for file {}".format(file)) def testPermissionErrorWrite(self): file = get_relative_path("testdata/existing_file.txt") @@ -88,4 +82,3 @@ def testPermissionErrorWrite(self): mock_file.side_effect = PermissionError() with self.assertRaises(SystemExit): utils.openOrDie(file, mode) - self.assertEqual(str(self.stderr), "You lack write permissions for file {}".format(file)) diff --git a/gromax/utils.py b/gromax/utils.py index 8b56c9b..218f661 100644 --- a/gromax/utils.py +++ b/gromax/utils.py @@ -1,11 +1,11 @@ +import logging import os import sys -def fatal_error(message): - # TODO log when we have logging support - sys.stderr.write(message) - sys.exit(1) +def fatalError(message): + logging.getLogger("gromax").error(message) + raise SystemExit(1) def openOrDie(file, mode): @@ -18,14 +18,14 @@ def openOrDie(file, mode): fin = open(file, mode) return fin except FileExistsError: - fatal_error("File {} already exists, will not overwrite".format(os.path.abspath(file))) + fatalError("File {} already exists, will not overwrite".format(os.path.abspath(file))) except FileNotFoundError: - fatal_error("File not found - could not find {}".format(os.path.abspath(file))) + fatalError("File not found - could not find {}".format(os.path.abspath(file))) except IsADirectoryError: - fatal_error("Path specified is a directory, not a file. Path in question: {}".format(os.path.abspath(file))) + fatalError("Path specified is a directory, not a file. Path in question: {}".format(os.path.abspath(file))) except PermissionError: if "w" in mode: attempt = "write" else: attempt = "read" - fatal_error("You lack {} permissions for file {}".format(attempt, os.path.abspath(file))) + fatalError("You lack {} permissions for file {}".format(attempt, os.path.abspath(file))) diff --git a/setup.py b/setup.py index 735bb54..237a78d 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup -VERSION = "0.0.1" +VERSION = "NEXT" setup( name="gromax", @@ -12,7 +12,7 @@ url="https://github.com/scal444/gromax", license="GPLv3", classifiers=[ - 'Development Status :: 3 - Alpha', + 'Development Status :: 4 - Beta', 'Environment :: Console', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3',