diff --git a/MANIFEST.in b/MANIFEST.in index 70ea571..cca16af 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,23 @@ +# This configuration file lists the elements to include in the source +# distribution of the Python package. +# See: https://packaging.python.org/en/latest/guides/using-manifest-in/ + +graft antareslauncher +graft doc +graft remote_scripts_templates +graft tests + +include *.ini +include *.md +include *.py +include *.txt +include Makefile + # The `data` directory is intended to store your configuration for integration testing with a SLURM server. # It should contain the application configuration file `configuration.yaml` and the SSH configuration file # `ssh_config.json`. The files in this directory should never be stored in the source distribution (except `README.md`). -include data/ include data/.gitignore include data/README.md recursive-exclude data * + +global-exclude *.py[cod] .DS_Store diff --git a/README.md b/README.md index 009f0ab..d527a61 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,108 @@ # Antares Launcher -This program is meant to allow the user to send a list of Antares simulations -on a remote Linux machine that can run them using *SLURM Workload Manager*. +This program is intended to allow the user to send a list of Antares simulations to +a remote Linux machine that can run them using *SLURM Workload Manager*. -At present this program: +Currently, this program: -- is configured to work with Antares studies of version 6.1.3 and 7.0.0 (the configuration can be changed in YAML file) -- needs a remote unix server that uses *SLURM Workload Manager* +- Is configured to work with Antares studies from version 7.0 through 8.5 + (the configuration can be changed in a YAML file). +- needs a remote UNIX server that uses *SLURM Workload Manager*. -A schema of the main workflow is as follow +The main workflow diagram is as follows: -![Antares Study Launcher](./doc/source/schema/antares_flow_chart_AS-FINAL-withbranch-wait.png) +![Antares Study Launcher](https://raw.githubusercontent.com/AntaresSimulatorTeam/antares-launcher/main/doc/source/schema/antares_flow_chart_AS-FINAL-withbranch-wait.png) -### Requirements -see [Requirements file](./requirements.txt) -minimum version : python 3.6 +## Requirements -paramiko, pytest, pytest-cov, pytest-ordering, tinydb, black, pyinstaller, setuptools==44.1.1, tqdm -pytest for the tests -pyinstaller to 'compile' the project into binary file +See [`setup.py`](https://github.com/AntaresSimulatorTeam/antares-launcher/blob/main/setup.py) -#### For documentation -sphinx +Minimum version : python 3.8 -##### To convert the doc from markdown to sphinx (rst) -m2r +### Main Python libraries + +The following libraries are required to run the application in a production (or staging) environment: + +- paramiko +- PyYAML +- tinydb +- tqdm + +To install this library on production, you can run: + +```shell +pip install Antares-Launcher +``` + +### Development and Unit Testing + +To start developing, you can clone the repository from GitHub and create a Python virtualenv: + +```shell +cd ~/workspace/ +git clone https://github.com/AntaresSimulatorTeam/antares-launcher.git +cd ~/workspace/antares-launcher/ +python3 -m venv venv +source venv/bin/activate +``` + +To run the unit tests, you need to install: + +- pytest +- pytest-cov +- pytest-xdist + +To install this library in development mode for testing, you can run: + +```shell +pip install -e .[test] +``` + +Additional dependencies could also be used for development, for instance: + +- black +- check-manifest +- isort +- mypy + +### Documentation + +In this project, we use Sphinx to generate the documentation. + +Extra requirements are: + +- m2r +- recommonmark +- sphinx +- sphinx_rtd_theme ## Installation ### Generation of the binary executable + In order to generate the binary file, execute the following command: ``` pyinstaller --additional-hooks-dir=antareslauncher/hooks/ -F antareslauncher/main_launcher.py -n Antares_Launcher ``` -In order to generate the binary file of the light version of the launcher (reduced set of options), execute the following command: +In order to generate the binary file of the light version of the launcher (reduced set of options), execute the +following command: ``` pyinstaller --additional-hooks-dir=antareslauncher/hooks/ -F antareslauncher/main_launcher_light.py -n Antares_Launcher_Light ``` -The generated file will be inside the dist directory. Note that pyinstaller does not enable the cross-compilation: e binary file generated on windows can only be expected with the windows OS +The generated file will be inside the dist directory. Note that pyinstaller does not enable the cross-compilation: e +binary file generated on windows can only be expected with the windows OS ## Use Antares_Launcher ### Run Antares_Launcher + **Antares Launcher** can be used by running the executable file -By default the program will +By default, the program will: - look for a configuration file necessary for the connection named *ssh_config.json*. @@ -60,26 +114,27 @@ If no value is given, it will look for it in default location with this order: A default *ssh_config.json* file can be found in this repository in the `./data` directory of the project - - look for an rsa-private ssh-key to access to the remote server. The path of the key is specified in the `ssh_config.json` file - look for a directory containing -the Antares studies to be run on the remote machine -named *STUDIES-IN*. + the Antares studies to be run on the remote machine + named *STUDIES-IN*. - put the results in the directory named -*FINISHED* + *FINISHED* - create a directory *LOGS* that contains the logs of the programs -and several directories containing the three log files specific of each simulation. -Currently **antares_launcher** uses a specific configuration attached to the specific setting of -`data/launchAntares-${SCRIPT-VERSION}.sh` + and several directories containing the three log files specific of each simulation. + Currently **antares_launcher** uses a specific configuration attached to the specific setting of + `data/launchAntares-${SCRIPT-VERSION}.sh` #### Get the *how-to* + ``` Antares_Launcher --help ``` + will show how to use the program. ### SLURM script on the remote machine @@ -88,34 +143,43 @@ In order to submit new jobs to the *SLURM* queue manager, **Antares_Launcher** launches a bash-SLURM script the name of the script is set in `data/configuration.yaml`. If Antares_Launcher fails to find this script an exception will be raised and the execution will stop. - + The specification of the script can be found in the class `SlurmScriptFeatures` in the module `antareslauncher.slurm_script_features.py`. See [Deploy Antares Launcher](#deploy-antares-launcher) for specific values. ## Useful commands + Since the addition of the Makefile to the project, one can now easily set a virtual environment, install requirements, generate binary file, run tests, generate the doc and deploy it... At the root of the directory, all the available commands can be seen with typing: make -![Antares Study Launcher](./doc/source/schema/make_example.png) +![Antares Study Launcher](https://raw.githubusercontent.com/AntaresSimulatorTeam/antares-launcher/main/doc/source/schema/make_example.png) If for example, you would like to run the test, a simple ``make test`` will do the trick -![Antares Study Launcher](./doc/source/schema/make_test_example.png) +![Antares Study Launcher](https://raw.githubusercontent.com/AntaresSimulatorTeam/antares-launcher/main/doc/source/schema/make_test_example.png) -## More useful commands -`pytest --cov=../antareslauncher` +## Useful commands -`pytest --cov=../antareslauncher --cov-report term-missing` +Run unit tests: -`ssh dev-antares@163.104.210.38` +```shell +pytest -v tests/ +``` -`pytest --cov=../antareslauncher --cov-report term-missing -p no:warnings` +Run unit tests with code coverage: + +```shell +pytest --cov=antareslauncher --cov-report=term-missing --cov-report=html --cov-branch tests/ +open htmlcov/index.html +``` # Deploy Antares Launcher + ## Installation on the remote server + In order to be able to accept jobs from Antares_Launcher, the remote machine needs to be ready: the binaries and script expected by **Antares_Launcher** need to be installed and the required ssh-public-keys need to be added to the `authorizedkeys` file @@ -124,22 +188,24 @@ of the account of the remote server. ### Things to do - `launchAntares-${SCRIPT-VERSION}.sh` should be copied to the remove server -and ist path should be set in `data/configuration.yaml` + and ist path should be set in `data/configuration.yaml` -- Install the Antares solver binary `antares-x.x-solver` on the remote server. -set its installation path in `launchAntares-${SCRIPT-VERSION}.sh` +- Install the Antares solver binary `antares-x.x-solver` on the remote server. + set its installation path in `launchAntares-${SCRIPT-VERSION}.sh` - The R Xpansion script, `data/XpansionArgsRun.R`, -has to be copied to the remote server and - its path should be set in `launchAntares-${SCRIPT-VERSION}.sh`. - + has to be copied to the remote server and + its path should be set in `launchAntares-${SCRIPT-VERSION}.sh`. + #### Important notice + The users currently copy the executable every time they need to use it. -This is not practical, an alternative should be developed. +This is not practical, an alternative should be developed. ## Installation of R packages on the remote server -In order to correctly install or update packages to be used on the remote server + +In order to correctly install or update packages to be used on the remote server the *R*repositories and installation-destination need to be set. The `launchAntares-${SCRIPT-VERSION}.sh` set the variable where the *R*libraries are installed runtime, - no need to create a `.Renviron` file. +no need to create a `.Renviron` file. diff --git a/antareslauncher/__init__.py b/antareslauncher/__init__.py index 8028c16..33c15ec 100644 --- a/antareslauncher/__init__.py +++ b/antareslauncher/__init__.py @@ -1,8 +1,49 @@ -#: Project's name (str) -PROJECT_NAME = "Antares_Launcher" +""" +Antares Launcher -#: Project's description (str) -DESCRIPTION = "Antares_Launcher to run Antares on a remote linux machine" +This program is meant to allow the user to send a list of Antares simulations +on a remote Linux machine that can run them using *SLURM Workload Manager*. -#: Project's version (:py:class:`Version `) -VERSION = "1.1.3" +This module contains the project metadata. +""" + +# Standard project metadata + +__version__ = "1.1.4" +__author__ = "RTE, Antares Web Team" +__date__ = "2021-05-25" +# noinspection SpellCheckingInspection +__credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" + +# Extra project metadata +__project_name__ = "Antares_Launcher" + + +def _check_metadata(): + # noinspection SpellCheckingInspection + """ + Check the project metadata. + + To update the project metadata, you can run the following command: + ```shell + python setup.py egg_info + ``` + + To get the list of tags and release dates, you can also run: + ```shell + git for-each-ref --sort=-creatordate --format '%(refname:strip=2) (%(creatordate:short))' refs/tags + ``` + """ + from pkg_resources import get_distribution + + dist = get_distribution(__project_name__) + # Expected distribution name should be "antares-launcher": + dist_name = __project_name__.lower().replace("_", "-") + assert dist.key == dist_name, dist.key + assert dist.version == __version__, dist.version + print("Project metadata OK.") + + +if __name__ == "__main__": + # run the shell command `python antareslauncher/__init__.py ` to check + _check_metadata() diff --git a/antareslauncher/advanced_launch.py b/antareslauncher/advanced_launch.py index a78f368..6fc2f61 100644 --- a/antareslauncher/advanced_launch.py +++ b/antareslauncher/advanced_launch.py @@ -1,23 +1,17 @@ from pathlib import Path -from antareslauncher.main import run_with, MainParameters -from antareslauncher.main_option_parser import ( - MainOptionParser, - ParserParameters, -) +from antareslauncher.config import Config, get_config_path +from antareslauncher.main import MainParameters, run_with +from antareslauncher.main_option_parser import MainOptionParser, ParserParameters from antareslauncher.parameters_reader import ParametersReader -HERE = Path(__file__).parent.resolve() -PACKAGE_NAME = "antareslauncher" -PROJECT_DIR = next(iter(p for p in HERE.parents if p.joinpath(PACKAGE_NAME).exists())) -DATA_DIR = PROJECT_DIR / "data" -SSH_JSON_FILE = DATA_DIR / "ssh_config.json" -YAML_CONF_FILE = DATA_DIR / "configuration.yaml" - def main(): + config_path: Path = get_config_path() + config = Config.load_config(config_path) param_reader = ParametersReader( - json_ssh_conf=SSH_JSON_FILE, yaml_filepath=YAML_CONF_FILE + json_ssh_conf=config.ssh_config.config_path, + yaml_filepath=config.config_path, ) parser_parameters: ParserParameters = param_reader.get_parser_parameters() parser: MainOptionParser = MainOptionParser(parser_parameters) diff --git a/antareslauncher/basic_launch.py b/antareslauncher/basic_launch.py index e78d471..9b45481 100644 --- a/antareslauncher/basic_launch.py +++ b/antareslauncher/basic_launch.py @@ -1,21 +1,17 @@ -import sys from pathlib import Path -from antareslauncher.main import run_with, MainParameters -from antareslauncher.main_option_parser import ( - MainOptionParser, - ParserParameters, -) +from antareslauncher.config import Config, get_config_path +from antareslauncher.main import MainParameters, run_with +from antareslauncher.main_option_parser import MainOptionParser, ParserParameters from antareslauncher.parameters_reader import ParametersReader -DATA_DIR = Path(__file__).parent.resolve() / "data" -SSH_JSON_FILE = DATA_DIR / "ssh_config.json" -YAML_CONF_FILE = DATA_DIR / "configuration.yaml" - def main(): + config_path: Path = get_config_path() + config = Config.load_config(config_path) param_reader = ParametersReader( - json_ssh_conf=SSH_JSON_FILE, yaml_filepath=YAML_CONF_FILE + json_ssh_conf=config.ssh_config.config_path, + yaml_filepath=config.config_path, ) parser_parameters: ParserParameters = param_reader.get_parser_parameters() parser: MainOptionParser = MainOptionParser(parser_parameters) diff --git a/antareslauncher/config.py b/antareslauncher/config.py new file mode 100644 index 0000000..07b689e --- /dev/null +++ b/antareslauncher/config.py @@ -0,0 +1,312 @@ +""" +Parse the configuration file `configuration.yaml`. +""" +import dataclasses +import getpass +import json +import os +import pathlib +import sys +from typing import Any, Dict, List, Optional + +import yaml + +from antareslauncher import __author__, __project_name__, __version__ +from antareslauncher.exceptions import ( + InvalidConfigValueError, + UnknownFileSuffixError, + ConfigFileNotFoundError, +) + +APP_NAME = __project_name__ +APP_AUTHOR = __author__.split(",")[0] +APP_VERSION = ".".join(__version__.split(".")[:2]) # "MAJOR.MINOR" + +CONFIGURATION_YAML = "configuration.yaml" + + +def parse_config(config_path: pathlib.Path) -> Dict[str, Any]: + """ + This function takes a file path, checks its extension, and reads the contents + of the file as either YAML or JSON and returns it as a dictionary. + + Args: + config_path: Full path of the configuration file. + + Returns: + Content of the configuration file as a dictionary. + + Raises: + `UnknownFileSuffixError`: If the file extension is not '.yaml', '.yml', or '.json'. + """ + suffix = config_path.suffix.lower() + if suffix in {".yaml", ".yml"}: + with config_path.open(encoding="utf-8") as fd: + obj = yaml.load(fd, Loader=yaml.FullLoader) + elif suffix in {".json"}: + text = config_path.read_text(encoding="utf-8") + obj = json.loads(text) + else: + raise UnknownFileSuffixError(config_path, config_path.suffix) + return obj + + +def dump_config(config_path: pathlib.Path, obj: Dict[str, Any]) -> None: + """ + This function takes a file path, checks its extension, and write + the dictionary object in the file as either YAML or JSON. + + Args: + config_path: Full path of the configuration file. + obj: Configuration file to write. + """ + suffix = config_path.suffix.lower() + if suffix in {".yaml", ".yml"}: + with config_path.open(mode="w", encoding="utf-8") as fd: + yaml.dump(obj, fd, allow_unicode=True, sort_keys=False, indent=2) + elif suffix in {".json"}: + config_path.write_text( + json.dumps(obj, indent=2, sort_keys=False), + encoding="utf-8", + ) + else: + raise UnknownFileSuffixError(config_path, config_path.suffix) + + +@dataclasses.dataclass +class SSHConfig: + """ + Configuration of the SSH access. + + Attributes: + - config_path: Path to the SSH configuration file. + - username: SSH username. + - hostname: Hostname or IP address of the SSH server. + - port: Port number of the SSH service (defaults to 22). + - private_key_file: Path to the private key file (defaults to None). + - key_password: Password to use with the private key file (defaults to an empty string). + - password: Password to use for classic authentication (defaults to an empty string). + """ + + config_path: pathlib.Path + username: str + hostname: str + port: int = 22 + private_key_file: Optional[pathlib.Path] = None + key_password: str = "" + password: str = "" + + @classmethod + def load_config(cls, ssh_config_path: pathlib.Path) -> "SSHConfig": + """ + Load the SSH configuration from a file and return it as an `SSHConfig` object. + + Args: + ssh_config_path: Path to the SSH configuration file. + + Returns: + An `SSHConfig` object populated with the values from the SSH configuration file. + + The file should contain key-value pairs in either YAML or JSON format. + The keys are case-insensitive and will be converted to lowercase before being used + to populate the fields of the `SSHConfig` object. + The `private_key_file` key (if present) will be converted to a `pathlib.Path` object. + + Raises: + `UnknownFileSuffixError`: If the file extension is not '.yaml', '.yml', or '.json'. + `InvalidConfigValueError`: If a value is invalid or missing. + """ + obj = parse_config(ssh_config_path) + kwargs = {k.lower(): v for k, v in obj.items()} + private_key_file = kwargs.pop("private_key_file", None) + kwargs["private_key_file"] = ( + None if private_key_file is None else pathlib.Path(private_key_file) + ) + try: + return cls(config_path=ssh_config_path, **kwargs) + except TypeError as exc: + raise InvalidConfigValueError(ssh_config_path, str(exc)) from None + + def save_config(self, ssh_config_path: pathlib.Path) -> None: + """ + Save the SSH configuration file. + + Args: + ssh_config_path: Path to the SSH configuration file. + + Raises: + `UnknownFileSuffixError`: If the file extension is not '.yaml', '.yml', or '.json'. + """ + obj = dataclasses.asdict(self) + del obj["config_path"] + obj = { + k: v + for k, v in obj.items() + if v or k not in {"private_key_file", "key_password", "password"} + } + if "private_key_file" in obj: + obj["private_key_file"] = obj["private_key_file"].as_posix() + dump_config(ssh_config_path, obj) + + +@dataclasses.dataclass +class Config: + """ + Represents the configuration for launching studies on a SLURM server using Antares Launcher. + + Attributes: + - config_path: Path to the Antares Launcher configuration file. + - log_dir: Path to the directory where logs will be stored. + - json_dir: Path to the directory where the JSON database will be stored. + - studies_in_dir: Path to the directory where studies will be read. + - finished_dir: Path to the directory where finished studies will be downloaded and placed. + - default_time_limit: The default time limit (in seconds) for each study simulation job. + - default_n_cpu: The default number of CPUs to be used by each study simulation job. + - default_wait_time: The default wait time (in seconds) between study simulation jobs. + - db_primary_key: A string representing the primary key used in the database. + - default_ssh_configfile_name: The default name of the SSH configuration file. + - ssh_config_file_is_required: A flag indicating whether an SSH configuration file is required. + - slurm_script_path: Path to the SLURM script used to launch studies (a Shell script). + - remote_solver_versions: A list of strings representing the available Antares Solver + versions on the remote server. + - ssh_config: An `SSHConfig` object representing the SSH configuration. + """ + + config_path: pathlib.Path + log_dir: pathlib.Path + json_dir: pathlib.Path + studies_in_dir: pathlib.Path + finished_dir: pathlib.Path + default_time_limit: int + default_n_cpu: int + default_wait_time: int + db_primary_key: str + ssh_config_file_is_required: bool + slurm_script_path: pathlib.Path + remote_solver_versions: List[str] + + ssh_config: SSHConfig + + @classmethod + def load_config(cls, config_path: pathlib.Path) -> "Config": + """ + Load the Antares Launcher configuration from a file and return it as a `Config` object. + + Args: + config_path: Path to the Antares Launcher configuration file. + + Returns: + A `Config` object populated with the values from the Antares Launcher configuration file. + + The file should contain key-value pairs in either YAML or JSON format. + The keys are case-insensitive and will be converted to lowercase before being used + to populate the fields of the `Config` object. Paths will be converted to `pathlib.Path` objects. + The SSH configuration file is specified by the `default_ssh_configfile_name` field + and will be loaded as an `SSHConfig` object. + + Raises: + `UnknownFileSuffixError`: If the file extension is not '.yaml', '.yml', or '.json'. + `InvalidConfigValueError`: If a value is invalid or missing. + """ + obj = parse_config(config_path) + kwargs = {k.lower(): v for k, v in obj.items()} + try: + kwargs["remote_solver_versions"] = kwargs.pop( + "antares_versions_on_remote_server" + ) + # handle paths + for key in [ + "log_dir", + "json_dir", + "studies_in_dir", + "finished_dir", + "slurm_script_path", + ]: + kwargs[key] = pathlib.Path(kwargs[key]) + ssh_configfile_name = kwargs.pop("default_ssh_configfile_name") + except KeyError as exc: + raise InvalidConfigValueError( + config_path, f"missing parameter '{exc}'" + ) from None + # handle SSH configuration + config_dir = config_path.parent + ssh_config_path = config_dir.joinpath(ssh_configfile_name) + kwargs["ssh_config"] = SSHConfig.load_config(ssh_config_path) + try: + return cls(config_path=config_path, **kwargs) + except TypeError as exc: + raise InvalidConfigValueError(config_path, str(exc)) from None + + def save_config(self, config_path: pathlib.Path) -> None: + """ + Save the Antares Launcher configuration file. + + Args: + config_path: Path to the configuration file. + + Raises: + `UnknownFileSuffixError`: If the file extension is not '.yaml', '.yml', or '.json'. + """ + obj = dataclasses.asdict(self) + del obj["config_path"] + del obj["ssh_config"] + obj["antares_versions_on_remote_server"] = obj.pop("remote_solver_versions") + for key in [ + "log_dir", + "json_dir", + "studies_in_dir", + "finished_dir", + "slurm_script_path", + ]: + obj[key] = obj[key].as_posix() + obj[ + "default_ssh_configfile_name" + ] = ssh_config_name = self.ssh_config.config_path.name + dump_config(config_path, obj) + config_dir = config_path.parent + self.ssh_config.save_config(config_dir.joinpath(ssh_config_name)) + + +def get_user_config_dir(system: str = ""): + """ + Retrieve the user configuration directory based on the system platform. + + - On Windows, it returns the path + `C:\\Users\\\\AppData\\Local\\\\\\`. + - On macOS, it returns `~/Library/Preferences//`. + - On other platforms, it returns `~/.config//` + depending on the `XDG_CONFIG_HOME` environment variable if it is set. + + Args: + system: Platform system, by default the platform is auto-detected. + + Returns: + The user configuration directory path. + """ + username = getpass.getuser() + system = system or sys.platform + if system == "win32": + config_dir = pathlib.WindowsPath( + rf"C:\Users\{username}\AppData\Local\{APP_AUTHOR}" + ) + elif system == "darwin": + config_dir = pathlib.PosixPath("~/Library/Preferences").expanduser() + else: + path = os.getenv("XDG_CONFIG_HOME", "~/.config") + config_dir = pathlib.PosixPath(path).expanduser() + return config_dir.joinpath(APP_NAME, APP_VERSION) + + +def get_config_path(config_name: str = CONFIGURATION_YAML) -> pathlib.Path: + env_value = os.environ.get("ANTARES_LAUNCHER_CONFIG_PATH") + if env_value is not None: + config_path = pathlib.Path(env_value) + if config_path.exists(): + return config_path + raise ConfigFileNotFoundError([config_path.parent], config_path.name) + possible_dirs = [get_user_config_dir(), pathlib.Path("."), pathlib.Path("./data")] + for config_dir in possible_dirs: + config_path = config_dir.joinpath(config_name) + if config_path.exists(): + return config_path + raise ConfigFileNotFoundError(possible_dirs, config_name) diff --git a/antareslauncher/exceptions.py b/antareslauncher/exceptions.py new file mode 100644 index 0000000..201abfa --- /dev/null +++ b/antareslauncher/exceptions.py @@ -0,0 +1,74 @@ +""" +Antares Launcher Exceptions +""" +import pathlib +from typing import Sequence + + +class AntaresLauncherException(Exception): + """The base-class of all Antares Launcher exceptions.""" + + +class ConfigFileNotFoundError(AntaresLauncherException): + """Configuration file not found.""" + + def __init__( + self, possible_dirs: Sequence[pathlib.Path], config_name: str, *args + ) -> None: + super().__init__(possible_dirs, config_name, *args) + + @property + def possible_dirs(self) -> Sequence[pathlib.Path]: + return self.args[0] + + @property + def config_name(self) -> str: + return self.args[1] + + def __str__(self): + possible_dirs = ", ".join(f"'{p}'" for p in self.possible_dirs) + config_name = self.config_name + return f"Configuration file '{config_name}' not found in the following locations: {possible_dirs}" + + +class ConfigError(AntaresLauncherException): + """A problem with a config file, or a value in one.""" + + def __init__(self, config_path: pathlib.Path, *args): + super().__init__(config_path, *args) + + @property + def config_path(self) -> pathlib.Path: + return self.args[0] + + def __str__(self): + config_path = self.config_path + return f"Invalid configuration file '{config_path}'" + + +class UnknownFileSuffixError(ConfigError): + def __init__(self, config_path: pathlib.Path, suffix: str): + super().__init__(config_path, suffix) + + @property + def suffix(self) -> str: + return self.args[1] + + def __str__(self): + parent_msg = super().__str__() + suffix = self.suffix + return f"{parent_msg}: unknown file suffix '{suffix}'" + + +class InvalidConfigValueError(ConfigError): + def __init__(self, config_path: pathlib.Path, error_msg: str): + super().__init__(config_path, error_msg) + + @property + def error_msg(self) -> str: + return self.args[1] + + def __str__(self): + parent_msg = super().__str__() + error_msg = self.error_msg + return f"{parent_msg}: Invalid config value: {error_msg}" diff --git a/antareslauncher/main.py b/antareslauncher/main.py index 8d7dd7b..bd312a4 100644 --- a/antareslauncher/main.py +++ b/antareslauncher/main.py @@ -3,7 +3,7 @@ from pathlib import Path from typing import List, Dict -from antareslauncher import VERSION +from antareslauncher import __version__ from antareslauncher.antares_launcher import AntaresLauncher from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.display.display_terminal import DisplayTerminal @@ -80,7 +80,7 @@ def run_with( ): """Instantiates all the objects necessary to antares-launcher, and runs the program""" if arguments.version: - print(f"Antares_Launcher v{VERSION}") + print(f"Antares_Launcher v{__version__}") return if show_banner: diff --git a/antareslauncher/use_cases/retrieve/clean_remote_server.py b/antareslauncher/use_cases/retrieve/clean_remote_server.py index f8ab3a2..8f158f1 100644 --- a/antareslauncher/use_cases/retrieve/clean_remote_server.py +++ b/antareslauncher/use_cases/retrieve/clean_remote_server.py @@ -31,9 +31,9 @@ def _should_clean_remote_server(self): self._current_study.remote_server_is_clean is False ) and self._final_zip_downloaded() - def _final_zip_downloaded(self): + def _final_zip_downloaded(self) -> bool: if isinstance(self._current_study.local_final_zipfile_path, str): - return self._current_study.local_final_zipfile_path is not "" + return bool(self._current_study.local_final_zipfile_path) else: return False diff --git a/doc/source/binary_generation.md b/doc/source/binary_generation.md index 873b1ee..6016f18 100644 --- a/doc/source/binary_generation.md +++ b/doc/source/binary_generation.md @@ -1,8 +1,17 @@ In order to generate the binary file, execute the following command (require pyinstaller): +To install pyinstaller, you can run: +```shell +pip install -e .[pyinstaller] ``` -pyinstaller -F antareslauncher/main_launcher.py -n Antares_Launcher + +Then run the following command to compile the application on you platform: + +``` +pyinstaller -F antareslauncher/basic_launch.py -n Antares_Launcher ``` -The generated binary file will be in dist directory. Note that pyinstaller does not enable the multi-platform cross-compilation. For instance the binary file generated on Windows can be executed only on the Windows OS. +The generated binary file will be in `dist` directory. +Note that pyinstaller does not enable the multi-platform cross-compilation. +For instance the binary file generated on Windows can be executed only on the Windows OS. diff --git a/doc/source/conf.py b/doc/source/conf.py index 4e8159c..b6ad34d 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -12,7 +12,7 @@ # import os import sys -import recommonmark +# import recommonmark from m2r import MdInclude # sys.path.insert(0, os.path.abspath('.')) @@ -64,7 +64,7 @@ # # This is also used if you do content translation via gettext catalogs. # Usually you set "language" from the command line for these cases. -language = None +language = "en" # The name of the Pygments (syntax highlighting) style to use. diff --git a/doc/source/developper_documentation.rst b/doc/source/developper_documentation.rst index 7e212ca..2d4c7f2 100644 --- a/doc/source/developper_documentation.rst +++ b/doc/source/developper_documentation.rst @@ -2,7 +2,7 @@ Dev documentation ***************** -This part of the documentation is for developpers. +This part of the documentation is for developers. The aim is to have a design doc ready to be shared with any other dev. It should greatly help people getting in the project. diff --git a/doc/source/first_use_of_antares_launcher.rst b/doc/source/first_use_of_antares_launcher.rst index 93f6bf7..c133034 100644 --- a/doc/source/first_use_of_antares_launcher.rst +++ b/doc/source/first_use_of_antares_launcher.rst @@ -99,7 +99,9 @@ Basic execution --------------- By simply launching Antares_Launcher with no options or by double click on the executable file: + :: + Antares_Launcher(.exe) The execution flows look like this: @@ -118,7 +120,7 @@ To open the Windows command prompt, open the ``BASE`` directory and press **shif Here are the basic options you can use : 1. Wait mode -^^^^^^^^^^^ +^^^^^^^^^^^^ :: @@ -287,34 +289,34 @@ You can get the list of options by using this command : -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|option command |explaination |by default | -+========================================+==================================================+=================================+ -|--version |show program's version number and exit | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-i STUDIES_IN or |directory containing the studies to be executed |the studies that are looking for | -|--studies-in-dir=STUDIES_IN | |should be in STUDIES-IN | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-w or --wait-mode |activate the wait mode : the antares_launcher |desactivated | -| |waits all the jobs to finish | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-o OUTPUT_DIR or --output-dir=OUTPUT_DIR|directory where the finished studies will be |done jobs are put inside | -| |downloaded and extracted |FINISHED directory | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-t TIME_LIMIT or --time-limit=TIME_LIMIT|time limit in seconds of a single job |5h=18000 secondes | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|--wait-time=WAIT_TIME |number of seconds between each verification |60 seconds | -| |of the end of the simulations | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-q |displays the queue of Antares jobs | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-x |run all new (compatible) studies in xpansion mode | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-p |trigger *R* post-processing | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ -|-k JOB_ID_TO_KILL or |kill a job by specifying its JOB_ID | | -|--kill-job JOB_ID_TO_KILL |It overrides the -q or the standard execution | | -+----------------------------------------+--------------------------------------------------+---------------------------------+ ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| command option | explanation | default value | ++==========================================+====================================================+==================================+ +| --version | show program version number and exit | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -i STUDIES_IN or | directory containing the studies to be executed | the studies we are looking for | +| --studies-in-dir=STUDIES_IN | | must be in ``STUDIES-IN`` | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -w or --wait-mode | activate the wait mode: the antares_launcher | deactivated | +| | waits all the jobs to finish | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -o OUTPUT_DIR or --output-dir=OUTPUT_DIR | directory where the finished studies will be | done jobs are put inside | +| | downloaded and extracted | ``FINISHED`` directory | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -t TIME_LIMIT or --time-limit=TIME_LIMIT | time limit in seconds of a single job | 5h = 18000 seconds | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| --wait-time=WAIT_TIME | number of seconds between each verification | 60 seconds | +| | of the end of the simulations | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -q | displays the queue of Antares jobs | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -x | run all new (compatible) studies in xpansion mode | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -p | trigger *R* post-processing | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ +| -k JOB_ID_TO_KILL or | kill a job by specifying its JOB_ID | | +| --kill-job JOB_ID_TO_KILL | It overrides the ``-q`` or the standard execution | | ++------------------------------------------+----------------------------------------------------+----------------------------------+ Errors and exception diff --git a/doc/source/index.rst b/doc/source/index.rst index 62d1e6e..49e738d 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -52,8 +52,6 @@ This diagram describes the behavior of the module between your local computer an roadmap.rst changelog.rst -.. date:: - Indices and tables ================== diff --git a/requirements-docs.txt b/requirements-docs.txt new file mode 100644 index 0000000..65ec6d5 --- /dev/null +++ b/requirements-docs.txt @@ -0,0 +1,32 @@ +-r requirements.txt + +alabaster==0.7.13 +Babel==2.11.0 +certifi==2022.12.7 +charset-normalizer==3.0.1 +commonmark==0.9.1 +docutils==0.18.1 +idna==3.4 +imagesize==1.4.1 +importlib-metadata==6.0.0 +Jinja2==3.1.2 +m2r==0.3.1 +MarkupSafe==2.1.2 +mistune==0.8.4 +packaging==23.0 +Pygments==2.14.0 +pytz==2022.7.1 +recommonmark==0.7.1 +requests==2.28.2 +snowballstemmer==2.2.0 +sphinx==6.1.3 +sphinx-rtd-theme==1.2.0 +sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-jquery==2.0.0 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.5 +urllib3==1.26.14 +zipp==3.14.0 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..e16fd09 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,13 @@ +-r requirements.txt + +attrs==22.2.0 +coverage==7.1.0 +exceptiongroup==1.1.0 +execnet==1.9.0 +iniconfig==2.0.0 +packaging==23.0 +pluggy==1.0.0 +pytest==7.2.1 +pytest-cov==4.0.0 +pytest-xdist==3.1.0 +tomli==2.0.1 diff --git a/requirements.txt b/requirements.txt index 07edf1c..0eea766 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,18 +1,10 @@ -paramiko -pytest-xdist -pytest -pytest-cov -tinydb -black -pyinstaller -setuptools -tqdm -pyyaml - -# for documentation -sphinx -sphinx_rtd_theme -recommonmark - -# for documentation, from markdown to sphinx (rst) -m2r +bcrypt==4.0.1 +cffi==1.15.1 +cryptography==39.0.1 +paramiko==2.12.0 +pycparser==2.21 +PyNaCl==1.5.0 +PyYAML==6.0 +six==1.16.0 +tinydb==4.7.1 +tqdm==4.64.1 diff --git a/setup.py b/setup.py index ec7fb83..867800f 100644 --- a/setup.py +++ b/setup.py @@ -1,28 +1,102 @@ #!/usr/bin/env python +import pathlib + +from setuptools import setup, find_packages + +from antareslauncher import __author__, __project_name__, __version__ + +# Dependencies required to install the application in "production" or "development" mode. +# Use `pip install -e .` to install in "development" mode. +# The version numbers are loosely constrained to allow installation of bugfix versions, +# but sufficiently constrained to avoid incompatibilities. +# It is the developer's responsibility to update versions: unit tests should +# detect incompatibility problems. +# Warning: this package is used as a library, so you should not constrain the versions too much. +install_requires = [ + "paramiko < 3.0", # version 3.0.0 is not mature yet (2023-01-22) + "PyYAML < 6.1", + "tinydb < 4.8", + "tqdm < 4.65", +] + +# Extra dependencies used for testing in "development" mode. +# Use `pip install -e .[test]` to install. +test_requires = [ + "pytest ~= 7.2.1", + "pytest-cov ~= 4.0.0", + "pytest-xdist ~= 3.1.0", +] + +# Extra dependencies used for developing. +# Use `pip install -e .[dev]` to install. +dev_requires = [ + "black", + "check-manifest", + "isort", + "mypy", +] + +# Extra dependencies used for documentation generation. +# Use `pip install -e .[docs]` to install. +docs_requires = [ + "m2r", + "recommonmark", + "sphinx", + "sphinx_rtd_theme", + # sphinx-rtd-theme 1.2.0 has requirement docutils<0.19 + "docutils<0.19", +] + +# Extra dependencies used to create the executable package. +# Use `pip install -e .[pyinstaller]` to install. +pyinstaller_requires = [ + "pyinstaller", +] -from distutils.core import setup -from antareslauncher import PROJECT_NAME -from antareslauncher import DESCRIPTION -from antareslauncher import VERSION setup( - name=PROJECT_NAME, - version=VERSION, - description=DESCRIPTION, - author="RTE, SGATTONI Andrea, MOZGAWA Marc, BION Charly", + name=__project_name__, + version=__version__, + description="Antares_Launcher to run Antares on a remote linux machine", + long_description=pathlib.Path("README.md").read_text(encoding="utf-8"), + long_description_content_type="text/markdown", + author=__author__, author_email="andrea.sgattoni@rte-france.com", - url="https://devin-source.rte-france.com/antares/Antares_Launcher.git", - packages=["antareslauncher"], + url="https://github.com/AntaresSimulatorTeam/antares-launcher.git", + packages=find_packages(exclude=["tests*"]), + install_requires=install_requires, + extras_require={ + "test": test_requires, + "dev": dev_requires, + "docs": docs_requires, + "pyinstaller": pyinstaller_requires, + }, + license="Apache Software License", + platforms=[ + "linux-x86_64", + "macosx-10.14-x86_64", + "macosx-10.15-x86_64", + "macosx-11-x86_64", + "macosx-12-x86_64", + "macosx-13-x86_64", + "win-amd64", + ], classifiers=[ + "Development Status :: 5 - Production/Stable", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Environment :: Console", - "License :: Other/Proprietary License", + "License :: OSI Approved :: Apache Software License", "Natural Language :: English", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", + "Operating System :: MacOS :: MacOS X", ], entry_points={ "console_scripts": ["Antares_Launcher = antareslauncher.advanced_launch:main"], }, + python_requires=">=3.8, <4", ) diff --git a/standards.md b/standards.md deleted file mode 100644 index 253e7ac..0000000 --- a/standards.md +++ /dev/null @@ -1,31 +0,0 @@ -# standards - -Limit blank lines to what is suggested to pep8 -For example in the tests: - -```python - def test_given_repo_when_get_list_of_studies_called_then_repo_get_list_of_studies_is_called(self): - # given - repo_mock = mock.Mock() - repo_mock.get_list_of_studies = mock.Mock() - study_list_composer = StudyListComposer(repo=repo_mock, file_system=None) - # when - study_list_composer.get_list_of_studies() - # then - repo_mock.get_list_of_studies.assert_called_once() -``` - -The class do not need to inherit from `object` - -Instead of -```python -class LaunchController(object): - def __init__(self): - pass -``` -use -```python -class LaunchController: - def __init__(self): - pass -``` diff --git a/tests/end_to_end/config.md b/tests/end_to_end/config.md new file mode 100644 index 0000000..832072b --- /dev/null +++ b/tests/end_to_end/config.md @@ -0,0 +1,40 @@ +# End-to-end Configuration for integration tests + +## Configuration file templates + +The configuration is defined by two file. In this directory, you can find the templates that must be customized +according to your needs: + +- The application configuration file: [configuration.yaml](configuration.yaml) +- The SSH configuration file: [ssh_config.json](ssh_config.json) files. + +## Default configuration directory + +Those files can be stored in the following locations: + +- On Windows: `C:\Users\{username}\AppData\Local\RTE`, +- On Mac: `~/Library/Preferences`, +- On Linux: `~/.config` (set by the `XDG_CONFIG_HOME` environment variable). + +To run the end-to-end test, you can do: + +```shell +cd ~/workspace/antares-launcher +pytest -v --basetemp=target/pytest/ tests/end_to_end/ +``` + +## Custom configuration directory + +An alternative is to use a custom location by setting the `ANTARES_LAUNCHER_CONFIG_PATH` environment variable. +This path should point to the `configuration.yaml` file. + +In this situation, to run the end-to-end test, you can do: + +```shell +export ANTARES_LAUNCHER_CONFIG_PATH=target/config_dir/configuration.yaml +cd ~/workspace/antares-launcher +pytest -v --basetemp=target/pytest/ tests/end_to_end/ +``` + +> **NOTE:** if the configuration file is not found, end-to-end tests are ignored +> and a warning message is printed on the console. diff --git a/tests/end_to_end/configuration.yaml b/tests/end_to_end/configuration.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/end_to_end/ssh_config.json b/tests/end_to_end/ssh_config.json new file mode 100644 index 0000000..8a4e6f6 --- /dev/null +++ b/tests/end_to_end/ssh_config.json @@ -0,0 +1,8 @@ +{ + "username": "john.doe", + "hostname": "localhost", + "port": 22, + "private_key_file": "path/to/private.key", + "key_password": "key_password", + "password": "S3Cr3T" +} \ No newline at end of file diff --git a/tests/end_to_end/test_end_to_end.py b/tests/end_to_end/test_end_to_end.py index c10c051..e69de29 100644 --- a/tests/end_to_end/test_end_to_end.py +++ b/tests/end_to_end/test_end_to_end.py @@ -1,113 +0,0 @@ -""" -This file will contain all Integration tests. Ie, global from start to finish tests -It needs a proper ssh configuration and working remote server -""" - -import contextlib -import getpass -import os -import shutil -from pathlib import Path - -import pytest - -from antareslauncher import main -from antareslauncher.main import MainParameters -from antareslauncher.main_option_parser import (MainOptionParser, - ParserParameters) -from antareslauncher.parameters_reader import ParametersReader -from tests.data import DATA_DIR - - -def get_test_config(): - return (DATA_DIR / "configuration.yaml").exists() - - -# You should define the `ANTARES_LAUNCHER_CONFIG_PATH` environment variable -# to run end-to-end tests. This variable should point to your "configuration.yaml" file. -TEST_CONFIG = get_test_config() - -ANTARES_STUDY = DATA_DIR / "STUDIES-IN-FOR-TEST" / "one_node_v7" -EXAMPLE_STUDIES_IN = DATA_DIR / "STUDIES-IN-FOR-TEST" -SSH_JSON_FILE = DATA_DIR / "ssh_config.json" -YAML_CONF_FILE = DATA_DIR / "configuration.yaml" - - -class TestEndToEnd: - def setup_method(self): - self.studies_in_path = Path.cwd() / "STUDIES-IN" - self.finished_path = Path.cwd() / "FINISHED" - self.log_path = Path.cwd() / "LOGS" - self.json_db_file_path = ( - Path.cwd() / f"{getpass.getuser()}_antares_launcher_db.json" - ) - with contextlib.suppress(FileNotFoundError): - self.json_db_file_path.unlink() - param_reader = ParametersReader( - json_ssh_conf=SSH_JSON_FILE, yaml_filepath=YAML_CONF_FILE - ) - parser_parameters: ParserParameters = param_reader.get_parser_parameters() - self.parser: MainOptionParser = MainOptionParser(parameters=parser_parameters) - self.parser.add_basic_arguments().add_advanced_arguments() - self.main_parameters: MainParameters = param_reader.get_main_parameters() - - def teardown_method(self): - shutil.rmtree(self.finished_path) - shutil.rmtree(self.log_path) - self.json_db_file_path.unlink() - - @pytest.mark.end_to_end_test - @pytest.mark.skipif( - not TEST_CONFIG, - reason="end-to-end config not found: read 'config.md' for more info", - ) - def test_when_run_on_an_empty_directory_the_tree_structure_is_initialised( - self, - ): - input_arguments = self.parser.parse_args([]) - - main.run_with(input_arguments, self.main_parameters) - - assert self.studies_in_path.is_dir() - assert self.finished_path.is_dir() - assert self.log_path.is_dir() - assert self.json_db_file_path.is_file() - - @pytest.mark.end_to_end_test - @pytest.mark.skipif( - not TEST_CONFIG, - reason="end-to-end config not found: read 'config.md' for more info", - ) - def test_one_study_is_correctly_processed(self): - arg_wait_mode = ["-w"] - arg_wait_time = ["--wait-time", "2"] - arg_2_cpu = ["-n", "2"] - arg_studies_in = ["-i", f"{str(EXAMPLE_STUDIES_IN)}"] - arguments = arg_wait_mode + arg_wait_time + arg_2_cpu + arg_studies_in - input_arguments = self.parser.parse_args(arguments) - - main.run_with(input_arguments, self.main_parameters) - - assert os.listdir(self.finished_path / ANTARES_STUDY.name) - assert os.listdir(self.finished_path / ANTARES_STUDY.name / "output") - - @pytest.mark.end_to_end_test - @pytest.mark.skipif( - not TEST_CONFIG, - reason="end-to-end config not found: read 'config.md' for more info", - ) - def test_one_xpansion_study_is_correctly_processed(self): - arg_xpansion = ["-x", "r"] - arg_wait_mode = ["-w"] - arg_wait_time = ["--wait-time", "2"] - arg_2_cpu = ["-n", "2"] - arg_studies_in = ["-i", f"{str(EXAMPLE_STUDIES_IN)}"] - arguments = ( - arg_xpansion + arg_wait_mode + arg_wait_time + arg_2_cpu + arg_studies_in - ) - input_arguments = self.parser.parse_args(arguments) - - main.run_with(input_arguments, self.main_parameters) - - assert os.listdir(self.finished_path / ANTARES_STUDY.name) - assert os.listdir(self.finished_path / ANTARES_STUDY.name / "output") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..8327fd8 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,401 @@ +import contextlib +import getpass +import json +import pathlib +from pathlib import Path +from unittest.mock import patch + +import pytest +import yaml +from antareslauncher.config import ( + APP_AUTHOR, + APP_NAME, + APP_VERSION, + CONFIGURATION_YAML, + Config, + SSHConfig, + dump_config, + get_config_path, + get_user_config_dir, + parse_config, +) +from antareslauncher.exceptions import ( + ConfigFileNotFoundError, + InvalidConfigValueError, + UnknownFileSuffixError, +) + + +class TestParseConfig: + @pytest.mark.parametrize("suffix", [".yaml", ".yml", ".json", ".py"]) + @pytest.mark.parametrize("casing", [None, str.upper, str.title]) + def test_parse_config(self, tmp_path, suffix, casing): + data = {"key1": "value1", "key2": 56} + # noinspection PyArgumentList + new_suffix = suffix if casing is None else casing(suffix) + config_path = tmp_path.joinpath(f"my_config{new_suffix}") + if suffix in {".yaml", ".yml"}: + with config_path.open(mode="w", encoding="utf-8") as fd: + yaml.dump(data, fd) + actual = parse_config(config_path) + assert actual == data + elif suffix in {".json"}: + config_path.write_text(json.dumps(data), encoding="utf-8") + actual = parse_config(config_path) + assert actual == data + else: + config_path.write_text(repr(data), encoding="utf-8") + with pytest.raises(UnknownFileSuffixError): + parse_config(config_path) + + +class TestSaveConfig: + @pytest.mark.parametrize("suffix", [".yaml", ".yml", ".json", ".py"]) + @pytest.mark.parametrize("casing", [None, str.upper, str.title]) + def test_save_config(self, tmp_path, suffix, casing): + data = {"key1": "value1", "key2": 56} + # noinspection PyArgumentList + new_suffix = suffix if casing is None else casing(suffix) + config_path = tmp_path.joinpath(f"my_config{new_suffix}") + if suffix in {".yaml", ".yml"}: + dump_config(config_path, data) + with config_path.open(encoding="utf-8") as fd: + actual = yaml.load(fd, Loader=yaml.FullLoader) + assert actual == data + elif suffix in {".json"}: + dump_config(config_path, data) + actual = json.loads(config_path.read_text(encoding="utf-8")) + assert actual == data + else: + with pytest.raises(UnknownFileSuffixError): + dump_config(config_path, data) + + +class TestSSHConfig: + def test_load_config__with_private_key_file(self, tmp_path): + data = { + "username": "john.doe", + "hostname": "localhost", + "port": 22, + "private_key_file": "path/to/private.key", + "key_password": "key_password", + } + config_path = tmp_path.joinpath("my_ssh_config.json") + config_path.write_text(json.dumps(data), encoding="utf-8") + config = SSHConfig.load_config(config_path) + assert config.config_path == config_path + assert config.username == data["username"] + assert config.hostname == data["hostname"] + assert config.port == data["port"] + assert config.private_key_file == Path(data["private_key_file"]) + assert config.key_password == data["key_password"] + assert config.password == "" + + def test_load_config__with_password(self, tmp_path): + data = { + "username": "john.doe", + "hostname": "localhost", + "port": 22, + "password": "S3Cr3T", + } + config_path = tmp_path.joinpath("my_ssh_config.json") + config_path.write_text(json.dumps(data), encoding="utf-8") + config = SSHConfig.load_config(config_path) + assert config.config_path == config_path + assert config.username == data["username"] + assert config.hostname == data["hostname"] + assert config.port == data["port"] + assert config.private_key_file is None + assert config.key_password == "" + assert config.password == data["password"] + + @pytest.mark.parametrize("required", ["username", "hostname"]) + def test_load_config__missing_parameter(self, tmp_path, required): + data = { + "username": "john.doe", + "hostname": "localhost", + "port": 22, + "password": "S3Cr3T", + } + del data[required] + config_path = tmp_path.joinpath("my_ssh_config.json") + config_path.write_text(json.dumps(data), encoding="utf-8") + with pytest.raises(InvalidConfigValueError): + SSHConfig.load_config(config_path) + + def test_save_config__with_private_key_file(self, tmp_path): + config_path = tmp_path.joinpath("my_ssh_config.json") + config = SSHConfig( + config_path=config_path, + username="john.doe", + hostname="localhost", + port=22, + private_key_file=pathlib.Path("path/to/private.key"), + key_password="key_password", + ) + config.save_config(config_path) + actual = json.loads(config_path.read_text(encoding="utf-8")) + assert "config_path" not in actual + assert actual["username"] == config.username + assert actual["hostname"] == config.hostname + assert actual["port"] == config.port + assert actual["private_key_file"] == str(config.private_key_file) + assert actual["key_password"] == config.key_password + assert "password" not in actual + + def test_save_config__with_password(self, tmp_path): + config_path = tmp_path.joinpath("my_ssh_config.json") + config = SSHConfig( + config_path=config_path, + username="john.doe", + hostname="localhost", + port=22, + password="S3Cr3T", + ) + config.save_config(config_path) + actual = json.loads(config_path.read_text(encoding="utf-8")) + assert "config_path" not in actual + assert actual["username"] == config.username + assert actual["hostname"] == config.hostname + assert actual["port"] == config.port + assert "private_key_file" not in actual + assert "key_password" not in actual + assert actual["password"] == config.password + + +class TestConfig: + @pytest.fixture(name="ssh_config_path") + def fixture_ssh_config_path(self, tmp_path) -> pathlib.Path: + data = { + "username": "john.doe", + "hostname": "localhost", + "port": 22, + "password": "S3Cr3T", + } + config_path = tmp_path.joinpath("ssh_config.json") + config_path.write_text(json.dumps(data), encoding="utf-8") + return config_path + + @pytest.fixture(name="ssh_config") + def fixture_ssh_config(self, tmp_path) -> SSHConfig: + config_path = tmp_path.joinpath("ssh_config.json") + return SSHConfig( + config_path=config_path, + username="john.doe", + hostname="localhost", + port=22, + password="S3Cr3T", + ) + + def test_load_config__nominal(self, tmp_path, ssh_config_path): + log_dir = tmp_path.joinpath("log_dir") + json_dir = tmp_path.joinpath("json_dir") + studies_in_dir = tmp_path.joinpath("studies_in_dir") + finished_dir = tmp_path.joinpath("finished_dir") + slurm_script_path = tmp_path.joinpath("slurm_script.sh") + data = { + "log_dir": str(log_dir), + "json_dir": str(json_dir), + "studies_in_dir": str(studies_in_dir), + "finished_dir": str(finished_dir), + "default_time_limit": 3600, + "default_n_cpu": 2, + "default_wait_time": 60, + "db_primary_key": "pk", + "default_ssh_configfile_name": ssh_config_path.name, + "ssh_config_file_is_required": True, + "slurm_script_path": str(slurm_script_path), + "antares_versions_on_remote_server": ["8.4.0", "8.5.0"], + } + config_path = tmp_path.joinpath("configuration.yaml") + with config_path.open(mode="w", encoding="utf-8") as fd: + yaml.dump(data, fd) + config = Config.load_config(config_path) + assert config.config_path == config_path + assert config.log_dir == log_dir + assert config.json_dir == json_dir + assert config.studies_in_dir == studies_in_dir + assert config.finished_dir == finished_dir + assert config.default_time_limit == data["default_time_limit"] + assert config.default_n_cpu == data["default_n_cpu"] + assert config.default_wait_time == data["default_wait_time"] + assert config.db_primary_key == data["db_primary_key"] + assert config.ssh_config_file_is_required == data["ssh_config_file_is_required"] + assert config.slurm_script_path == slurm_script_path + assert ( + config.remote_solver_versions == data["antares_versions_on_remote_server"] + ) + + def test_save_config__nominal(self, tmp_path, ssh_config): + config_path = tmp_path.joinpath("configuration.yaml") + log_dir = tmp_path.joinpath("log_dir") + json_dir = tmp_path.joinpath("json_dir") + studies_in_dir = tmp_path.joinpath("studies_in_dir") + finished_dir = tmp_path.joinpath("finished_dir") + slurm_script_path = tmp_path.joinpath("slurm_script.sh") + config = Config( + config_path=config_path, + log_dir=log_dir, + json_dir=json_dir, + studies_in_dir=studies_in_dir, + finished_dir=finished_dir, + default_time_limit=3600, + default_n_cpu=2, + default_wait_time=60, + db_primary_key="pk", + ssh_config_file_is_required=True, + slurm_script_path=slurm_script_path, + remote_solver_versions=["8.4.0", "8.5.0"], + ssh_config=ssh_config, + ) + config.save_config(config_path) + with config_path.open(encoding="utf-8") as fd: + actual = yaml.load(fd, Loader=yaml.FullLoader) + + assert "config_path" not in actual + assert actual["log_dir"] == str(log_dir) + assert actual["json_dir"] == str(json_dir) + assert actual["studies_in_dir"] == str(studies_in_dir) + assert actual["finished_dir"] == str(finished_dir) + assert actual["default_time_limit"] == config.default_time_limit + assert actual["default_n_cpu"] == config.default_n_cpu + assert actual["default_wait_time"] == config.default_wait_time + assert actual["db_primary_key"] == config.db_primary_key + assert ( + actual["ssh_config_file_is_required"] == config.ssh_config_file_is_required + ) + assert actual["slurm_script_path"] == str(slurm_script_path) + assert ( + actual["antares_versions_on_remote_server"] == config.remote_solver_versions + ) + assert "ssh_config" not in actual + + @pytest.mark.parametrize( + "required", + [ + "log_dir", + "json_dir", + "studies_in_dir", + "finished_dir", + "default_time_limit", + "default_n_cpu", + "default_wait_time", + "db_primary_key", + "default_ssh_configfile_name", + "ssh_config_file_is_required", + "slurm_script_path", + "antares_versions_on_remote_server", + ], + ) + def test_load_config__missing_parameter(self, tmp_path, ssh_config_path, required): + log_dir = tmp_path.joinpath("log_dir") + json_dir = tmp_path.joinpath("json_dir") + studies_in_dir = tmp_path.joinpath("studies_in_dir") + finished_dir = tmp_path.joinpath("finished_dir") + slurm_script_path = tmp_path.joinpath("slurm_script.sh") + data = { + "log_dir": str(log_dir), + "json_dir": str(json_dir), + "studies_in_dir": str(studies_in_dir), + "finished_dir": str(finished_dir), + "default_time_limit": 3600, + "default_n_cpu": 2, + "default_wait_time": 60, + "db_primary_key": "pk", + "default_ssh_configfile_name": ssh_config_path.name, + "ssh_config_file_is_required": True, + "slurm_script_path": str(slurm_script_path), + "antares_versions_on_remote_server": ["8.4.0", "8.5.0"], + } + del data[required] + config_path = tmp_path.joinpath("configuration.yaml") + with config_path.open(mode="w", encoding="utf-8") as fd: + yaml.dump(data, fd) + with pytest.raises(InvalidConfigValueError): + Config.load_config(config_path) + + +class TestGetUserConfigDir: + @pytest.mark.parametrize( + "system, expected", + [ + ( + "win32", + r"C:\Users\{username}\AppData\Local\{APP_AUTHOR}\{APP_NAME}\{APP_VERSION}", + ), + ( + "darwin", + "~/Library/Preferences/{APP_NAME}/{APP_VERSION}", + ), + ( + "linux2", + "~/.config/{APP_NAME}/{APP_VERSION}", + ), + ], + ) + def test_get_user_config_dir(self, system, expected, monkeypatch): + # ignore error `XDG_CONFIG_HOME` environment variable + monkeypatch.delenv("XDG_CONFIG_HOME", raising=False) + # ignore error: "cannot instantiate 'WindowsPath'/'PosixPath' on your system" + with contextlib.suppress(NotImplementedError): + actual = get_user_config_dir(system) + expected = expected.format( + username=getpass.getuser(), + APP_AUTHOR=APP_AUTHOR, + APP_NAME=APP_NAME, + APP_VERSION=APP_VERSION, + ) + expected = Path(expected).expanduser() + assert actual == expected + + +class TestGetConfigPath: + def test_get_config_path__from_env(self, monkeypatch, tmp_path): + config_path = tmp_path.joinpath("my_config.yaml") + config_path.touch() + monkeypatch.setenv("ANTARES_LAUNCHER_CONFIG_PATH", str(config_path)) + actual = get_config_path() + assert actual == config_path + + def test_get_config_path__from_env__not_found(self, monkeypatch, tmp_path): + config_path = tmp_path.joinpath("my_config.yaml") + monkeypatch.setenv("ANTARES_LAUNCHER_CONFIG_PATH", str(config_path)) + with pytest.raises(ConfigFileNotFoundError): + get_config_path() + + @pytest.mark.parametrize("config_name", [None, CONFIGURATION_YAML, "my_config.yaml"]) + def test_get_config_path__from_user_config_dir( + self, monkeypatch, tmp_path, config_name + ): + config_path = tmp_path.joinpath(config_name or CONFIGURATION_YAML) + config_path.touch() + monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) + # noinspection SpellCheckingInspection + with patch("antareslauncher.config.get_user_config_dir", new=lambda: tmp_path): + args = (config_name,) if config_name else () + actual = get_config_path(*args) + assert actual == config_path + + @pytest.mark.parametrize("relpath", ["", "data"]) + @pytest.mark.parametrize("config_name", [None, CONFIGURATION_YAML, "my_config.yaml"]) + def test_get_config_path__from_curr_dir(self, monkeypatch, tmp_path, relpath, config_name): + data_dir = tmp_path.joinpath(relpath) + data_dir.mkdir(exist_ok=True) + config_path: pathlib.Path = tmp_path.joinpath(data_dir, config_name or CONFIGURATION_YAML) + config_path.touch() + monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) + monkeypatch.chdir(tmp_path) + args = (config_name,) if config_name else () + actual = get_config_path(*args) + assert actual == config_path.relative_to(tmp_path) + + @pytest.mark.parametrize("relpath", ["", "data"]) + def test_get_config_path__from_curr_dir__not_found( + self, monkeypatch, tmp_path, relpath + ): + data_dir = tmp_path.joinpath(relpath) + data_dir.mkdir(exist_ok=True) + monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) + monkeypatch.chdir(tmp_path) + with pytest.raises(ConfigFileNotFoundError): + get_config_path()