diff --git a/antareslauncher/antares_launcher.py b/antareslauncher/antares_launcher.py index 07a526f..52166d4 100644 --- a/antareslauncher/antares_launcher.py +++ b/antareslauncher/antares_launcher.py @@ -1,16 +1,12 @@ from dataclasses import dataclass from typing import Optional -from antareslauncher.use_cases.check_remote_queue.check_queue_controller import ( - CheckQueueController, -) +from antareslauncher.use_cases.check_remote_queue.check_queue_controller import CheckQueueController from antareslauncher.use_cases.create_list.study_list_composer import StudyListComposer from antareslauncher.use_cases.kill_job.job_kill_controller import JobKillController from antareslauncher.use_cases.launch.launch_controller import LaunchController from antareslauncher.use_cases.retrieve.retrieve_controller import RetrieveController -from antareslauncher.use_cases.wait_loop_controller.wait_controller import ( - WaitController, -) +from antareslauncher.use_cases.wait_loop_controller.wait_controller import WaitController @dataclass diff --git a/antareslauncher/config.py b/antareslauncher/config.py index 760d526..67f2564 100644 --- a/antareslauncher/config.py +++ b/antareslauncher/config.py @@ -12,11 +12,7 @@ import yaml from antareslauncher import __author__, __project_name__, __version__ -from antareslauncher.exceptions import ( - ConfigFileNotFoundError, - InvalidConfigValueError, - UnknownFileSuffixError, -) +from antareslauncher.exceptions import ConfigFileNotFoundError, InvalidConfigValueError, UnknownFileSuffixError APP_NAME = __project_name__ APP_AUTHOR = __author__.split(",")[0] @@ -119,9 +115,7 @@ def load_config(cls, ssh_config_path: pathlib.Path) -> "SSHConfig": obj = parse_config(ssh_config_path) kwargs = {k.lower(): v for k, v in obj.items()} private_key_file = kwargs.pop("private_key_file", None) - kwargs["private_key_file"] = ( - None if private_key_file is None else pathlib.Path(private_key_file) - ) + kwargs["private_key_file"] = None if private_key_file is None else pathlib.Path(private_key_file) try: return cls(config_path=ssh_config_path, **kwargs) except TypeError as exc: @@ -139,11 +133,7 @@ def save_config(self, ssh_config_path: pathlib.Path) -> None: """ obj = dataclasses.asdict(self) del obj["config_path"] - obj = { - k: v - for k, v in obj.items() - if v or k not in {"private_key_file", "key_password", "password"} - } + obj = {k: v for k, v in obj.items() if v or k not in {"private_key_file", "key_password", "password"}} if "private_key_file" in obj: obj["private_key_file"] = obj["private_key_file"].as_posix() dump_config(ssh_config_path, obj) @@ -212,9 +202,7 @@ def load_config(cls, config_path: pathlib.Path) -> "Config": obj = parse_config(config_path) kwargs = {k.lower(): v for k, v in obj.items()} try: - kwargs["remote_solver_versions"] = kwargs.pop( - "antares_versions_on_remote_server" - ) + kwargs["remote_solver_versions"] = kwargs.pop("antares_versions_on_remote_server") # handle paths for key in [ "log_dir", @@ -226,9 +214,7 @@ def load_config(cls, config_path: pathlib.Path) -> "Config": kwargs[key] = pathlib.Path(kwargs[key]) ssh_configfile_name = kwargs.pop("default_ssh_configfile_name") except KeyError as exc: - raise InvalidConfigValueError( - config_path, f"missing parameter '{exc}'" - ) from None + raise InvalidConfigValueError(config_path, f"missing parameter '{exc}'") from None # handle SSH configuration config_dir = config_path.parent ssh_config_path = config_dir.joinpath(ssh_configfile_name) @@ -287,9 +273,7 @@ def get_user_config_dir(system: str = ""): username = getpass.getuser() system = system or sys.platform if system == "win32": - config_dir = pathlib.WindowsPath( - rf"C:\Users\{username}\AppData\Local\{APP_AUTHOR}" - ) + config_dir = pathlib.WindowsPath(rf"C:\Users\{username}\AppData\Local\{APP_AUTHOR}") elif system == "darwin": config_dir = pathlib.PosixPath("~/Library/Preferences").expanduser() else: diff --git a/antareslauncher/display/display_terminal.py b/antareslauncher/display/display_terminal.py index acebffc..cbd0fbc 100644 --- a/antareslauncher/display/display_terminal.py +++ b/antareslauncher/display/display_terminal.py @@ -58,8 +58,6 @@ def generate_progress_bar( desc=desc, leave=False, dynamic_ncols=True, - bar_format="[" - + str(now.strftime(self.format)) - + "] {l_bar}{bar}| {n_fmt}/{total_fmt} ", + bar_format="[" + str(now.strftime(self.format)) + "] {l_bar}{bar}| {n_fmt}/{total_fmt} ", ) return progress_bar diff --git a/antareslauncher/exceptions.py b/antareslauncher/exceptions.py index 201abfa..491e532 100644 --- a/antareslauncher/exceptions.py +++ b/antareslauncher/exceptions.py @@ -12,9 +12,7 @@ class AntaresLauncherException(Exception): class ConfigFileNotFoundError(AntaresLauncherException): """Configuration file not found.""" - def __init__( - self, possible_dirs: Sequence[pathlib.Path], config_name: str, *args - ) -> None: + def __init__(self, possible_dirs: Sequence[pathlib.Path], config_name: str, *args) -> None: super().__init__(possible_dirs, config_name, *args) @property diff --git a/antareslauncher/file_manager/file_manager.py b/antareslauncher/file_manager/file_manager.py index 4cc15b3..ad7d548 100644 --- a/antareslauncher/file_manager/file_manager.py +++ b/antareslauncher/file_manager/file_manager.py @@ -51,9 +51,7 @@ def _get_list_of_files_recursively(self, element_path): Returns: List of all the files inside a directory recursively """ - self.logger.info( - f"Getting list of all files inside the directory {element_path}" - ) + self.logger.info(f"Getting list of all files inside the directory {element_path}") element_file_paths = [] for root, _, files in os.walk(element_path): for filename in files: @@ -71,9 +69,7 @@ def _get_complete_list_of_files_and_dirs_in_list_dir(self, dir_path, list_dir): file_paths.extend(element_file_paths) return file_paths - def zip_file_paths_with_rootdir_to_zipfile_path( - self, zipfile_path, file_paths, root_dir - ): + def zip_file_paths_with_rootdir_to_zipfile_path(self, zipfile_path, file_paths, root_dir): """Zips all the files in file_paths inside zipfile_path while printing a progress bar on the terminal @@ -85,12 +81,8 @@ def zip_file_paths_with_rootdir_to_zipfile_path( root_dir: Root directory """ self.logger.info(f"Zipping list of files to archive {zipfile_path}") - with zipfile.ZipFile( - zipfile_path, "w", compression=zipfile.ZIP_DEFLATED - ) as my_zip: - loading_bar = self.display.generate_progress_bar( - file_paths, desc="Compressing files: " - ) + with zipfile.ZipFile(zipfile_path, "w", compression=zipfile.ZIP_DEFLATED) as my_zip: + loading_bar = self.display.generate_progress_bar(file_paths, desc="Compressing files: ") for f in loading_bar: my_zip.write(f, os.path.relpath(f, root_dir)) @@ -105,13 +97,9 @@ def zip_dir_excluding_subdir(self, dir_path, zipfile_path, subdir_to_exclude): subdir_to_exclude: Subdirectory that will not be zipped """ list_dir = self._get_list_dir_without_subdir(dir_path, subdir_to_exclude) - file_paths = self._get_complete_list_of_files_and_dirs_in_list_dir( - dir_path, list_dir - ) + file_paths = self._get_complete_list_of_files_and_dirs_in_list_dir(dir_path, list_dir) root_dir = str(Path(dir_path).parent) - self.zip_file_paths_with_rootdir_to_zipfile_path( - zipfile_path, file_paths, root_dir - ) + self.zip_file_paths_with_rootdir_to_zipfile_path(zipfile_path, file_paths, root_dir) return Path(zipfile_path).is_file() def make_dir(self, directory_name): @@ -124,9 +112,7 @@ def convert_json_file_to_dict(self, file_path): with open(file_path, "r") as readFile: config = json.load(readFile) except OSError: - self.logger.error( - f"Unable to convert {file_path} to json (file not found or invalid type)" - ) + self.logger.error(f"Unable to convert {file_path} to json (file not found or invalid type)") config = None return config diff --git a/antareslauncher/logger_initializer.py b/antareslauncher/logger_initializer.py index e242193..cca6d11 100644 --- a/antareslauncher/logger_initializer.py +++ b/antareslauncher/logger_initializer.py @@ -13,12 +13,8 @@ def init_logger(self): Returns: """ - formatter = logging.Formatter( - "%(asctime)s - %(levelname)s - %(name)s - %(message)s" - ) - f_handler = RotatingFileHandler( - self.file_path, maxBytes=200000, backupCount=5, mode="a+" - ) + formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(name)s - %(message)s") + f_handler = RotatingFileHandler(self.file_path, maxBytes=200000, backupCount=5, mode="a+") f_handler.setFormatter(formatter) f_handler.setLevel(logging.DEBUG) logging.basicConfig(level=logging.INFO, handlers=[f_handler]) diff --git a/antareslauncher/main.py b/antareslauncher/main.py index 233910d..2f2eb77 100644 --- a/antareslauncher/main.py +++ b/antareslauncher/main.py @@ -10,30 +10,17 @@ from antareslauncher.file_manager.file_manager import FileManager from antareslauncher.logger_initializer import LoggerInitializer from antareslauncher.remote_environnement import ssh_connection -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) -from antareslauncher.remote_environnement.slurm_script_features import ( - SlurmScriptFeatures, -) -from antareslauncher.use_cases.check_remote_queue.check_queue_controller import ( - CheckQueueController, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm +from antareslauncher.remote_environnement.slurm_script_features import SlurmScriptFeatures +from antareslauncher.use_cases.check_remote_queue.check_queue_controller import CheckQueueController from antareslauncher.use_cases.check_remote_queue.slurm_queue_show import SlurmQueueShow -from antareslauncher.use_cases.create_list.study_list_composer import ( - StudyListComposer, - StudyListComposerParameters, -) -from antareslauncher.use_cases.generate_tree_structure.tree_structure_initializer import ( - TreeStructureInitializer, -) +from antareslauncher.use_cases.create_list.study_list_composer import StudyListComposer, StudyListComposerParameters +from antareslauncher.use_cases.generate_tree_structure.tree_structure_initializer import TreeStructureInitializer from antareslauncher.use_cases.kill_job.job_kill_controller import JobKillController from antareslauncher.use_cases.launch.launch_controller import LaunchController from antareslauncher.use_cases.retrieve.retrieve_controller import RetrieveController from antareslauncher.use_cases.retrieve.state_updater import StateUpdater -from antareslauncher.use_cases.wait_loop_controller.wait_controller import ( - WaitController, -) +from antareslauncher.use_cases.wait_loop_controller.wait_controller import WaitController class NoJsonConfigFileError(Exception): @@ -88,9 +75,7 @@ class MainParameters: quality_of_service: str = "" -def run_with( - arguments: argparse.Namespace, parameters: MainParameters, show_banner=False -): +def run_with(arguments: argparse.Namespace, parameters: MainParameters, show_banner=False): """Instantiates all the objects necessary to antares-launcher, and runs the program""" if arguments.version: print(f"Antares_Launcher v{__version__}") @@ -113,9 +98,7 @@ def run_with( ) tree_structure_initializer.init_tree_structure() - logger_initializer = LoggerInitializer( - str(Path(arguments.log_dir) / "antares_launcher.log") - ) + logger_initializer = LoggerInitializer(str(Path(arguments.log_dir) / "antares_launcher.log")) logger_initializer.init_logger() # connection @@ -133,9 +116,7 @@ def run_with( quality_of_service=parameters.quality_of_service, ) environment = RemoteEnvironmentWithSlurm(connection, slurm_script_features) - data_repo = DataRepoTinydb( - database_file_path=db_json_file_path, db_primary_key=parameters.db_primary_key - ) + data_repo = DataRepoTinydb(database_file_path=db_json_file_path, db_primary_key=parameters.db_primary_key) study_list_composer = StudyListComposer( repo=data_repo, display=display, diff --git a/antareslauncher/main_option_parser.py b/antareslauncher/main_option_parser.py index e37160a..b9780de 100644 --- a/antareslauncher/main_option_parser.py +++ b/antareslauncher/main_option_parser.py @@ -49,9 +49,7 @@ def __init__(self, parameters: ParserParameters) -> None: def parse_args(self, args: t.Union[t.Sequence[str], None]) -> argparse.Namespace: return self.parser.parse_args(args) - def add_basic_arguments( - self, *, antares_versions: t.Sequence[str] = () - ) -> MainOptionParser: + def add_basic_arguments(self, *, antares_versions: t.Sequence[str] = ()) -> MainOptionParser: """Adds to the parser all the arguments for the light mode""" self.parser.add_argument( "-w", @@ -234,15 +232,9 @@ def look_for_default_ssh_conf_file( Returns: path to the ssh config file is it exists, None otherwise """ - if ( - parameters.ssh_configfile_path_alternate1 - and parameters.ssh_configfile_path_alternate1.is_file() - ): + if parameters.ssh_configfile_path_alternate1 and parameters.ssh_configfile_path_alternate1.is_file(): return parameters.ssh_configfile_path_alternate1 - elif ( - parameters.ssh_configfile_path_alternate2 - and parameters.ssh_configfile_path_alternate2.is_file() - ): + elif parameters.ssh_configfile_path_alternate2 and parameters.ssh_configfile_path_alternate2.is_file(): return parameters.ssh_configfile_path_alternate2 else: return None diff --git a/antareslauncher/parameters_reader.py b/antareslauncher/parameters_reader.py index 0b94695..6e04777 100644 --- a/antareslauncher/parameters_reader.py +++ b/antareslauncher/parameters_reader.py @@ -87,7 +87,5 @@ def _get_ssh_dict_from_json(self) -> t.Dict[str, t.Any]: with open(self.json_ssh_conf) as ssh_connection_json: ssh_dict = json.load(ssh_connection_json) if "private_key_file" in ssh_dict: - ssh_dict["private_key_file"] = os.path.expanduser( - ssh_dict["private_key_file"] - ) + ssh_dict["private_key_file"] = os.path.expanduser(ssh_dict["private_key_file"]) return ssh_dict diff --git a/antareslauncher/remote_environnement/remote_environment_with_slurm.py b/antareslauncher/remote_environnement/remote_environment_with_slurm.py index d1f0a03..90b8e4a 100644 --- a/antareslauncher/remote_environnement/remote_environment_with_slurm.py +++ b/antareslauncher/remote_environnement/remote_environment_with_slurm.py @@ -9,10 +9,7 @@ import typing as t from pathlib import Path, PurePosixPath -from antareslauncher.remote_environnement.slurm_script_features import ( - ScriptParametersDTO, - SlurmScriptFeatures, -) +from antareslauncher.remote_environnement.slurm_script_features import ScriptParametersDTO, SlurmScriptFeatures from antareslauncher.remote_environnement.ssh_connection import SshConnection from antareslauncher.study_dto import StudyDTO @@ -25,20 +22,13 @@ class RemoteEnvBaseError(Exception): class GetJobStateError(RemoteEnvBaseError): def __init__(self, job_id: int, job_name: str, reason: str): - msg = ( - f"Unable to retrieve the status of the SLURM job {job_id}" - f" (study job '{job_name})." - f" {reason}" - ) + msg = f"Unable to retrieve the status of the SLURM job {job_id} (study job '{job_name}). {reason}" super().__init__(msg) class JobNotFoundError(RemoteEnvBaseError): def __init__(self, job_id: int, job_name: str): - msg = ( - f"Unable to retrieve the status of the SLURM job {job_id}" - f" (study job '{job_name}): Job not found." - ) + msg = f"Unable to retrieve the status of the SLURM job {job_id} (study job '{job_name}): Job not found." super().__init__(msg) @@ -141,9 +131,7 @@ def __init__( def _initialise_remote_path(self): remote_home_dir = PurePosixPath(self.connection.home_dir) - remote_base_path = remote_home_dir.joinpath( - f"REMOTE_{getpass.getuser()}_{socket.gethostname()}" - ) + remote_base_path = remote_home_dir.joinpath(f"REMOTE_{getpass.getuser()}_{socket.gethostname()}") self.remote_base_path = str(remote_base_path) if not self.connection.make_dir(self.remote_base_path): raise NoRemoteBaseDirError(remote_base_path) @@ -212,9 +200,7 @@ def submit_job(self, my_study: StudyDTO): Raises: SubmitJobErrorException if the job has not been successfully submitted """ - time_limit = self.convert_time_limit_from_seconds_to_minutes( - my_study.time_limit - ) + time_limit = self.convert_time_limit_from_seconds_to_minutes(my_study.time_limit) script_params = ScriptParametersDTO( study_dir_name=Path(my_study.path).name, input_zipfile_name=Path(my_study.zipfile_path).name, @@ -233,15 +219,10 @@ def submit_job(self, my_study: StudyDTO): raise SubmitJobError(my_study.name, reason) # should match "Submitted batch job 123456" - if match := re.match( - r"Submitted.*?(?P\d+)", output, flags=re.IGNORECASE - ): + if match := re.match(r"Submitted.*?(?P\d+)", output, flags=re.IGNORECASE): return int(match["job_id"]) - reason = ( - f"The command [{command}] return an non-parsable output:" - f"\n{textwrap.indent(output, 'OUTPUT> ')}" - ) + reason = f"The command [{command}] return an non-parsable output:\n{textwrap.indent(output, 'OUTPUT> ')}" raise SubmitJobError(my_study.name, reason) def get_job_state_flags( @@ -270,8 +251,7 @@ def get_job_state_flags( if job_state is None: # noinspection SpellCheckingInspection logger.info( - f"Job '{study.job_id}' no longer active in SLURM," - f" the job status is read from the SACCT database..." + f"Job '{study.job_id}' no longer active in SLURM, the job status is read from the SACCT database..." ) job_state = self._retrieve_slurm_acct_state( study.job_id, @@ -333,10 +313,7 @@ def _retrieve_slurm_control_state( if match := re.search(r"JobState=(\w+)", output): return JobStateCodes(match[1]) - reason = ( - f"The command [{command}] return an non-parsable output:" - f"\n{textwrap.indent(output, 'OUTPUT> ')}" - ) + reason = f"The command [{command}] return an non-parsable output:\n{textwrap.indent(output, 'OUTPUT> ')}" raise GetJobStateError(job_id, job_name, reason) def _retrieve_slurm_acct_state( @@ -374,10 +351,7 @@ def _retrieve_slurm_acct_state( last_error = error time.sleep(sleep_time) else: - reason = ( - f"The command [{command}] failed after {attempts} attempts:" - f" {last_error}" - ) + reason = f"The command [{command}] failed after {attempts} attempts: {last_error}" raise GetJobStateError(job_id, job_name, reason) # When the output is empty it mean that the job is not found @@ -395,10 +369,7 @@ def _retrieve_slurm_acct_state( job_state_str = re.match(r"(\w+)", out_state)[1] return JobStateCodes(job_state_str) - reason = ( - f"The command [{command}] return an non-parsable output:" - f"\n{textwrap.indent(output, 'OUTPUT> ')}" - ) + reason = f"The command [{command}] return an non-parsable output:\n{textwrap.indent(output, 'OUTPUT> ')}" raise GetJobStateError(job_id, job_name, reason) def upload_file(self, src) -> bool: @@ -474,9 +445,7 @@ def remove_input_zipfile(self, study: StudyDTO) -> bool: """ if not study.input_zipfile_removed: zip_name = Path(study.zipfile_path).name - study.input_zipfile_removed = self.connection.remove_file( - f"{self.remote_base_path}/{zip_name}" - ) + study.input_zipfile_removed = self.connection.remove_file(f"{self.remote_base_path}/{zip_name}") return study.input_zipfile_removed def remove_remote_final_zipfile(self, study: StudyDTO) -> bool: @@ -488,9 +457,7 @@ def remove_remote_final_zipfile(self, study: StudyDTO) -> bool: Returns: True if the file has been successfully removed, False otherwise """ - return self.connection.remove_file( - f"{self.remote_base_path}/{Path(study.local_final_zipfile_path).name}" - ) + return self.connection.remove_file(f"{self.remote_base_path}/{Path(study.local_final_zipfile_path).name}") def clean_remote_server(self, study: StudyDTO) -> bool: """ @@ -505,6 +472,5 @@ def clean_remote_server(self, study: StudyDTO) -> bool: return ( False if study.remote_server_is_clean - else self.remove_remote_final_zipfile(study) - & self.remove_input_zipfile(study) + else self.remove_remote_final_zipfile(study) & self.remove_input_zipfile(study) ) diff --git a/antareslauncher/remote_environnement/ssh_connection.py b/antareslauncher/remote_environnement/ssh_connection.py index e74b98b..7335bda 100644 --- a/antareslauncher/remote_environnement/ssh_connection.py +++ b/antareslauncher/remote_environnement/ssh_connection.py @@ -109,9 +109,7 @@ def __str__(self) -> str: # 0 duration total_duration # 0% percent 100% duration = time.time() - self._start_time - eta = int( - duration * (self.total_size - total_transferred) / total_transferred - ) + eta = int(duration * (self.total_size - total_transferred) / total_transferred) return f"{self.msg:<20} ETA: {eta}s [{rate:.0%}]" return f"{self.msg:<20} ETA: ??? [{rate:.0%}]" @@ -154,15 +152,11 @@ def __init__(self, config: dict = None): self.logger.info("Loading ssh connection from config dictionary") self.__init_from_config(config) else: - error = InvalidConfigError( - config, "missing values: 'hostname', 'username', 'password'..." - ) + error = InvalidConfigError(config, "missing values: 'hostname', 'username', 'password'...") self.logger.debug(str(error)) raise error self.initialize_home_dir() - self.logger.info( - f"Connection created with host = {self.host} and username = {self.username}" - ) + self.logger.info(f"Connection created with host = {self.host} and username = {self.username}") def __initialise_public_key(self, key_file_name, key_password): """Initialises self.private_key @@ -174,15 +168,11 @@ def __initialise_public_key(self, key_file_name, key_password): True if a valid key was found, False otherwise """ try: - self.private_key = paramiko.Ed25519Key.from_private_key_file( - filename=key_file_name - ) + self.private_key = paramiko.Ed25519Key.from_private_key_file(filename=key_file_name) return True except paramiko.SSHException: try: - self.private_key = paramiko.RSAKey.from_private_key_file( - filename=key_file_name, password=key_password - ) + self.private_key = paramiko.RSAKey.from_private_key_file(filename=key_file_name, password=key_password) return True except paramiko.SSHException: self.private_key = None @@ -195,9 +185,7 @@ def __init_from_config(self, config: dict): self.password = config.get("password") key_password = config.get("key_password") if key_file := config.get("private_key_file"): - self.__initialise_public_key( - key_file_name=key_file, key_password=key_password - ) + self.__initialise_public_key(key_file_name=key_file, key_password=key_password) elif self.password is None: error = InvalidConfigError(config, "missing 'password'") self.logger.debug(str(error)) @@ -250,27 +238,17 @@ def ssh_client(self) -> paramiko.SSHClient: look_for_keys=False, ) except paramiko.AuthenticationException as e: - self.logger.exception( - f"paramiko.AuthenticationException: {paramiko.AuthenticationException}" - ) - raise ConnectionFailedException( - self.host, self.port, self.username - ) from e + self.logger.exception(f"paramiko.AuthenticationException: {paramiko.AuthenticationException}") + raise ConnectionFailedException(self.host, self.port, self.username) from e except paramiko.SSHException as e: self.logger.exception(f"paramiko.SSHException: {paramiko.SSHException}") - raise ConnectionFailedException( - self.host, self.port, self.username - ) from e + raise ConnectionFailedException(self.host, self.port, self.username) from e except socket.timeout as e: self.logger.exception(f"socket.timeout: {socket.timeout}") - raise ConnectionFailedException( - self.host, self.port, self.username - ) from e + raise ConnectionFailedException(self.host, self.port, self.username) from e except socket.error as e: self.logger.exception(f"socket.error: {socket.error}") - raise ConnectionFailedException( - self.host, self.port, self.username - ) from e + raise ConnectionFailedException(self.host, self.port, self.username) from e yield client finally: @@ -394,9 +372,7 @@ def download_files( The paths of the downloaded files on the local filesystem. """ try: - return self._download_files( - src_dir, dst_dir, (pattern,) + patterns, remove=remove - ) + return self._download_files(src_dir, dst_dir, (pattern,) + patterns, remove=remove) except TimeoutError as exc: self.logger.error(f"Timeout: {exc}", exc_info=True) return [] @@ -432,17 +408,13 @@ def _download_files( The paths of the downloaded files on the local filesystem. """ with self.ssh_client() as client: - with contextlib.closing( - client.open_sftp() - ) as sftp: # type: paramiko.sftp_client.SFTPClient + with contextlib.closing(client.open_sftp()) as sftp: # type: paramiko.sftp_client.SFTPClient # Get list of files to download remote_attrs = sftp.listdir_attr(str(src_dir)) remote_files = [file_attr.filename for file_attr in remote_attrs] total_size = sum((file_attr.st_size or 0) for file_attr in remote_attrs) files_to_download = [ - f - for f in remote_files - if any(fnmatch.fnmatch(f, pattern) for pattern in patterns) + f for f in remote_files if any(fnmatch.fnmatch(f, pattern) for pattern in patterns) ] # Monitor the download progression monitor = DownloadMonitor(total_size, logger=self.logger) diff --git a/antareslauncher/use_cases/check_remote_queue/slurm_queue_show.py b/antareslauncher/use_cases/check_remote_queue/slurm_queue_show.py index c9563db..f27c0b2 100644 --- a/antareslauncher/use_cases/check_remote_queue/slurm_queue_show.py +++ b/antareslauncher/use_cases/check_remote_queue/slurm_queue_show.py @@ -1,9 +1,7 @@ from dataclasses import dataclass from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm @dataclass diff --git a/antareslauncher/use_cases/create_list/study_list_composer.py b/antareslauncher/use_cases/create_list/study_list_composer.py index 4c25487..6220226 100644 --- a/antareslauncher/use_cases/create_list/study_list_composer.py +++ b/antareslauncher/use_cases/create_list/study_list_composer.py @@ -68,9 +68,7 @@ def __init__( self.antares_version = parameters.antares_version self._new_study_added = False self.DEFAULT_JOB_LOG_DIR_PATH = str(Path(self.log_dir) / "JOB_LOGS") - self.ANTARES_VERSIONS_ON_REMOTE_SERVER = [ - int(v) for v in parameters.antares_versions_on_remote_server - ] + self.ANTARES_VERSIONS_ON_REMOTE_SERVER = [int(v) for v in parameters.antares_versions_on_remote_server] def get_list_of_studies(self): """Retrieve the list of studies from the repo @@ -125,12 +123,8 @@ def update_study_database(self): f"{__name__}.{__class__.__name__}", ) - def _update_database_with_new_study( - self, antares_version, directory_path, xpansion_mode: str - ): - buffer_study = self._create_study( - directory_path, antares_version, xpansion_mode - ) + def _update_database_with_new_study(self, antares_version, directory_path, xpansion_mode: str): + buffer_study = self._create_study(directory_path, antares_version, xpansion_mode) self._update_database_with_study(buffer_study) def _update_database_with_directory(self, directory_path: Path): @@ -151,21 +145,15 @@ def _update_database_with_directory(self, directory_path: Path): __name__ + "." + self.__class__.__name__, ) else: - candidates_file_path = directory_path.joinpath( - "user", "expansion", "candidates.ini" - ) + candidates_file_path = directory_path.joinpath("user", "expansion", "candidates.ini") is_xpansion_study = candidates_file_path.is_file() xpansion_mode = is_xpansion_study and self.xpansion_mode - valid_xpansion_candidate = ( - self.xpansion_mode in ["r", "cpp"] and is_xpansion_study - ) + valid_xpansion_candidate = self.xpansion_mode in ["r", "cpp"] and is_xpansion_study valid_antares_candidate = not self.xpansion_mode if valid_antares_candidate or valid_xpansion_candidate: - self._update_database_with_new_study( - antares_version, directory_path, xpansion_mode - ) + self._update_database_with_new_study(antares_version, directory_path, xpansion_mode) def _update_database_with_study(self, buffer_study): if not self._repo.is_study_inside_database(buffer_study): diff --git a/antareslauncher/use_cases/generate_tree_structure/tree_structure_initializer.py b/antareslauncher/use_cases/generate_tree_structure/tree_structure_initializer.py index 5ed387e..df933a1 100644 --- a/antareslauncher/use_cases/generate_tree_structure/tree_structure_initializer.py +++ b/antareslauncher/use_cases/generate_tree_structure/tree_structure_initializer.py @@ -17,6 +17,4 @@ def init_tree_structure(self): self.file_manager.make_dir(self.studies_in) self.file_manager.make_dir(self.log_dir) self.file_manager.make_dir(self.finished) - self.display.show_message( - "Tree structure initialized", __name__ + "." + __class__.__name__ - ) + self.display.show_message("Tree structure initialized", __name__ + "." + __class__.__name__) diff --git a/antareslauncher/use_cases/kill_job/job_kill_controller.py b/antareslauncher/use_cases/kill_job/job_kill_controller.py index f6e9a83..a6a56bb 100644 --- a/antareslauncher/use_cases/kill_job/job_kill_controller.py +++ b/antareslauncher/use_cases/kill_job/job_kill_controller.py @@ -2,9 +2,7 @@ from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm @dataclass @@ -23,9 +21,7 @@ def kill_job(self, job_id: int): job_id: The ID of the slurm job to be killed """ if self._check_if_job_is_killable(job_id): - self.display.show_message( - f"Killing job {job_id}", __name__ + "." + self.__class__.__name__ - ) + self.display.show_message(f"Killing job {job_id}", __name__ + "." + self.__class__.__name__) self.env.kill_remote_job(job_id) else: self.display.show_message( diff --git a/antareslauncher/use_cases/launch/launch_controller.py b/antareslauncher/use_cases/launch/launch_controller.py index add508f..45b8067 100644 --- a/antareslauncher/use_cases/launch/launch_controller.py +++ b/antareslauncher/use_cases/launch/launch_controller.py @@ -2,9 +2,7 @@ from antareslauncher.data_repo.data_reporter import DataReporter from antareslauncher.display.display_terminal import DisplayTerminal from antareslauncher.file_manager.file_manager import FileManager -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.launch.study_submitter import StudySubmitter from antareslauncher.use_cases.launch.study_zip_cleaner import StudyZipCleaner @@ -38,9 +36,7 @@ def _upload_zipfile(self): def _remove_input_zipfile(self): if self._current_study.zip_is_sent is True: - self._current_study = self._zipfile_cleaner.remove_input_zipfile( - self._current_study - ) + self._current_study = self._zipfile_cleaner.remove_input_zipfile(self._current_study) self.reporter.save_study(self._current_study) def _submit_job(self): diff --git a/antareslauncher/use_cases/launch/study_submitter.py b/antareslauncher/use_cases/launch/study_submitter.py index 0d187bf..b0af500 100644 --- a/antareslauncher/use_cases/launch/study_submitter.py +++ b/antareslauncher/use_cases/launch/study_submitter.py @@ -2,9 +2,7 @@ from pathlib import Path from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO diff --git a/antareslauncher/use_cases/launch/study_zip_uploader.py b/antareslauncher/use_cases/launch/study_zip_uploader.py index 9b019cd..4559b4c 100644 --- a/antareslauncher/use_cases/launch/study_zip_uploader.py +++ b/antareslauncher/use_cases/launch/study_zip_uploader.py @@ -2,9 +2,7 @@ from pathlib import Path from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO diff --git a/antareslauncher/use_cases/launch/study_zipper.py b/antareslauncher/use_cases/launch/study_zipper.py index 69b1212..d62188a 100644 --- a/antareslauncher/use_cases/launch/study_zipper.py +++ b/antareslauncher/use_cases/launch/study_zipper.py @@ -24,9 +24,7 @@ def zip(self, study) -> StudyDTO: def _do_zip(self): zipfile_path = f"{self._current_study.path}-{getpass.getuser()}.zip" - success = self.file_manager.zip_dir_excluding_subdir( - self._current_study.path, zipfile_path, None - ) + success = self.file_manager.zip_dir_excluding_subdir(self._current_study.path, zipfile_path, None) if success is True: self._current_study.zipfile_path = zipfile_path self._display_success_message() diff --git a/antareslauncher/use_cases/retrieve/clean_remote_server.py b/antareslauncher/use_cases/retrieve/clean_remote_server.py index 22f5d46..05391b4 100644 --- a/antareslauncher/use_cases/retrieve/clean_remote_server.py +++ b/antareslauncher/use_cases/retrieve/clean_remote_server.py @@ -1,7 +1,5 @@ from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO LOG_NAME = f"{__name__}.RemoteServerCleaner" diff --git a/antareslauncher/use_cases/retrieve/download_final_zip.py b/antareslauncher/use_cases/retrieve/download_final_zip.py index 86f814a..6179573 100644 --- a/antareslauncher/use_cases/retrieve/download_final_zip.py +++ b/antareslauncher/use_cases/retrieve/download_final_zip.py @@ -1,9 +1,7 @@ from pathlib import Path from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO LOG_NAME = f"{__name__}.FinalZipDownloader" @@ -30,11 +28,7 @@ def download(self, study: StudyDTO): The updated data transfer object, with its `local_final_zipfile_path` attribute set if the download was successful. """ - if ( - study.finished - and not study.with_error - and not study.local_final_zipfile_path - ): + if study.finished and not study.with_error and not study.local_final_zipfile_path: self._display.show_message( f'"{study.name}": downloading final ZIP...', LOG_NAME, diff --git a/antareslauncher/use_cases/retrieve/final_zip_extractor.py b/antareslauncher/use_cases/retrieve/final_zip_extractor.py index 1a4fd26..e5810ef 100644 --- a/antareslauncher/use_cases/retrieve/final_zip_extractor.py +++ b/antareslauncher/use_cases/retrieve/final_zip_extractor.py @@ -19,12 +19,7 @@ def extract_final_zip(self, study: StudyDTO) -> None: Args: study: The current study """ - if ( - study.finished - and not study.with_error - and study.local_final_zipfile_path - and not study.final_zip_extracted - ): + if study.finished and not study.with_error and study.local_final_zipfile_path and not study.final_zip_extracted: zip_path = Path(study.local_final_zipfile_path) target_dir = zip_path.with_suffix("") try: diff --git a/antareslauncher/use_cases/retrieve/log_downloader.py b/antareslauncher/use_cases/retrieve/log_downloader.py index 065ad6b..8339145 100644 --- a/antareslauncher/use_cases/retrieve/log_downloader.py +++ b/antareslauncher/use_cases/retrieve/log_downloader.py @@ -1,9 +1,7 @@ from pathlib import Path from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO LOG_NAME = f"{__name__}.LogDownloader" diff --git a/antareslauncher/use_cases/retrieve/retrieve_controller.py b/antareslauncher/use_cases/retrieve/retrieve_controller.py index b936b91..a6fab0d 100644 --- a/antareslauncher/use_cases/retrieve/retrieve_controller.py +++ b/antareslauncher/use_cases/retrieve/retrieve_controller.py @@ -1,10 +1,7 @@ from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.data_repo.data_reporter import DataReporter from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.file_manager.file_manager import FileManager -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.use_cases.retrieve.clean_remote_server import RemoteServerCleaner from antareslauncher.use_cases.retrieve.download_final_zip import FinalZipDownloader from antareslauncher.use_cases.retrieve.final_zip_extractor import FinalZipExtractor diff --git a/antareslauncher/use_cases/retrieve/state_updater.py b/antareslauncher/use_cases/retrieve/state_updater.py index f6e495c..63d9b6c 100644 --- a/antareslauncher/use_cases/retrieve/state_updater.py +++ b/antareslauncher/use_cases/retrieve/state_updater.py @@ -1,9 +1,7 @@ import typing as t from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO LOG_NAME = f"{__name__}.RetrieveController" diff --git a/antareslauncher/use_cases/wait_loop_controller/wait_controller.py b/antareslauncher/use_cases/wait_loop_controller/wait_controller.py index 9f57386..c540ddb 100644 --- a/antareslauncher/use_cases/wait_loop_controller/wait_controller.py +++ b/antareslauncher/use_cases/wait_loop_controller/wait_controller.py @@ -38,8 +38,6 @@ def _wait_loop(self, seconds_to_wait: int) -> None: mins, secs = divmod(seconds_to_wait, 60) formatted_time = "{:02d}:{:02d}".format(mins, secs) - self.display.show_message( - text_4_countdown + formatted_time, __name__, end="\r" - ) + self.display.show_message(text_4_countdown + formatted_time, __name__, end="\r") time.sleep(seconds_between_messages) seconds_to_wait -= seconds_between_messages diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e56b84b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[tool.black] +target-version = ["py38"] +line-length = 120 +exclude = "(data/*|docs/*|remote_scripts_templates/*|target/*)" + +[tool.isort] +profile = "black" +line_length = 120 +src_paths = ["antareslauncher", "tests"] +skip_gitignore = true +extend_skip_glob = [ + "data/*", + "doc/*", + "remote_scripts_templates/*", + "target/*", +] diff --git a/tests/integration/test_integration_check_queue_controller.py b/tests/integration/test_integration_check_queue_controller.py index a41beb2..17baf93 100644 --- a/tests/integration/test_integration_check_queue_controller.py +++ b/tests/integration/test_integration_check_queue_controller.py @@ -3,15 +3,9 @@ import pytest from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) -from antareslauncher.remote_environnement.slurm_script_features import ( - SlurmScriptFeatures, -) -from antareslauncher.use_cases.check_remote_queue.check_queue_controller import ( - CheckQueueController, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm +from antareslauncher.remote_environnement.slurm_script_features import SlurmScriptFeatures +from antareslauncher.use_cases.check_remote_queue.check_queue_controller import CheckQueueController from antareslauncher.use_cases.check_remote_queue.slurm_queue_show import SlurmQueueShow from antareslauncher.use_cases.retrieve.state_updater import StateUpdater @@ -34,9 +28,7 @@ def setup_method(self): slurm_queue_show = SlurmQueueShow(env_mock, display_mock) state_updater = StateUpdater(env_mock, display_mock) repo = mock.MagicMock(spec=DataRepoTinydb) - self.check_queue_controller = CheckQueueController( - slurm_queue_show, state_updater, repo - ) + self.check_queue_controller = CheckQueueController(slurm_queue_show, state_updater, repo) @pytest.mark.integration_test def test_check_queue_controller_check_queue_calls_connection_execute_command( diff --git a/tests/integration/test_integration_job_kill_controller.py b/tests/integration/test_integration_job_kill_controller.py index f36039f..861d26a 100644 --- a/tests/integration/test_integration_job_kill_controller.py +++ b/tests/integration/test_integration_job_kill_controller.py @@ -2,12 +2,8 @@ import pytest -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) -from antareslauncher.remote_environnement.slurm_script_features import ( - SlurmScriptFeatures, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm +from antareslauncher.remote_environnement.slurm_script_features import SlurmScriptFeatures from antareslauncher.use_cases.kill_job.job_kill_controller import JobKillController @@ -28,12 +24,8 @@ def test_job_kill_controller_kill_job_calls_connection_execute_command( ): # given job_id = 42 - self.job_kill_controller.env.connection.execute_command = mock.Mock( - return_value=("", "") - ) + self.job_kill_controller.env.connection.execute_command = mock.Mock(return_value=("", "")) # when self.job_kill_controller.kill_job(job_id) # then - self.job_kill_controller.env.connection.execute_command.assert_called_once_with( - f"scancel {job_id}" - ) + self.job_kill_controller.env.connection.execute_command.assert_called_once_with(f"scancel {job_id}") diff --git a/tests/integration/test_integration_launch_controller.py b/tests/integration/test_integration_launch_controller.py index 40a4fe4..25bee7f 100644 --- a/tests/integration/test_integration_launch_controller.py +++ b/tests/integration/test_integration_launch_controller.py @@ -6,12 +6,8 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) -from antareslauncher.remote_environnement.slurm_script_features import ( - SlurmScriptFeatures, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm +from antareslauncher.remote_environnement.slurm_script_features import SlurmScriptFeatures from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.launch.launch_controller import LaunchController @@ -89,20 +85,20 @@ def test_execute_command__called_with_the_correct_parameters( ) home_dir = "Submitted" - remote_base_path = ( - f"{home_dir}/REMOTE_{getpass.getuser()}_{socket.gethostname()}" - ) + remote_base_path = f"{home_dir}/REMOTE_{getpass.getuser()}_{socket.gethostname()}" zipfile_name = Path(study1.zipfile_path).name job_type = "ANTARES" post_processing = False other_options = "" bash_options = ( + # fmt: off f" {zipfile_name}" f" {study1.antares_version}" f" {job_type}" f" {post_processing}" f" '{other_options}'" + # fmt: on ) command = ( f"cd {remote_base_path} && " diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 410c237..3465ece 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -3,12 +3,10 @@ from unittest import mock import pytest + from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.use_cases.create_list.study_list_composer import ( - StudyListComposer, - StudyListComposerParameters, -) +from antareslauncher.use_cases.create_list.study_list_composer import StudyListComposer, StudyListComposerParameters from tests.unit.assets import ASSETS_DIR diff --git a/tests/unit/launcher/test_launch_controller.py b/tests/unit/launcher/test_launch_controller.py index a4b10f0..5aa4ec6 100644 --- a/tests/unit/launcher/test_launch_controller.py +++ b/tests/unit/launcher/test_launch_controller.py @@ -12,9 +12,7 @@ from antareslauncher.data_repo.data_reporter import DataReporter from antareslauncher.display.display_terminal import DisplayTerminal from antareslauncher.file_manager.file_manager import FileManager -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.launch import launch_controller from antareslauncher.use_cases.launch.launch_controller import StudyLauncher @@ -97,20 +95,14 @@ def test_with_one_study_the_compressor_is_called_once(self): file_manager = mock.Mock(spec_set=FileManager) file_manager.zip_dir_excluding_subdir = mock.Mock() - my_launcher = launch_controller.LaunchController( - self.data_repo, remote_env_mock, file_manager, self.display - ) + my_launcher = launch_controller.LaunchController(self.data_repo, remote_env_mock, file_manager, self.display) my_launcher.launch_all_studies() zipfile_path = f"{my_study.path}-{getpass.getuser()}.zip" - file_manager.zip_dir_excluding_subdir.assert_called_once_with( - my_study.path, zipfile_path, None - ) + file_manager.zip_dir_excluding_subdir.assert_called_once_with(my_study.path, zipfile_path, None) @pytest.mark.unit_test - def test_given_one_study_then_repo_is_called_to_save_the_study_with_updated_zip_is_sent( - self, my_launch_controller - ): + def test_given_one_study_then_repo_is_called_to_save_the_study_with_updated_zip_is_sent(self, my_launch_controller): # given my_launcher, expected_study = my_launch_controller # when @@ -142,9 +134,7 @@ def test_given_one_study_when_launcher_is_called_then_study_is_saved_with_job_id assert first_argument.job_id == 42 @pytest.mark.unit_test - def test_given_one_study_when_submit_fails_then_exception_is_raised( - self, my_launch_controller - ): + def test_given_one_study_when_submit_fails_then_exception_is_raised(self, my_launch_controller): # given my_launcher, expected_study = my_launch_controller # when @@ -152,20 +142,14 @@ def test_given_one_study_when_submit_fails_then_exception_is_raised( my_launcher.env.submit_job = mock.Mock(return_value=None) my_launcher.repo.save_study = mock.Mock() # then - with pytest.raises( - antareslauncher.use_cases.launch.study_submitter.FailedSubmissionException - ): + with pytest.raises(antareslauncher.use_cases.launch.study_submitter.FailedSubmissionException): my_launcher.launch_all_studies() @pytest.mark.unit_test - def test_given_one_study_when_zip_fails_then_return_none( - self, my_launch_controller - ): + def test_given_one_study_when_zip_fails_then_return_none(self, my_launch_controller): # given my_launcher, expected_study = my_launch_controller - my_launcher.file_manager.zip_dir_excluding_subdir = mock.Mock( - return_value=False - ) + my_launcher.file_manager.zip_dir_excluding_subdir = mock.Mock(return_value=False) # when my_launcher.launch_all_studies() # then diff --git a/tests/unit/launcher/test_submitter.py b/tests/unit/launcher/test_submitter.py index 72caa03..9ca8991 100644 --- a/tests/unit/launcher/test_submitter.py +++ b/tests/unit/launcher/test_submitter.py @@ -5,9 +5,7 @@ import antareslauncher.use_cases from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.launch.study_submitter import StudySubmitter @@ -26,9 +24,7 @@ def test_submit_study_shows_message_if_submit_succeeds(self): new_study = self.study_submitter.submit_job(study) expected_message = f'"hello": was submitted' - self.display_mock.show_message.assert_called_once_with( - expected_message, mock.ANY - ) + self.display_mock.show_message.assert_called_once_with(expected_message, mock.ANY) assert new_study.job_id == 42 @pytest.mark.unit_test @@ -38,15 +34,11 @@ def test_submit_study_shows_error_if_submit_fails_and_exception_is_raised( self.remote_env.submit_job = mock.Mock(return_value=None) study = StudyDTO(path="hello") - with pytest.raises( - antareslauncher.use_cases.launch.study_submitter.FailedSubmissionException - ): + with pytest.raises(antareslauncher.use_cases.launch.study_submitter.FailedSubmissionException): self.study_submitter.submit_job(study) expected_error_message = f'"hello": was not submitted' - self.display_mock.show_error.assert_called_once_with( - expected_error_message, mock.ANY - ) + self.display_mock.show_error.assert_called_once_with(expected_error_message, mock.ANY) @pytest.mark.unit_test def test_remote_env_not_called_if_study_has_already_a_jobid(self): diff --git a/tests/unit/launcher/test_zip_uploader.py b/tests/unit/launcher/test_zip_uploader.py index 128a79f..64079cd 100644 --- a/tests/unit/launcher/test_zip_uploader.py +++ b/tests/unit/launcher/test_zip_uploader.py @@ -5,14 +5,9 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO -from antareslauncher.use_cases.launch.study_zip_uploader import ( - FailedUploadException, - StudyZipfileUploader, -) +from antareslauncher.use_cases.launch.study_zip_uploader import FailedUploadException, StudyZipfileUploader class TestZipfileUploader: @@ -48,12 +43,8 @@ def test_upload_study_shows_error_if_upload_fails_and_exception_is_raised( expected_welcome_message = f'"hello": uploading study ...' expected_error_message = f'"hello": was not uploaded' - self.display_mock.show_message.assert_called_once_with( - expected_welcome_message, mock.ANY - ) - self.display_mock.show_error.assert_called_once_with( - expected_error_message, mock.ANY - ) + self.display_mock.show_message.assert_called_once_with(expected_welcome_message, mock.ANY) + self.display_mock.show_error.assert_called_once_with(expected_error_message, mock.ANY) @pytest.mark.unit_test def test_remote_env_not_called_if_upload_was_done(self): diff --git a/tests/unit/launcher/test_zipper.py b/tests/unit/launcher/test_zipper.py index 6c6c5e6..a25e22f 100644 --- a/tests/unit/launcher/test_zipper.py +++ b/tests/unit/launcher/test_zipper.py @@ -23,9 +23,7 @@ def test_zip_study_show_message_if_zip_succeeds(self): self.study_zipper.zip(study) expected_message = '"hello": was zipped' - self.display_mock.show_message.assert_called_once_with( - expected_message, mock.ANY - ) + self.display_mock.show_message.assert_called_once_with(expected_message, mock.ANY) @pytest.mark.unit_test def test_zip_study_show_error_if_zip_fails(self): @@ -61,7 +59,5 @@ def test_file_manager_is_called_if_zip_doesnt_exist(self): new_study = self.study_zipper.zip(study) expected_zipfile_path = f"{study.path}-{getpass.getuser()}.zip" - self.file_manager.zip_dir_excluding_subdir.assert_called_once_with( - study_path, expected_zipfile_path, None - ) + self.file_manager.zip_dir_excluding_subdir.assert_called_once_with(study_path, expected_zipfile_path, None) assert new_study.zipfile_path == expected_zipfile_path diff --git a/tests/unit/retriever/conftest.py b/tests/unit/retriever/conftest.py index 77f3ae9..b0d3465 100644 --- a/tests/unit/retriever/conftest.py +++ b/tests/unit/retriever/conftest.py @@ -22,20 +22,14 @@ def pending_study_fixture(tmp_path: Path) -> StudyDTO: @pytest.fixture(name="started_study") def started_study_fixture(pending_study: StudyDTO) -> StudyDTO: - return dataclasses.replace( - pending_study, started=True, finished=False, with_error=False - ) + return dataclasses.replace(pending_study, started=True, finished=False, with_error=False) @pytest.fixture(name="finished_study") def finished_study_fixture(pending_study: StudyDTO) -> StudyDTO: - return dataclasses.replace( - pending_study, started=True, finished=True, with_error=False - ) + return dataclasses.replace(pending_study, started=True, finished=True, with_error=False) @pytest.fixture(name="with_error_study") def with_error_study_fixture(pending_study: StudyDTO) -> StudyDTO: - return dataclasses.replace( - pending_study, started=True, finished=True, with_error=True - ) + return dataclasses.replace(pending_study, started=True, finished=True, with_error=True) diff --git a/tests/unit/retriever/test_download_final_zip.py b/tests/unit/retriever/test_download_final_zip.py index 87b0526..b2a0cde 100644 --- a/tests/unit/retriever/test_download_final_zip.py +++ b/tests/unit/retriever/test_download_final_zip.py @@ -5,9 +5,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.download_final_zip import FinalZipDownloader @@ -64,9 +62,7 @@ def test_download__with_error_study(self, with_error_study: StudyDTO) -> None: display.show_error.assert_not_called() @pytest.mark.unit_test - def test_download__finished_study__download_ok( - self, finished_study: StudyDTO - ) -> None: + def test_download__finished_study__download_ok(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.download_final_zip = download_final_zip display = mock.Mock(spec=DisplayTerminal) @@ -85,9 +81,7 @@ def test_download__finished_study__download_ok( assert len(zip_files) == 1 @pytest.mark.unit_test - def test_download__finished_study__reentrancy( - self, finished_study: StudyDTO - ) -> None: + def test_download__finished_study__reentrancy(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.download_final_zip = download_final_zip display = mock.Mock(spec=DisplayTerminal) @@ -111,9 +105,7 @@ def test_download__finished_study__reentrancy( assert zip_files1 == zip_files2 @pytest.mark.unit_test - def test_download__finished_study__download_nothing( - self, finished_study: StudyDTO - ) -> None: + def test_download__finished_study__download_nothing(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.download_final_zip = lambda _: [] display = mock.Mock(spec=DisplayTerminal) @@ -132,9 +124,7 @@ def test_download__finished_study__download_nothing( assert not zip_files @pytest.mark.unit_test - def test_download__finished_study__download_error( - self, finished_study: StudyDTO - ) -> None: + def test_download__finished_study__download_error(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.download_final_zip.side_effect = Exception("Connection error") display = mock.Mock(spec=DisplayTerminal) diff --git a/tests/unit/retriever/test_final_zip_extractor.py b/tests/unit/retriever/test_final_zip_extractor.py index 0dc9381..f2c333c 100644 --- a/tests/unit/retriever/test_final_zip_extractor.py +++ b/tests/unit/retriever/test_final_zip_extractor.py @@ -55,9 +55,7 @@ def test_extract_final_zip__started_study(self, started_study: StudyDTO) -> None assert not started_study.final_zip_extracted @pytest.mark.unit_test - def test_extract_final_zip__finished_study__no_output( - self, finished_study: StudyDTO - ) -> None: + def test_extract_final_zip__finished_study__no_output(self, finished_study: StudyDTO) -> None: display = mock.Mock(spec=DisplayTerminal) # Initialize and execute the ZIP extraction @@ -70,9 +68,7 @@ def test_extract_final_zip__finished_study__no_output( assert not finished_study.final_zip_extracted @pytest.mark.unit_test - def test_extract_final_zip__finished_study__nominal( - self, finished_study: StudyDTO - ) -> None: + def test_extract_final_zip__finished_study__nominal(self, finished_study: StudyDTO) -> None: display = mock.Mock(spec=DisplayTerminal) display.generate_progress_bar = lambda names, *args, **kwargs: names @@ -94,9 +90,7 @@ def test_extract_final_zip__finished_study__nominal( assert result_dir.joinpath("simulation.log").is_file() @pytest.mark.unit_test - def test_extract_final_zip__finished_study__reentrancy( - self, finished_study: StudyDTO - ) -> None: + def test_extract_final_zip__finished_study__reentrancy(self, finished_study: StudyDTO) -> None: display = mock.Mock(spec=DisplayTerminal) display.generate_progress_bar = lambda names, *args, **kwargs: names @@ -115,16 +109,12 @@ def test_extract_final_zip__finished_study__reentrancy( assert study_state1 == study_state2 @pytest.mark.unit_test - def test_extract_final_zip__finished_study__missing( - self, finished_study: StudyDTO - ) -> None: + def test_extract_final_zip__finished_study__missing(self, finished_study: StudyDTO) -> None: display = mock.Mock(spec=DisplayTerminal) display.generate_progress_bar = lambda names, *args, **kwargs: names # Prepare a missing final ZIP - finished_study.local_final_zipfile_path = create_final_zip( - finished_study, scenario="missing" - ) + finished_study.local_final_zipfile_path = create_final_zip(finished_study, scenario="missing") # Initialize and execute the ZIP extraction extractor = FinalZipExtractor(display=display) @@ -141,16 +131,12 @@ def test_extract_final_zip__finished_study__missing( assert not result_dir.joinpath("simulation.log").exists() @pytest.mark.unit_test - def test_extract_final_zip__finished_study__corrupted( - self, finished_study: StudyDTO - ) -> None: + def test_extract_final_zip__finished_study__corrupted(self, finished_study: StudyDTO) -> None: display = mock.Mock(spec=DisplayTerminal) display.generate_progress_bar = lambda names, *args, **kwargs: names # Prepare a corrupted final ZIP - finished_study.local_final_zipfile_path = create_final_zip( - finished_study, scenario="corrupted" - ) + finished_study.local_final_zipfile_path = create_final_zip(finished_study, scenario="corrupted") # Initialize and execute the ZIP extraction extractor = FinalZipExtractor(display=display) diff --git a/tests/unit/retriever/test_log_downloader.py b/tests/unit/retriever/test_log_downloader.py index 3818748..b6335b8 100644 --- a/tests/unit/retriever/test_log_downloader.py +++ b/tests/unit/retriever/test_log_downloader.py @@ -5,9 +5,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.log_downloader import LogDownloader @@ -81,9 +79,7 @@ def test_run__started_study__reentrancy(self, started_study: StudyDTO) -> None: assert log_files1 == log_files2 @pytest.mark.unit_test - def test_run__started_study__download_nothing( - self, started_study: StudyDTO - ) -> None: + def test_run__started_study__download_nothing(self, started_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.download_logs = lambda _: [] display = mock.Mock(spec=DisplayTerminal) diff --git a/tests/unit/retriever/test_retrieve_controller.py b/tests/unit/retriever/test_retrieve_controller.py index 9cef2cc..c28372f 100644 --- a/tests/unit/retriever/test_retrieve_controller.py +++ b/tests/unit/retriever/test_retrieve_controller.py @@ -5,9 +5,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.retrieve_controller import RetrieveController from antareslauncher.use_cases.retrieve.state_updater import StateUpdater @@ -21,23 +19,17 @@ def setup_method(self): self.state_updater_mock = StateUpdater(self.env, self.display) @pytest.mark.unit_test - def test_given_one_study_when_retrieve_all_studies_call_then_study_retriever_is_called_once( - self, started_study - ): + def test_given_one_study_when_retrieve_all_studies_call_then_study_retriever_is_called_once(self, started_study): # given list_of_studies = [started_study] self.data_repo.get_list_of_studies = mock.Mock(return_value=list_of_studies) - my_retriever = RetrieveController( - self.data_repo, self.env, self.display, self.state_updater_mock - ) + my_retriever = RetrieveController(self.data_repo, self.env, self.display, self.state_updater_mock) my_retriever.study_retriever.retrieve = mock.Mock() self.display.show_message = mock.Mock() # when my_retriever.retrieve_all_studies() # then - self.display.show_message.assert_called_once_with( - "Retrieving all studies...", mock.ANY - ) + self.display.show_message.assert_called_once_with("Retrieving all studies...", mock.ANY) my_retriever.study_retriever.retrieve.assert_called_once_with(started_study) @pytest.mark.unit_test @@ -48,9 +40,7 @@ def test_given_a_list_of_done_studies_when_all_studies_done_called_then_return_t study = StudyDTO("path") study.done = True study_list = [deepcopy(study), deepcopy(study)] - my_retriever = RetrieveController( - self.data_repo, self.env, self.display, self.state_updater_mock - ) + my_retriever = RetrieveController(self.data_repo, self.env, self.display, self.state_updater_mock) my_retriever.repo.get_list_of_studies = mock.Mock(return_value=study_list) # when output = my_retriever.all_studies_done @@ -66,9 +56,7 @@ def test_given_a_list_of_done_studies_when_retrieve_all_studies_called_then_mess study.done = True study_list = [deepcopy(study), deepcopy(study)] display_mock = mock.Mock(spec=DisplayTerminal) - my_retriever = RetrieveController( - self.data_repo, self.env, display_mock, self.state_updater_mock - ) + my_retriever = RetrieveController(self.data_repo, self.env, display_mock, self.state_updater_mock) my_retriever.repo.get_list_of_studies = mock.Mock(return_value=study_list) display_mock.show_message = mock.Mock() # when diff --git a/tests/unit/retriever/test_server_cleaner.py b/tests/unit/retriever/test_server_cleaner.py index 901ec0c..9cc51a7 100644 --- a/tests/unit/retriever/test_server_cleaner.py +++ b/tests/unit/retriever/test_server_cleaner.py @@ -3,9 +3,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.clean_remote_server import RemoteServerCleaner @@ -73,9 +71,7 @@ def test_clean__finished_study__reentrancy(self, finished_study: StudyDTO) -> No assert finished_study.remote_server_is_clean @pytest.mark.unit_test - def test_clean__finished_study__cleaning_failed( - self, finished_study: StudyDTO - ) -> None: + def test_clean__finished_study__cleaning_failed(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.clean_remote_server.return_value = False display = mock.Mock(spec=DisplayTerminal) @@ -95,9 +91,7 @@ def test_clean__finished_study__cleaning_failed( assert finished_study.remote_server_is_clean @pytest.mark.unit_test - def test_clean__finished_study__cleaning_raise( - self, finished_study: StudyDTO - ) -> None: + def test_clean__finished_study__cleaning_raise(self, finished_study: StudyDTO) -> None: env = mock.Mock(spec=RemoteEnvironmentWithSlurm) env.clean_remote_server.side_effect = Exception("cleaning error") display = mock.Mock(spec=DisplayTerminal) diff --git a/tests/unit/retriever/test_state_updater.py b/tests/unit/retriever/test_state_updater.py index 9ab1639..4a9d636 100644 --- a/tests/unit/retriever/test_state_updater.py +++ b/tests/unit/retriever/test_state_updater.py @@ -4,9 +4,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.state_updater import StateUpdater @@ -21,13 +19,9 @@ (None, None, True, "Ended with error"), ], ) -def test_given_a_submitted_study_then_study_flags_are_updated( - started_flag, finished_flag, with_error_flag, status -): +def test_given_a_submitted_study_then_study_flags_are_updated(started_flag, finished_flag, with_error_flag, status): env = mock.Mock(spec=RemoteEnvironmentWithSlurm) - env.get_job_state_flags = mock.Mock( - return_value=(started_flag, finished_flag, with_error_flag) - ) + env.get_job_state_flags = mock.Mock(return_value=(started_flag, finished_flag, with_error_flag)) display = mock.Mock(spec=DisplayTerminal) my_study = StudyDTO(path="study_path", job_id=42) diff --git a/tests/unit/retriever/test_study_retriever.py b/tests/unit/retriever/test_study_retriever.py index 58b8e1e..4d9c506 100644 --- a/tests/unit/retriever/test_study_retriever.py +++ b/tests/unit/retriever/test_study_retriever.py @@ -5,9 +5,7 @@ from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.data_repo.data_reporter import DataReporter from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.remote_environnement.remote_environment_with_slurm import ( - RemoteEnvironmentWithSlurm, -) +from antareslauncher.remote_environnement.remote_environment_with_slurm import RemoteEnvironmentWithSlurm from antareslauncher.study_dto import StudyDTO from antareslauncher.use_cases.retrieve.clean_remote_server import RemoteServerCleaner from antareslauncher.use_cases.retrieve.download_final_zip import FinalZipDownloader @@ -91,25 +89,19 @@ def final_zip_downloader_download(study_: StudyDTO): study_.local_final_zipfile_path = "final-zipfile.zip" return study_ - self.final_zip_downloader.download = mock.Mock( - side_effect=final_zip_downloader_download - ) + self.final_zip_downloader.download = mock.Mock(side_effect=final_zip_downloader_download) def remote_server_cleaner_clean(study_: StudyDTO): study_.remote_server_is_clean = True return study_ - self.remote_server_cleaner.clean = mock.Mock( - side_effect=remote_server_cleaner_clean - ) + self.remote_server_cleaner.clean = mock.Mock(side_effect=remote_server_cleaner_clean) def zip_extractor_extract_final_zip(study_: StudyDTO): study_.final_zip_extracted = True return study_ - self.zip_extractor.extract_final_zip = mock.Mock( - side_effect=zip_extractor_extract_final_zip - ) + self.zip_extractor.extract_final_zip = mock.Mock(side_effect=zip_extractor_extract_final_zip) self.reporter.save_study = mock.Mock(return_value=True) self.study_retriever.retrieve(study) diff --git a/tests/unit/test_antares_launcher.py b/tests/unit/test_antares_launcher.py index 4803f52..4b9c875 100644 --- a/tests/unit/test_antares_launcher.py +++ b/tests/unit/test_antares_launcher.py @@ -4,9 +4,7 @@ import pytest from antareslauncher.antares_launcher import AntaresLauncher -from antareslauncher.use_cases.wait_loop_controller.wait_controller import ( - WaitController, -) +from antareslauncher.use_cases.wait_loop_controller.wait_controller import WaitController class TestAntaresLauncher: @@ -33,9 +31,7 @@ def test_given_job_id_to_kill_when_run_then_job_kill_controller_kills_job_with_g # when antares_launcher.run() # then - antares_launcher.job_kill_controller.kill_job.assert_called_once_with( - job_id_to_kill - ) + antares_launcher.job_kill_controller.kill_job.assert_called_once_with(job_id_to_kill) @pytest.mark.unit_test def test_given_true_check_queue_bool_when_run_then_check_queue_controller_checks_queue( diff --git a/tests/unit/test_check_queue_controller.py b/tests/unit/test_check_queue_controller.py index 8f378c2..5195228 100644 --- a/tests/unit/test_check_queue_controller.py +++ b/tests/unit/test_check_queue_controller.py @@ -4,9 +4,7 @@ from antareslauncher.data_repo.data_repo_tinydb import DataRepoTinydb from antareslauncher.study_dto import StudyDTO -from antareslauncher.use_cases.check_remote_queue.check_queue_controller import ( - CheckQueueController, -) +from antareslauncher.use_cases.check_remote_queue.check_queue_controller import CheckQueueController from antareslauncher.use_cases.check_remote_queue.slurm_queue_show import SlurmQueueShow from antareslauncher.use_cases.retrieve.state_updater import StateUpdater @@ -28,9 +26,7 @@ def setup_method(self): def test_check_queue_controller_calls_slurm_queue_show_once(self): # given self.slurm_queue_show.run = mock.Mock() - self.repo_mock.get_list_of_studies = ( - mock.MagicMock() - ) # mock.Mock(return_value=[]) + self.repo_mock.get_list_of_studies = mock.MagicMock() # mock.Mock(return_value=[]) # when self.check_queue_controller.check_queue() # then diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index f33997a..7c1ebbe 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -20,11 +20,7 @@ get_user_config_dir, parse_config, ) -from antareslauncher.exceptions import ( - ConfigFileNotFoundError, - InvalidConfigValueError, - UnknownFileSuffixError, -) +from antareslauncher.exceptions import ConfigFileNotFoundError, InvalidConfigValueError, UnknownFileSuffixError class TestParseConfig: @@ -223,9 +219,7 @@ def test_load_config__nominal(self, tmp_path, ssh_config_path): assert config.db_primary_key == data["db_primary_key"] assert config.ssh_config_file_is_required == data["ssh_config_file_is_required"] assert config.slurm_script_path == slurm_script_path - assert ( - config.remote_solver_versions == data["antares_versions_on_remote_server"] - ) + assert config.remote_solver_versions == data["antares_versions_on_remote_server"] def test_save_config__nominal(self, tmp_path, ssh_config): config_path = tmp_path.joinpath("configuration.yaml") @@ -262,13 +256,9 @@ def test_save_config__nominal(self, tmp_path, ssh_config): assert actual["default_n_cpu"] == config.default_n_cpu assert actual["default_wait_time"] == config.default_wait_time assert actual["db_primary_key"] == config.db_primary_key - assert ( - actual["ssh_config_file_is_required"] == config.ssh_config_file_is_required - ) + assert actual["ssh_config_file_is_required"] == config.ssh_config_file_is_required assert actual["slurm_script_path"] == slurm_script_path.as_posix() - assert ( - actual["antares_versions_on_remote_server"] == config.remote_solver_versions - ) + assert actual["antares_versions_on_remote_server"] == config.remote_solver_versions assert "ssh_config" not in actual @pytest.mark.parametrize( @@ -364,12 +354,8 @@ def test_get_config_path__from_env__not_found(self, monkeypatch, tmp_path): with pytest.raises(ConfigFileNotFoundError): get_config_path() - @pytest.mark.parametrize( - "config_name", [None, CONFIGURATION_YAML, "my_config.yaml"] - ) - def test_get_config_path__from_user_config_dir( - self, monkeypatch, tmp_path, config_name - ): + @pytest.mark.parametrize("config_name", [None, CONFIGURATION_YAML, "my_config.yaml"]) + def test_get_config_path__from_user_config_dir(self, monkeypatch, tmp_path, config_name): config_path = tmp_path.joinpath(config_name or CONFIGURATION_YAML) config_path.touch() monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) @@ -380,17 +366,11 @@ def test_get_config_path__from_user_config_dir( assert actual == config_path @pytest.mark.parametrize("relpath", ["", "data"]) - @pytest.mark.parametrize( - "config_name", [None, CONFIGURATION_YAML, "my_config.yaml"] - ) - def test_get_config_path__from_curr_dir( - self, monkeypatch, tmp_path, relpath, config_name - ): + @pytest.mark.parametrize("config_name", [None, CONFIGURATION_YAML, "my_config.yaml"]) + def test_get_config_path__from_curr_dir(self, monkeypatch, tmp_path, relpath, config_name): data_dir = tmp_path.joinpath(relpath) data_dir.mkdir(exist_ok=True) - config_path: pathlib.Path = tmp_path.joinpath( - data_dir, config_name or CONFIGURATION_YAML - ) + config_path: pathlib.Path = tmp_path.joinpath(data_dir, config_name or CONFIGURATION_YAML) config_path.touch() monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) monkeypatch.chdir(tmp_path) @@ -399,9 +379,7 @@ def test_get_config_path__from_curr_dir( assert actual == config_path.relative_to(tmp_path) @pytest.mark.parametrize("relpath", ["", "data"]) - def test_get_config_path__from_curr_dir__not_found( - self, monkeypatch, tmp_path, relpath - ): + def test_get_config_path__from_curr_dir__not_found(self, monkeypatch, tmp_path, relpath): data_dir = tmp_path.joinpath(relpath) data_dir.mkdir(exist_ok=True) monkeypatch.delenv("ANTARES_LAUNCHER_CONFIG_PATH", raising=False) diff --git a/tests/unit/test_main_option_parser.py b/tests/unit/test_main_option_parser.py index 82218aa..128cfac 100644 --- a/tests/unit/test_main_option_parser.py +++ b/tests/unit/test_main_option_parser.py @@ -2,11 +2,7 @@ import pytest -from antareslauncher.main_option_parser import ( - MainOptionParser, - ParserParameters, - look_for_default_ssh_conf_file, -) +from antareslauncher.main_option_parser import MainOptionParser, ParserParameters, look_for_default_ssh_conf_file class TestMainOptionParser: @@ -34,9 +30,7 @@ def setup_method(self): "n_cpu": 42, "job_id_to_kill": None, "post_processing": False, - "json_ssh_config": look_for_default_ssh_conf_file( - self.main_options_parameters - ), + "json_ssh_config": look_for_default_ssh_conf_file(self.main_options_parameters), } @pytest.fixture(scope="function") diff --git a/tests/unit/test_parameters_reader.py b/tests/unit/test_parameters_reader.py index 1ba30d6..58acf2b 100644 --- a/tests/unit/test_parameters_reader.py +++ b/tests/unit/test_parameters_reader.py @@ -82,9 +82,7 @@ def test_get_main_parameters_raises_exception_with_empty_file(self, tmp_path): ParametersReader(empty_json, empty_yaml).get_main_parameters() @pytest.mark.unit_test - def test_get_option_parameters_raises_exception_if_params_are_missing( - self, tmp_path - ): + def test_get_option_parameters_raises_exception_if_params_are_missing(self, tmp_path): empty_json = tmp_path / "dummy.json" config_yaml = tmp_path / "empty.yaml" config_yaml.write_text( @@ -119,23 +117,16 @@ def test_get_option_parameters_initializes_parameters_correctly(self, tmp_path): empty_json.write_text("{}") config_yaml = tmp_path / "empty.yaml" config_yaml.write_text(self.yaml_compulsory_content) - options_parameters = ParametersReader( - empty_json, config_yaml - ).get_parser_parameters() + options_parameters = ParametersReader(empty_json, config_yaml).get_parser_parameters() assert options_parameters.log_dir == self.LOG_DIR assert options_parameters.studies_in_dir == self.STUDIES_IN_DIR assert options_parameters.finished_dir == self.FINISHED_DIR assert options_parameters.default_time_limit == self.DEFAULT_TIME_LIMIT assert options_parameters.default_n_cpu == self.DEFAULT_N_CPU assert options_parameters.default_wait_time == self.DEFAULT_WAIT_TIME - assert ( - options_parameters.ssh_config_file_is_required - == self.SSH_CONFIG_FILE_IS_REQUIRED - ) + assert options_parameters.ssh_config_file_is_required == self.SSH_CONFIG_FILE_IS_REQUIRED alternate1 = Path.cwd() / self.DEFAULT_SSH_CONFIGFILE_NAME - alternate2 = ( - Path.home() / "antares_launcher_settings" / self.DEFAULT_SSH_CONFIGFILE_NAME - ) + alternate2 = Path.home() / "antares_launcher_settings" / self.DEFAULT_SSH_CONFIGFILE_NAME assert options_parameters.ssh_configfile_path_alternate1 == alternate1 assert options_parameters.ssh_configfile_path_alternate2 == alternate2 @@ -146,23 +137,15 @@ def test_get_main_parameters_initializes_parameters_correctly(self, tmp_path): config_yaml.write_text(self.yaml_compulsory_content) empty_json = tmp_path / "dummy.json" empty_json.write_text("{}") - main_parameters = ParametersReader( - empty_json, config_yaml - ).get_main_parameters() + main_parameters = ParametersReader(empty_json, config_yaml).get_main_parameters() assert main_parameters.json_dir == Path(self.JSON_DIR) assert main_parameters.slurm_script_path == self.SLURM_SCRIPT_PATH - assert ( - main_parameters.default_json_db_name - == f"{getpass.getuser()}_antares_launcher_db.json" - ) + assert main_parameters.default_json_db_name == f"{getpass.getuser()}_antares_launcher_db.json" assert main_parameters.partition == self.PARTITION assert main_parameters.quality_of_service == self.QUALITY_OF_SERVICE assert main_parameters.db_primary_key == self.DB_PRIMARY_KEY assert not main_parameters.default_ssh_dict - assert ( - main_parameters.antares_versions_on_remote_server - == self.ANTARES_SUPPORTED_VERSIONS - ) + assert main_parameters.antares_versions_on_remote_server == self.ANTARES_SUPPORTED_VERSIONS @pytest.mark.unit_test def test_get_main_parameters_initializes_default_ssh_dict_correctly(self, tmp_path): @@ -172,7 +155,5 @@ def test_get_main_parameters_initializes_default_ssh_dict_correctly(self, tmp_pa with open(ssh_json, "w") as file: json.dump(self.json_dict, file) - main_parameters = ParametersReader( - json_ssh_conf=ssh_json, yaml_filepath=config_yaml - ).get_main_parameters() + main_parameters = ParametersReader(json_ssh_conf=ssh_json, yaml_filepath=config_yaml).get_main_parameters() assert main_parameters.default_ssh_dict == self.json_dict diff --git a/tests/unit/test_remote_environment_with_slurm.py b/tests/unit/test_remote_environment_with_slurm.py index 9b52aba..1da3d07 100644 --- a/tests/unit/test_remote_environment_with_slurm.py +++ b/tests/unit/test_remote_environment_with_slurm.py @@ -17,10 +17,7 @@ RemoteEnvironmentWithSlurm, SubmitJobError, ) -from antareslauncher.remote_environnement.slurm_script_features import ( - ScriptParametersDTO, - SlurmScriptFeatures, -) +from antareslauncher.remote_environnement.slurm_script_features import ScriptParametersDTO, SlurmScriptFeatures from antareslauncher.study_dto import Modes, StudyDTO @@ -77,9 +74,7 @@ def test_initialise_remote_path_calls_connection_make_dir_with_correct_arguments ): # given remote_home_dir = "remote_home_dir" - remote_base_dir = ( - f"{remote_home_dir}/REMOTE_{getpass.getuser()}_{socket.gethostname()}" - ) + remote_base_dir = f"{remote_home_dir}/REMOTE_{getpass.getuser()}_{socket.gethostname()}" connection = mock.Mock(home_dir="path/to/home") connection.home_dir = remote_home_dir connection.make_dir = mock.Mock(return_value=True) @@ -151,9 +146,7 @@ def test_when_constructor_is_called_and_connection_check_file_not_empty_is_false RemoteEnvironmentWithSlurm(connection, slurm_script_features) @pytest.mark.unit_test - def test_get_queue_info_calls_connection_execute_command_with_correct_argument( - self, remote_env - ): + def test_get_queue_info_calls_connection_execute_command_with_correct_argument(self, remote_env): # given username = "username" host = "host" @@ -169,9 +162,7 @@ def test_get_queue_info_calls_connection_execute_command_with_correct_argument( remote_env.connection.execute_command.assert_called_with(command) @pytest.mark.unit_test - def test_when_connection_exec_command_has_an_error_then_get_queue_info_returns_the_error_string( - self, remote_env - ): + def test_when_connection_exec_command_has_an_error_then_get_queue_info_returns_the_error_string(self, remote_env): # given username = "username" remote_env.connection.username = username @@ -194,9 +185,7 @@ def test_kill_remote_job_execute_scancel_command(self, remote_env): remote_env.connection.execute_command.assert_called_with(command) @pytest.mark.unit_test - def test_when_kill_remote_job_is_called_and_exec_command_returns_error_exception_is_raised( - self, remote_env - ): + def test_when_kill_remote_job_is_called_and_exec_command_returns_error_exception_is_raised(self, remote_env): # when output = None error = "error" @@ -224,15 +213,11 @@ def test_when_submit_job_is_called_then_execute_command_is_called_with_specific_ post_processing=study.post_processing, other_options="", ) - command = remote_env.slurm_script_features.compose_launch_command( - remote_env.remote_base_path, script_params - ) + command = remote_env.slurm_script_features.compose_launch_command(remote_env.remote_base_path, script_params) remote_env.connection.execute_command.assert_called_once_with(command) @pytest.mark.unit_test - def test_when_submit_job_is_called_and_receives_submitted_420_returns_job_id_420( - self, remote_env, study - ): + def test_when_submit_job_is_called_and_receives_submitted_420_returns_job_id_420(self, remote_env, study): # when output = "Submitted 420" error = None @@ -241,9 +226,7 @@ def test_when_submit_job_is_called_and_receives_submitted_420_returns_job_id_420 assert remote_env.submit_job(study) == 420 @pytest.mark.unit_test - def test_when_submit_job_is_called_and_receives_error_then_exception_is_raised( - self, remote_env, study - ): + def test_when_submit_job_is_called_and_receives_error_then_exception_is_raised(self, remote_env, study): # when output = "" error = "error" @@ -392,9 +375,7 @@ def execute_command_mock(cmd: str): ("FAILED", (True, True, True)), ], ) - def test_get_job_state_flags__sacct_nominal_case( - self, remote_env, study, state, expected - ): + def test_get_job_state_flags__sacct_nominal_case(self, remote_env, study, state, expected): """ Check that the "get_job_state_flags" method is correctly returning the status flags ("started", "finished", and "with_error") @@ -552,9 +533,7 @@ def test_given_a_study_with_input_zipfile_removed_when_remove_input_zipfile_then assert output is True @pytest.mark.unit_test - def test_given_a_study_when_remove_input_zipfile_then_connection_remove_file_is_called( - self, remote_env, study - ): + def test_given_a_study_when_remove_input_zipfile_then_connection_remove_file_is_called(self, remote_env, study): # given study.input_zipfile_removed = False study.zipfile_path = "zipfile_path" @@ -586,9 +565,7 @@ def test_given_a_study_when_remove_remote_final_zipfile_then_connection_remove_f # given study.input_zipfile_removed = False study.zipfile_path = "zipfile_path" - command = ( - f"{remote_env.remote_base_path}/{Path(study.local_final_zipfile_path).name}" - ) + command = f"{remote_env.remote_base_path}/{Path(study.local_final_zipfile_path).name}" remote_env.connection.execute_command = mock.Mock(return_value=("", "")) # when remote_env.remove_remote_final_zipfile(study) @@ -608,9 +585,7 @@ def test_given_a_study_with_clean_remote_server_when_clean_remote_server_called_ assert output is False @pytest.mark.unit_test - def test_given_a_study_when_clean_remote_server_called_then_remove_zip_methods_are_called( - self, remote_env, study - ): + def test_given_a_study_when_clean_remote_server_called_then_remove_zip_methods_are_called(self, remote_env, study): # given study.remote_server_is_clean = False remote_env.remove_remote_final_zipfile = mock.Mock(return_value=False) @@ -622,9 +597,7 @@ def test_given_a_study_when_clean_remote_server_called_then_remove_zip_methods_a remote_env.remove_input_zipfile.assert_called_once_with(study) @pytest.mark.unit_test - def test_given_a_study_when_clean_remote_server_called_then_return_correct_result( - self, remote_env, study - ): + def test_given_a_study_when_clean_remote_server_called_then_return_correct_result(self, remote_env, study): # given study.remote_server_is_clean = False remote_env.remove_remote_final_zipfile = mock.Mock(return_value=False) @@ -665,9 +638,7 @@ def test_given_time_limit_lower_than_min_duration_when_convert_time_is_called_re # given time_lim_sec = 42 # when - output = RemoteEnvironmentWithSlurm.convert_time_limit_from_seconds_to_minutes( - time_lim_sec - ) + output = RemoteEnvironmentWithSlurm.convert_time_limit_from_seconds_to_minutes(time_lim_sec) # then assert output == 1 diff --git a/tests/unit/test_study_list_composer.py b/tests/unit/test_study_list_composer.py index 1b76ea1..902e402 100644 --- a/tests/unit/test_study_list_composer.py +++ b/tests/unit/test_study_list_composer.py @@ -2,10 +2,7 @@ import pytest -from antareslauncher.use_cases.create_list.study_list_composer import ( - StudyListComposer, - get_solver_version, -) +from antareslauncher.use_cases.create_list.study_list_composer import StudyListComposer, get_solver_version CONFIG_NOMINAL_VERSION = """\ [antares] diff --git a/tests/unit/test_wait_controller.py b/tests/unit/test_wait_controller.py index 555dc7a..b0a479e 100644 --- a/tests/unit/test_wait_controller.py +++ b/tests/unit/test_wait_controller.py @@ -3,9 +3,7 @@ import pytest from antareslauncher.display.display_terminal import DisplayTerminal -from antareslauncher.use_cases.wait_loop_controller.wait_controller import ( - WaitController, -) +from antareslauncher.use_cases.wait_loop_controller.wait_controller import WaitController class TestWaitController: