From 961929dd5b4966aa3fe782acef5be2b82efe1196 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 4 Aug 2023 14:00:14 +0200 Subject: [PATCH 01/13] test(export): add unit tests for `RawStudyService.export_study_flat` method --- .../rawstudy/test_raw_study_service.py | 97 ++++++++++--------- 1 file changed, 50 insertions(+), 47 deletions(-) diff --git a/tests/study/storage/rawstudy/test_raw_study_service.py b/tests/study/storage/rawstudy/test_raw_study_service.py index 98a680cf70..e306122f46 100644 --- a/tests/study/storage/rawstudy/test_raw_study_service.py +++ b/tests/study/storage/rawstudy/test_raw_study_service.py @@ -1,11 +1,10 @@ import datetime +import typing as t import zipfile from pathlib import Path -from typing import List, Optional import numpy as np import pytest -from sqlalchemy import create_engine # type: ignore from antarest.core.model import PublicMode from antarest.core.utils.fastapi_sqlalchemy import db @@ -24,6 +23,28 @@ from tests.helpers import with_db_context +def _collect_files_by_type(raw_study_path: Path) -> t.Tuple[t.Set[str], t.Set[str], t.Set[str]]: + """ + Collects files based on their types for comparison. + A tuple containing sets of study files, matrices, and outputs. + """ + study_files = set() + matrices = set() + outputs = set() + + for study_file in raw_study_path.rglob("*.*"): + relpath = study_file.relative_to(raw_study_path).as_posix() + + if study_file.suffixes == [".txt", ".link"]: + matrices.add(relpath.replace(".link", "")) + elif relpath.startswith("output/"): + outputs.add(relpath) + else: + study_files.add(relpath) + + return study_files, matrices, outputs + + class TestRawStudyService: # noinspection SpellCheckingInspection """ @@ -41,12 +62,12 @@ class TestRawStudyService: @pytest.mark.parametrize( "output_filter", [ - # fmt:off + # "20230802-1425eco" is a folder, + # "20230802-1628eco" is a ZIP file. pytest.param(None, id="no_filter"), pytest.param(["20230802-1425eco"], id="folder"), pytest.param(["20230802-1628eco"], id="zipped"), pytest.param(["20230802-1425eco", "20230802-1628eco"], id="both"), - # fmt:on ], ) @pytest.mark.parametrize( @@ -67,10 +88,10 @@ def test_export_study_flat( study_storage_service: StudyStorageService, # pytest parameters outputs: bool, - output_filter: Optional[List[str]], + output_filter: t.Optional[t.List[str]], denormalize: bool, ) -> None: - ## Prepare database objects + # Prepare database objects # noinspection PyArgumentList user = User(id=0, name="admin") db.session.add(user) @@ -100,7 +121,7 @@ def test_export_study_flat( db.session.add(raw_study) db.session.commit() - ## Prepare the RAW Study + # Prepare the RAW Study raw_study_service.create(raw_study) file_study = raw_study_service.get_raw(raw_study) @@ -110,10 +131,8 @@ def test_export_study_flat( patch_service=patch_service, ) - create_area_fr = CreateArea( - command_context=command_context, - area_name="fr", - ) + # For instance, we define an area "FR" with a short-term storage named "Storage1": + create_area_fr = CreateArea(command_context=command_context, area_name="fr") # noinspection SpellCheckingInspection pmax_injection = np.random.rand(8760, 1) @@ -144,39 +163,33 @@ def test_export_study_flat( storage_service=study_storage_service, ) - ## Prepare fake outputs + # Simulate generating results from an Antares Solver simulation. + # The results can be stored either as a sub-folder or as a ZIP file. + # In both cases, they are saved in the "output" directory. + + # Prepare fake simulation outputs my_solver_outputs = ["20230802-1425eco", "20230802-1628eco.zip"] for filename in my_solver_outputs: output_path = raw_study_path / "output" / filename # To simplify the checking, there is only one file in each output: if output_path.suffix.lower() == ".zip": - # Create a fake ZIP file + # Create a fake ZIP file and add a simulation log output_path.parent.mkdir(exist_ok=True, parents=True) with zipfile.ZipFile( output_path, mode="w", compression=zipfile.ZIP_DEFLATED, ) as zf: - zf.writestr("simulation.log", data="Simulation done") + zf.writestr("simulation.log", data="Simulation completed") else: - # Create a directory + # Create a directory and add a simulation log output_path.mkdir(exist_ok=True, parents=True) - (output_path / "simulation.log").write_text("Simulation done") - - ## Collect all files by types to prepare the comparison - src_study_files = set() - src_matrices = set() - src_outputs = set() - for study_file in raw_study_path.rglob("*.*"): - relpath = study_file.relative_to(raw_study_path).as_posix() - if study_file.suffixes == [".txt", ".link"]: - src_matrices.add(relpath.replace(".link", "")) - elif relpath.startswith("output/"): - src_outputs.add(relpath) - else: - src_study_files.add(relpath) + (output_path / "simulation.log").write_text("Simulation completed") + + # Collect all files by types to prepare the comparison + src_study_files, src_matrices, src_outputs = _collect_files_by_type(raw_study_path) - ## Run the export + # Run the export target_path = tmp_path / raw_study_path.with_suffix(".exported").name raw_study_service.export_study_flat( raw_study, @@ -186,22 +199,12 @@ def test_export_study_flat( denormalize=denormalize, ) - ## Collect the resulting files - res_study_files = set() - res_matrices = set() - res_outputs = set() - for study_file in target_path.rglob("*.*"): - relpath = study_file.relative_to(target_path).as_posix() - if study_file.suffixes == [".txt", ".link"]: - res_matrices.add(relpath.replace(".link", "")) - elif relpath.startswith("output/"): - res_outputs.add(relpath) - else: - res_study_files.add(relpath) + # Collect the resulting files + res_study_files, res_matrices, res_outputs = _collect_files_by_type(target_path) - ## Check the matrice - # If de-normalization is enabled, the previous loop won't find the matrices - # because the matrix extensions are ".txt" instead of ".txt.link". + # Check the matrice: + # If de-normalization is enabled, the `_collect_files_by_type` function won't + # find the matrices because the matrix extensions are ".txt" instead of ".txt.link". # Therefore, it is necessary to move the corresponding ".txt" files # from `res_study_files` to `res_matrices`. if denormalize: @@ -210,7 +213,7 @@ def test_export_study_flat( res_study_files -= res_matrices assert res_matrices == src_matrices - ## Check the outputs + # Check the outputs if outputs: # If `outputs` is True the filtering can occurs if output_filter is None: @@ -224,5 +227,5 @@ def test_export_study_flat( # whatever the value of the `output_list_filter` is assert not res_outputs - ## Check the study files + # Check the study files assert res_study_files == src_study_files From a31feecebc2ec9a5d778d96089a9be1d08cc0466 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 7 Aug 2023 22:26:47 +0200 Subject: [PATCH 02/13] test(export): add unit tests for `VariantStudyService.generate_task` method --- .../variantstudy/test_variant_study_service.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/study/storage/variantstudy/test_variant_study_service.py b/tests/study/storage/variantstudy/test_variant_study_service.py index 25317a9589..7c4bee3244 100644 --- a/tests/study/storage/variantstudy/test_variant_study_service.py +++ b/tests/study/storage/variantstudy/test_variant_study_service.py @@ -5,7 +5,6 @@ import numpy as np import pytest -from sqlalchemy import create_engine # type: ignore from antarest.core.model import PublicMode from antarest.core.requests import RequestParameters @@ -118,7 +117,7 @@ def test_generate_task( denormalize: bool, from_scratch: bool, ) -> None: - ## Prepare database objects + # Prepare database objects # noinspection PyArgumentList user = User(id=0, name="admin") db.session.add(user) @@ -129,7 +128,7 @@ def test_generate_task( db.session.add(group) db.session.commit() - ## First create a raw study (root of the variant) + # First create a raw study (root of the variant) raw_study_path = tmp_path / "My RAW Study" # noinspection PyArgumentList raw_study = RawStudy( @@ -149,7 +148,7 @@ def test_generate_task( db.session.add(raw_study) db.session.commit() - ## Prepare the RAW Study + # Prepare the RAW Study raw_study_service.create(raw_study) variant_study = variant_study_service.create_variant_study( @@ -161,7 +160,7 @@ def test_generate_task( ), ) - ## Prepare the RAW Study + # Prepare the RAW Study file_study = variant_study_service.get_raw(variant_study) command_context = CommandContext( @@ -175,7 +174,7 @@ def test_generate_task( area_name="fr", ) - ## Prepare the Variant Study Data + # Prepare the Variant Study Data # noinspection SpellCheckingInspection pmax_injection = np.random.rand(8760, 1) inflows = np.random.uniform(0, 1000, size=(8760, 1)) @@ -205,7 +204,7 @@ def test_generate_task( storage_service=study_storage_service, ) - ## Run the "generate" task + # Run the "generate" task actual_uui = variant_study_service.generate_task( variant_study, denormalize=denormalize, @@ -217,7 +216,7 @@ def test_generate_task( flags=re.IGNORECASE, ) - ## Collect the resulting files + # Collect the resulting files workspaces = variant_study_service.config.storage.workspaces internal_studies_dir: Path = workspaces["default"].path snapshot_dir = internal_studies_dir.joinpath(variant_study.snapshot.id, "snapshot") From 73ebae06f12637306044a4974e08708353c7f1ad Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Wed, 26 Jul 2023 14:45:15 +0200 Subject: [PATCH 03/13] refactor(export): request for Study export function (#1669) Co-authored-by: LAIDI Takfarinas (Externe) Co-authored-by: TLAIDI --- antarest/study/common/studystorage.py | 24 +------ antarest/study/service.py | 39 +++++++++-- .../study/storage/abstract_storage_service.py | 69 ++++++++++++++++++- .../storage/rawstudy/raw_study_service.py | 26 ------- antarest/study/storage/utils.py | 47 ------------- .../variantstudy/variant_study_service.py | 56 ++++++--------- tests/storage/business/test_export.py | 10 ++- tests/storage/integration/test_exporter.py | 4 -- .../variantstudy/model/test_variant_model.py | 22 +++--- 9 files changed, 141 insertions(+), 156 deletions(-) diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index 162266ebb2..f795b46d6c 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -230,27 +230,6 @@ def export_output(self, metadata: T, output_id: str, target: Path) -> None: """ raise NotImplementedError() - @abstractmethod - def export_study_flat( - self, - metadata: T, - dst_path: Path, - outputs: bool = True, - output_list_filter: Optional[List[str]] = None, - denormalize: bool = True, - ) -> None: - """ - Export study to destination - - Args: - metadata: study. - dst_path: destination path. - outputs: list of outputs to keep. - output_list_filter: list of outputs to keep (None indicate all outputs). - denormalize: denormalize the study (replace matrix links by real matrices). - """ - raise NotImplementedError() - @abstractmethod def get_synthesis(self, metadata: T, params: Optional[RequestParameters] = None) -> FileStudyTreeConfigDTO: """ @@ -274,3 +253,6 @@ def archive_study_output(self, study: T, output_id: str) -> bool: @abstractmethod def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) -> bool: raise NotImplementedError() + + def unarchive(self, study: T) -> None: + raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index 332fc384da..1523434822 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -2,6 +2,7 @@ import io import json import logging +import shutil import os from datetime import datetime, timedelta from http import HTTPStatus @@ -106,6 +107,7 @@ remove_from_cache, study_matcher, ) +from antarest.study.storage.abstract_storage_service import export_study_flat from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_comments import UpdateComments @@ -918,7 +920,17 @@ def export_study( def export_task(notifier: TaskUpdateNotifier) -> TaskResult: try: target_study = self.get_study(uuid) - self.storage_service.get_storage(target_study).export_study(target_study, export_path, outputs) + storage = self.storage_service.get_storage(target_study) + if isinstance(target_study, RawStudy): + if target_study.archived: + storage.unarchive(target_study) + try: + storage.export_study(target_study, export_path, outputs) + finally: + if target_study.archived: + shutil.rmtree(target_study.path) + else: + storage.export_study(target_study, export_path, outputs) self.file_transfer_manager.set_ready(export_id) return TaskResult(success=True, message=f"Study {uuid} successfully exported") except Exception as e: @@ -1020,9 +1032,28 @@ def export_study_flat( study = self.get_study(uuid) assert_permission(params.user, study, StudyPermissionType.READ) self._assert_study_unarchived(study) - - return self.storage_service.get_storage(study).export_study_flat( - study, dest, len(output_list or []) > 0, output_list + path_study = Path(study.path) + if isinstance(study, RawStudy): + if study.archived: + self.storage_service.get_storage(study).unarchive(study) + try: + return export_study_flat( + path_study=path_study, + dest=dest, + outputs=len(output_list or []) > 0, + output_list_filter=output_list, + ) + finally: + if study.archived: + shutil.rmtree(study.path) + snapshot_path = path_study / "snapshot" + output_src_path = path_study / "output" + export_study_flat( + path_study=snapshot_path, + dest=dest, + outputs=len(output_list or []) > 0, + output_list_filter=output_list, + output_src_path=output_src_path, ) def delete_study(self, uuid: str, children: bool, params: RequestParameters) -> None: diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index f056839ac6..bc466580cd 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -5,6 +5,10 @@ from pathlib import Path from typing import IO, List, Optional, Union from uuid import uuid4 +import time +from zipfile import ZipFile +import os + from antarest.core.config import Config from antarest.core.exceptions import BadOutputError, StudyOutputNotFoundError @@ -32,6 +36,60 @@ logger = logging.getLogger(__name__) +def export_study_flat( + path_study: Path, + dest: Path, + outputs: bool = True, + output_list_filter: Optional[List[str]] = None, + output_src_path: Optional[Path] = None, +) -> None: + """ + Export study to destination + + Args: + path_study: Study source path + dest: Destination path. + outputs: List of outputs to keep. + output_list_filter: List of outputs to keep (None indicate all outputs). + output_src_path: Denormalize the study (replace matrix links by real matrices). + + """ + start_time = time.time() + output_src_path = output_src_path or path_study / "output" + output_dest_path = dest / "output" + ignore_patterns = ( + lambda directory, contents: ["output"] + if str(directory) == str(path_study) + else [] + ) + + shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns) + if outputs and output_src_path.is_dir(): + if output_dest_path.exists(): + shutil.rmtree(output_dest_path) + if output_list_filter is not None: + os.mkdir(output_dest_path) + for output in output_list_filter: + zip_path = output_src_path / f"{output}.zip" + if zip_path.exists(): + with ZipFile(zip_path) as zf: + zf.extractall(output_dest_path / output) + else: + shutil.copytree( + src=output_src_path / output, + dst=output_dest_path / output, + ) + else: + shutil.copytree( + src=output_src_path, + dst=output_dest_path, + ) + + stop_time = time.time() + duration = "{:.3f}".format(stop_time - start_time) + logger.info(f"Study {path_study} exported (flat mode) in {duration}s") + + class AbstractStorageService(IStudyStorageService[T], ABC): def __init__( self, @@ -230,7 +288,16 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path: with tempfile.TemporaryDirectory(dir=self.config.storage.tmp_dir) as tmpdir: logger.info(f"Exporting study {metadata.id} to temporary path {tmpdir}") tmp_study_path = Path(tmpdir) / "tmp_copy" - self.export_study_flat(metadata, tmp_study_path, outputs) + if not isinstance(metadata, RawStudy): + snapshot_path = path_study / "snapshot" + output_src_path = path_study / "output" + export_study_flat( + path_study=snapshot_path, + dest=tmp_study_path, + outputs=outputs, + output_src_path=output_src_path, + ) + export_study_flat(path_study, tmp_study_path, outputs) stopwatch = StopWatch() zip_dir(tmp_study_path, target) stopwatch.log_elapsed(lambda x: logger.info(f"Study {path_study} exported (zipped mode) in {x}s")) diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index c6c73e63b5..c9ce8fd0d1 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -23,7 +23,6 @@ from antarest.study.storage.rawstudy.model.filesystem.lazy_node import LazyNode from antarest.study.storage.utils import ( create_new_empty_study, - export_study_flat, fix_study_root, get_default_workspace_path, is_managed, @@ -323,31 +322,6 @@ def import_study(self, metadata: RawStudy, stream: IO[bytes]) -> Study: metadata.path = str(path_study) return metadata - def export_study_flat( - self, - metadata: RawStudy, - dst_path: Path, - outputs: bool = True, - output_list_filter: Optional[List[str]] = None, - denormalize: bool = True, - ) -> None: - path_study = Path(metadata.path) - - if metadata.archived: - self.unarchive(metadata) - try: - export_study_flat( - path_study, - dst_path, - self.study_factory, - outputs, - output_list_filter, - denormalize, - ) - finally: - if metadata.archived: - shutil.rmtree(metadata.path) - def check_errors( self, metadata: RawStudy, diff --git a/antarest/study/storage/utils.py b/antarest/study/storage/utils.py index 6197180f50..d0906e128c 100644 --- a/antarest/study/storage/utils.py +++ b/antarest/study/storage/utils.py @@ -326,50 +326,3 @@ def get_start_date( first_week_size=first_week_size, level=level, ) - - -def export_study_flat( - path_study: Path, - dest: Path, - study_factory: StudyFactory, - outputs: bool = True, - output_list_filter: Optional[List[str]] = None, - denormalize: bool = True, - output_src_path: Optional[Path] = None, -) -> None: - start_time = time.time() - - output_src_path = output_src_path or path_study / "output" - output_dest_path = dest / "output" - ignore_patterns = lambda directory, contents: ["output"] if str(directory) == str(path_study) else [] - - shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns) - - if outputs and output_src_path.exists(): - if output_list_filter is None: - # Retrieve all directories or ZIP files without duplicates - output_list_filter = list( - {f.with_suffix("").name for f in output_src_path.iterdir() if f.is_dir() or f.suffix == ".zip"} - ) - # Copy each folder or uncompress each ZIP file to the destination dir. - shutil.rmtree(output_dest_path, ignore_errors=True) - output_dest_path.mkdir() - for output in output_list_filter: - zip_path = output_src_path / f"{output}.zip" - if zip_path.exists(): - with ZipFile(zip_path) as zf: - zf.extractall(output_dest_path / output) - else: - shutil.copytree( - src=output_src_path / output, - dst=output_dest_path / output, - ) - - stop_time = time.time() - duration = "{:.3f}".format(stop_time - start_time) - logger.info(f"Study {path_study} exported (flat mode) in {duration}s") - study = study_factory.create_from_fs(dest, "", use_cache=False) - if denormalize: - study.tree.denormalize() - duration = "{:.3f}".format(time.time() - stop_time) - logger.info(f"Study {path_study} denormalized in {duration}s") diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index f956564656..61ff09f025 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -34,14 +34,13 @@ from antarest.core.utils.utils import assert_this, suppress_exception from antarest.matrixstore.service import MatrixService from antarest.study.model import RawStudy, Study, StudyAdditionalData, StudyMetadataDTO, StudySimResultDTO -from antarest.study.storage.abstract_storage_service import AbstractStorageService +from antarest.study.storage.abstract_storage_service import AbstractStorageService, export_study_flat from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory from antarest.study.storage.rawstudy.raw_study_service import RawStudyService from antarest.study.storage.utils import ( assert_permission, - export_study_flat, get_default_workspace_path, is_managed, remove_from_cache, @@ -719,22 +718,30 @@ def _generate( last_executed_command_index = None if last_executed_command_index is None: - # Copy parent study to destination if isinstance(parent_study, VariantStudy): self._safe_generation(parent_study) - self.export_study_flat( - metadata=parent_study, - dst_path=dst_path, + path_study = Path(parent_study.path) + snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH + output_src_path = path_study / "output" + export_study_flat( + snapshot_path, + dst_path, outputs=False, - denormalize=False, + output_src_path=output_src_path, ) else: - self.raw_study_service.export_study_flat( - metadata=parent_study, - dst_path=dst_path, - outputs=False, - denormalize=False, - ) + path_study = Path(parent_study.path) + if parent_study.archived: + self.raw_study_service.unarchive(parent_study) + try: + export_study_flat( + path_study=path_study, + dest=dst_path, + outputs=False, + ) + finally: + if parent_study.archived: + shutil.rmtree(parent_study.path) command_start_index = last_executed_command_index + 1 if last_executed_command_index is not None else 0 logger.info(f"Generating study snapshot from command index {command_start_index}") @@ -1068,29 +1075,6 @@ def get_study_path(self, metadata: Study) -> Path: """ return Path(metadata.path) / SNAPSHOT_RELATIVE_PATH - def export_study_flat( - self, - metadata: VariantStudy, - dst_path: Path, - outputs: bool = True, - output_list_filter: Optional[List[str]] = None, - denormalize: bool = True, - ) -> None: - self._safe_generation(metadata) - path_study = Path(metadata.path) - - snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH - output_src_path = path_study / "output" - export_study_flat( - snapshot_path, - dst_path, - self.study_factory, - outputs, - output_list_filter, - denormalize, - output_src_path, - ) - def get_synthesis( self, metadata: VariantStudy, diff --git a/tests/storage/business/test_export.py b/tests/storage/business/test_export.py index 759bc99ed3..e45f68dfcd 100644 --- a/tests/storage/business/test_export.py +++ b/tests/storage/business/test_export.py @@ -8,6 +8,7 @@ from antarest.core.config import Config, StorageConfig from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy from antarest.study.storage.rawstudy.raw_study_service import RawStudyService +from antarest.study.storage.abstract_storage_service import export_study_flat @pytest.mark.unit_test @@ -104,17 +105,14 @@ def test_export_flat(tmp_path: Path): study_factory.create_from_fs.return_value = study_tree study = RawStudy(id="id", path=root) + path_study = Path(study.path) - study_service.export_study_flat(study, tmp_path / "copy_with_output", outputs=True) - + export_study_flat(path_study, tmp_path / "copy_with_output", outputs=True) copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") - assert root_hash == copy_with_output_hash - study_service.export_study_flat(study, tmp_path / "copy_without_output", outputs=False) - + export_study_flat(path_study, tmp_path / "copy_without_output", outputs=False) copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") - assert root_without_output_hash == copy_without_output_hash diff --git a/tests/storage/integration/test_exporter.py b/tests/storage/integration/test_exporter.py index 3e4e5666f3..be3d5c5c9f 100644 --- a/tests/storage/integration/test_exporter.py +++ b/tests/storage/integration/test_exporter.py @@ -100,13 +100,11 @@ def test_exporter_file_no_output(tmp_path: Path, sta_mini_zip_path: Path) -> Non @pytest.mark.parametrize("outputs", [True, False, "prout"]) @pytest.mark.parametrize("output_list", [None, [], ["20201014-1427eco"], ["20201014-1430adq-2"]]) -@pytest.mark.parametrize("denormalize", [True, False]) def test_export_flat( tmp_path: Path, sta_mini_zip_path: Path, outputs: bool, output_list: Optional[List[str]], - denormalize: bool, ) -> None: path_studies = tmp_path / "studies" path_studies.mkdir(exist_ok=True) @@ -120,10 +118,8 @@ def test_export_flat( export_study_flat( path_studies / "STA-mini", export_path / "STA-mini-export", - Mock(), outputs, output_list, - denormalize=denormalize, ) export_output_path = export_path / "STA-mini-export" / "output" diff --git a/tests/variantstudy/model/test_variant_model.py b/tests/variantstudy/model/test_variant_model.py index d3a1760077..d1cebd2ce9 100644 --- a/tests/variantstudy/model/test_variant_model.py +++ b/tests/variantstudy/model/test_variant_model.py @@ -1,6 +1,6 @@ import datetime from pathlib import Path -from unittest.mock import ANY, Mock +from unittest.mock import ANY, Mock, patch from sqlalchemy import create_engine @@ -13,7 +13,6 @@ from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware, db from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, StudyAdditionalData from antarest.study.storage.variantstudy.command_factory import CommandFactory -from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.model.model import CommandDTO, GenerationResultInfoDTO from antarest.study.storage.variantstudy.repository import VariantStudyRepository from antarest.study.storage.variantstudy.variant_study_service import SNAPSHOT_RELATIVE_PATH, VariantStudyService @@ -64,6 +63,7 @@ def test_commands_service(tmp_path: Path, command_factory: CommandFactory): id=origin_id, name="my-study", additional_data=StudyAdditionalData(), + path=str(tmp_path), ) repository.save(origin_study) @@ -138,7 +138,8 @@ def test_commands_service(tmp_path: Path, command_factory: CommandFactory): assert study.snapshot.id == study.id -def test_smart_generation(tmp_path: Path, command_factory: CommandFactory) -> None: +@patch("antarest.study.storage.variantstudy.variant_study_service.export_study_flat") +def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandFactory) -> None: engine = create_engine( "sqlite:///:memory:", echo=False, @@ -173,17 +174,16 @@ def test_smart_generation(tmp_path: Path, command_factory: CommandFactory) -> No # noinspection PyUnusedLocal def export_flat( - metadata: VariantStudy, - dst_path: Path, + path_study: Path, + dest: Path, outputs: bool = True, denormalize: bool = True, ) -> None: - dst_path.mkdir(parents=True) - (dst_path / "user").mkdir() - (dst_path / "user" / "some_unmanaged_config").touch() - - service.raw_study_service.export_study_flat.side_effect = export_flat + dest.mkdir(parents=True) + (dest / "user").mkdir() + (dest / "user" / "some_unmanaged_config").touch() + mock_export.side_effect = export_flat with db(): origin_id = "base-study" # noinspection PyArgumentList @@ -194,6 +194,7 @@ def export_flat( workspace=DEFAULT_WORKSPACE_NAME, additional_data=StudyAdditionalData(), updated_at=datetime.datetime(year=2000, month=1, day=1), + path=str(tmp_path), ) repository.save(origin_study) @@ -233,7 +234,6 @@ def export_flat( ], SADMIN, ) - assert unmanaged_user_config_path.exists() unmanaged_user_config_path.write_text("hello") service._generate(variant_id, SADMIN, False) From ed3f6bf3acc00fc5789575c51a6098012579698b Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Thu, 27 Jul 2023 19:33:31 +0200 Subject: [PATCH 04/13] fix(api): add missing denormalized rules in refactor export_study_flat (#1646) --- antarest/study/service.py | 2 ++ antarest/study/storage/abstract_storage_service.py | 12 +++++++++++- .../storage/variantstudy/variant_study_service.py | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index 1523434822..f4bb7304dd 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1039,6 +1039,7 @@ def export_study_flat( try: return export_study_flat( path_study=path_study, + study_factory=study, dest=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, @@ -1050,6 +1051,7 @@ def export_study_flat( output_src_path = path_study / "output" export_study_flat( path_study=snapshot_path, + study_factory=study, dest=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index bc466580cd..be565c6127 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -39,6 +39,7 @@ def export_study_flat( path_study: Path, dest: Path, + study_factory: StudyFactory, outputs: bool = True, output_list_filter: Optional[List[str]] = None, output_src_path: Optional[Path] = None, @@ -49,6 +50,7 @@ def export_study_flat( Args: path_study: Study source path dest: Destination path. + study_factory: StudyFactory, outputs: List of outputs to keep. output_list_filter: List of outputs to keep (None indicate all outputs). output_src_path: Denormalize the study (replace matrix links by real matrices). @@ -89,6 +91,11 @@ def export_study_flat( duration = "{:.3f}".format(stop_time - start_time) logger.info(f"Study {path_study} exported (flat mode) in {duration}s") + study = study_factory.create_from_fs(dest, "", use_cache=False) + study.tree.denormalize() + duration = "{:.3f}".format(time.time() - stop_time) + logger.info(f"Study {path_study} denormalized in {duration}s") + class AbstractStorageService(IStudyStorageService[T], ABC): def __init__( @@ -293,11 +300,14 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path: output_src_path = path_study / "output" export_study_flat( path_study=snapshot_path, + study_factory=self.study_factory, dest=tmp_study_path, outputs=outputs, output_src_path=output_src_path, ) - export_study_flat(path_study, tmp_study_path, outputs) + export_study_flat( + path_study, tmp_study_path, self.study_factory, outputs + ) stopwatch = StopWatch() zip_dir(tmp_study_path, target) stopwatch.log_elapsed(lambda x: logger.info(f"Study {path_study} exported (zipped mode) in {x}s")) diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 61ff09f025..db134d18ad 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -726,6 +726,7 @@ def _generate( export_study_flat( snapshot_path, dst_path, + parent_study, outputs=False, output_src_path=output_src_path, ) @@ -737,6 +738,7 @@ def _generate( export_study_flat( path_study=path_study, dest=dst_path, + study_factory=parent_study, outputs=False, ) finally: From 71cfa5267436b0bdfae42e802b91a6dd74e6a78e Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Fri, 28 Jul 2023 15:13:42 +0200 Subject: [PATCH 05/13] fix(api): fix failed tests (#1646) --- tests/storage/integration/test_exporter.py | 1 + tests/variantstudy/model/test_variant_model.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/storage/integration/test_exporter.py b/tests/storage/integration/test_exporter.py index be3d5c5c9f..2ab14e26c2 100644 --- a/tests/storage/integration/test_exporter.py +++ b/tests/storage/integration/test_exporter.py @@ -118,6 +118,7 @@ def test_export_flat( export_study_flat( path_studies / "STA-mini", export_path / "STA-mini-export", + Mock(), outputs, output_list, ) diff --git a/tests/variantstudy/model/test_variant_model.py b/tests/variantstudy/model/test_variant_model.py index d1cebd2ce9..3b2a66c4f5 100644 --- a/tests/variantstudy/model/test_variant_model.py +++ b/tests/variantstudy/model/test_variant_model.py @@ -13,6 +13,7 @@ from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware, db from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, StudyAdditionalData from antarest.study.storage.variantstudy.command_factory import CommandFactory +from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.model.model import CommandDTO, GenerationResultInfoDTO from antarest.study.storage.variantstudy.repository import VariantStudyRepository from antarest.study.storage.variantstudy.variant_study_service import SNAPSHOT_RELATIVE_PATH, VariantStudyService @@ -28,7 +29,12 @@ ) -def test_commands_service(tmp_path: Path, command_factory: CommandFactory): +@patch( + "antarest.study.storage.variantstudy.variant_study_service.export_study_flat" +) +def test_commands_service( + mock_export, tmp_path: Path, command_factory: CommandFactory +): engine = create_engine( "sqlite:///:memory:", echo=False, @@ -41,6 +47,7 @@ def test_commands_service(tmp_path: Path, command_factory: CommandFactory): custom_engine=engine, session_args={"autocommit": False, "autoflush": False}, ) + repository = VariantStudyRepository(LocalCache()) service = VariantStudyService( raw_study_service=Mock(), @@ -176,6 +183,7 @@ def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandF def export_flat( path_study: Path, dest: Path, + study_factory: VariantStudy, outputs: bool = True, denormalize: bool = True, ) -> None: From aa12b42dbd1ca36d2d5729368f1d4171179e338e Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Fri, 28 Jul 2023 17:59:12 +0200 Subject: [PATCH 06/13] fix(api): fix failed tests (#1646) --- tests/storage/business/test_export.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/storage/business/test_export.py b/tests/storage/business/test_export.py index e45f68dfcd..04f47b6d51 100644 --- a/tests/storage/business/test_export.py +++ b/tests/storage/business/test_export.py @@ -107,11 +107,16 @@ def test_export_flat(tmp_path: Path): study = RawStudy(id="id", path=root) path_study = Path(study.path) - export_study_flat(path_study, tmp_path / "copy_with_output", outputs=True) + export_study_flat(path_study, tmp_path / "copy_with_output", study_factory, outputs=True) copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") assert root_hash == copy_with_output_hash - export_study_flat(path_study, tmp_path / "copy_without_output", outputs=False) + export_study_flat( + path_study, + tmp_path / "copy_without_output", + study_factory, + outputs=False, + ) copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") assert root_without_output_hash == copy_without_output_hash From f65effa2a3eeb064f8abce4ad0af70d87f25502c Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Mon, 31 Jul 2023 10:22:31 +0200 Subject: [PATCH 07/13] fix(api): fix again failed tests (#1646) --- antarest/study/storage/variantstudy/variant_study_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index db134d18ad..765c9b053a 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -726,7 +726,7 @@ def _generate( export_study_flat( snapshot_path, dst_path, - parent_study, + self.study_factory, outputs=False, output_src_path=output_src_path, ) @@ -738,7 +738,7 @@ def _generate( export_study_flat( path_study=path_study, dest=dst_path, - study_factory=parent_study, + study_factory=self.study_factory, outputs=False, ) finally: From 998756f91897b2dc097ee3a43b9cfefcf1cce294 Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Tue, 1 Aug 2023 11:21:46 +0200 Subject: [PATCH 08/13] fix(api): refactor and fix failed tests (#1646) --- antarest/study/common/studystorage.py | 12 +++++ antarest/study/service.py | 14 +++--- .../study/storage/abstract_storage_service.py | 20 ++++---- .../storage/rawstudy/raw_study_service.py | 22 ++++++++- .../variantstudy/variant_study_service.py | 47 ++++++++++++++++--- tests/storage/business/test_export.py | 22 ++------- tests/storage/integration/test_exporter.py | 3 +- .../variantstudy/model/test_variant_model.py | 33 ++++--------- 8 files changed, 105 insertions(+), 68 deletions(-) diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index f795b46d6c..1eca3a0f1d 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -256,3 +256,15 @@ def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) - def unarchive(self, study: T) -> None: raise NotImplementedError() + + # def export_study_flat(self, **kwargs) -> None: + # raise NotImplementedError() + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index f4bb7304dd..756954e51a 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1033,26 +1033,26 @@ def export_study_flat( assert_permission(params.user, study, StudyPermissionType.READ) self._assert_study_unarchived(study) path_study = Path(study.path) + storage = self.storage_service.get_storage(study) if isinstance(study, RawStudy): if study.archived: - self.storage_service.get_storage(study).unarchive(study) + storage.unarchive(study) try: - return export_study_flat( + return storage.export_study_flat( path_study=path_study, - study_factory=study, - dest=dest, + dst_path=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, + output_src_path=None, ) finally: if study.archived: shutil.rmtree(study.path) snapshot_path = path_study / "snapshot" output_src_path = path_study / "output" - export_study_flat( + return storage.export_study_flat( path_study=snapshot_path, - study_factory=study, - dest=dest, + dst_path=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, output_src_path=output_src_path, diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index be565c6127..fb11a5bf50 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -39,7 +39,6 @@ def export_study_flat( path_study: Path, dest: Path, - study_factory: StudyFactory, outputs: bool = True, output_list_filter: Optional[List[str]] = None, output_src_path: Optional[Path] = None, @@ -91,11 +90,6 @@ def export_study_flat( duration = "{:.3f}".format(stop_time - start_time) logger.info(f"Study {path_study} exported (flat mode) in {duration}s") - study = study_factory.create_from_fs(dest, "", use_cache=False) - study.tree.denormalize() - duration = "{:.3f}".format(time.time() - stop_time) - logger.info(f"Study {path_study} denormalized in {duration}s") - class AbstractStorageService(IStudyStorageService[T], ABC): def __init__( @@ -295,19 +289,25 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path: with tempfile.TemporaryDirectory(dir=self.config.storage.tmp_dir) as tmpdir: logger.info(f"Exporting study {metadata.id} to temporary path {tmpdir}") tmp_study_path = Path(tmpdir) / "tmp_copy" - if not isinstance(metadata, RawStudy): + if isinstance(metadata, RawStudy): + export_study_flat( + path_study=path_study, + dest=tmp_study_path, + outputs=outputs, + ) + else: snapshot_path = path_study / "snapshot" output_src_path = path_study / "output" export_study_flat( path_study=snapshot_path, - study_factory=self.study_factory, dest=tmp_study_path, outputs=outputs, output_src_path=output_src_path, ) - export_study_flat( - path_study, tmp_study_path, self.study_factory, outputs + study = self.study_factory.create_from_fs( + tmp_study_path, "", use_cache=False ) + study.tree.denormalize() stopwatch = StopWatch() zip_dir(tmp_study_path, target) stopwatch.log_elapsed(lambda x: logger.info(f"Study {path_study} exported (zipped mode) in {x}s")) diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index c9ce8fd0d1..e8946b4fca 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -16,7 +16,7 @@ from antarest.core.requests import RequestParameters from antarest.core.utils.utils import extract_zip from antarest.study.model import DEFAULT_WORKSPACE_NAME, Patch, RawStudy, Study, StudyAdditionalData -from antarest.study.storage.abstract_storage_service import AbstractStorageService +from antarest.study.storage.abstract_storage_service import AbstractStorageService, export_study_flat from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory @@ -322,6 +322,26 @@ def import_study(self, metadata: RawStudy, stream: IO[bytes]) -> Study: metadata.path = str(path_study) return metadata + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + export_study_flat( + path_study=path_study, + dest=dst_path, + outputs=outputs, + output_list_filter=output_list_filter, + output_src_path=output_src_path, + ) + study = self.study_factory.create_from_fs( + dst_path, "", use_cache=False + ) + study.tree.denormalize() + def check_errors( self, metadata: RawStudy, diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 765c9b053a..5ecc992900 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -719,14 +719,28 @@ def _generate( if last_executed_command_index is None: if isinstance(parent_study, VariantStudy): + # self._safe_generation(parent_study) + # self.export_study_flat( + # metadata=parent_study, + # dst_path=dst_path, + # outputs=False, + # denormalize=False, + # ) + # else: + # self.raw_study_service.export_study_flat( + # metadata=parent_study, + # dst_path=dst_path, + # outputs=False, + # denormalize=False, + # ) + self._safe_generation(parent_study) path_study = Path(parent_study.path) snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH output_src_path = path_study / "output" - export_study_flat( - snapshot_path, - dst_path, - self.study_factory, + self.export_study_flat( + path_study=snapshot_path, + dst_path=dst_path, outputs=False, output_src_path=output_src_path, ) @@ -735,10 +749,9 @@ def _generate( if parent_study.archived: self.raw_study_service.unarchive(parent_study) try: - export_study_flat( + self.raw_study_service.export_study_flat( path_study=path_study, - dest=dst_path, - study_factory=self.study_factory, + dst_path=dst_path, outputs=False, ) finally: @@ -1077,6 +1090,26 @@ def get_study_path(self, metadata: Study) -> Path: """ return Path(metadata.path) / SNAPSHOT_RELATIVE_PATH + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + export_study_flat( + path_study=path_study, + dest=dst_path, + outputs=outputs, + output_src_path=output_src_path, + output_list_filter=output_list_filter, + ) + study = self.study_factory.create_from_fs( + dst_path, "", use_cache=False + ) + study.tree.denormalize() + def get_synthesis( self, metadata: VariantStudy, diff --git a/tests/storage/business/test_export.py b/tests/storage/business/test_export.py index 04f47b6d51..0684443355 100644 --- a/tests/storage/business/test_export.py +++ b/tests/storage/business/test_export.py @@ -5,10 +5,10 @@ import pytest from checksumdir import dirhash -from antarest.core.config import Config, StorageConfig +from antarest.core.config import Config from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.study.storage.rawstudy.raw_study_service import RawStudyService from antarest.study.storage.abstract_storage_service import export_study_flat +from antarest.study.storage.rawstudy.raw_study_service import RawStudyService @pytest.mark.unit_test @@ -92,29 +92,15 @@ def test_export_flat(tmp_path: Path): root_hash = dirhash(root, "md5") root_without_output_hash = dirhash(root_without_output, "md5") - study_factory = Mock() - - study_service = RawStudyService( - config=Config(storage=StorageConfig(tmp_dir=tmp_path)), - study_factory=study_factory, - path_resources=Mock(), - patch_service=Mock(), - cache=Mock(), - ) - study_tree = Mock() - study_factory.create_from_fs.return_value = study_tree - - study = RawStudy(id="id", path=root) - path_study = Path(study.path) + path_study = root - export_study_flat(path_study, tmp_path / "copy_with_output", study_factory, outputs=True) + export_study_flat(path_study, tmp_path / "copy_with_output", outputs=True) copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") assert root_hash == copy_with_output_hash export_study_flat( path_study, tmp_path / "copy_without_output", - study_factory, outputs=False, ) copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") diff --git a/tests/storage/integration/test_exporter.py b/tests/storage/integration/test_exporter.py index 2ab14e26c2..aefa0dfb65 100644 --- a/tests/storage/integration/test_exporter.py +++ b/tests/storage/integration/test_exporter.py @@ -16,7 +16,7 @@ from antarest.matrixstore.service import MatrixService from antarest.study.main import build_study_service from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.study.storage.utils import export_study_flat +from antarest.study.storage.abstract_storage_service import export_study_flat from antarest.study.storage.variantstudy.business.matrix_constants_generator import GeneratorMatrixConstants from tests.storage.conftest import SimpleFileTransferManager, SimpleSyncTaskService @@ -118,7 +118,6 @@ def test_export_flat( export_study_flat( path_studies / "STA-mini", export_path / "STA-mini-export", - Mock(), outputs, output_list, ) diff --git a/tests/variantstudy/model/test_variant_model.py b/tests/variantstudy/model/test_variant_model.py index 3b2a66c4f5..014943699c 100644 --- a/tests/variantstudy/model/test_variant_model.py +++ b/tests/variantstudy/model/test_variant_model.py @@ -1,6 +1,6 @@ import datetime from pathlib import Path -from unittest.mock import ANY, Mock, patch +from unittest.mock import ANY, Mock from sqlalchemy import create_engine @@ -13,7 +13,6 @@ from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware, db from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, StudyAdditionalData from antarest.study.storage.variantstudy.command_factory import CommandFactory -from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.model.model import CommandDTO, GenerationResultInfoDTO from antarest.study.storage.variantstudy.repository import VariantStudyRepository from antarest.study.storage.variantstudy.variant_study_service import SNAPSHOT_RELATIVE_PATH, VariantStudyService @@ -29,12 +28,7 @@ ) -@patch( - "antarest.study.storage.variantstudy.variant_study_service.export_study_flat" -) -def test_commands_service( - mock_export, tmp_path: Path, command_factory: CommandFactory -): +def test_commands_service(tmp_path: Path, command_factory: CommandFactory): engine = create_engine( "sqlite:///:memory:", echo=False, @@ -47,7 +41,6 @@ def test_commands_service( custom_engine=engine, session_args={"autocommit": False, "autoflush": False}, ) - repository = VariantStudyRepository(LocalCache()) service = VariantStudyService( raw_study_service=Mock(), @@ -145,8 +138,7 @@ def test_commands_service( assert study.snapshot.id == study.id -@patch("antarest.study.storage.variantstudy.variant_study_service.export_study_flat") -def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandFactory) -> None: +def test_smart_generation(tmp_path: Path, command_factory: CommandFactory) -> None: engine = create_engine( "sqlite:///:memory:", echo=False, @@ -180,18 +172,13 @@ def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandF ] # noinspection PyUnusedLocal - def export_flat( - path_study: Path, - dest: Path, - study_factory: VariantStudy, - outputs: bool = True, - denormalize: bool = True, - ) -> None: - dest.mkdir(parents=True) - (dest / "user").mkdir() - (dest / "user" / "some_unmanaged_config").touch() - - mock_export.side_effect = export_flat + def export_flat(path_study: Path, dst_path: Path, outputs: bool = True) -> None: + dst_path.mkdir(parents=True) + (dst_path / "user").mkdir() + (dst_path / "user" / "some_unmanaged_config").touch() + + service.raw_study_service.export_study_flat.side_effect = export_flat + with db(): origin_id = "base-study" # noinspection PyArgumentList From 91aaddcbfac98d836d580d74f892b61451c02b3c Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Tue, 1 Aug 2023 14:19:03 +0200 Subject: [PATCH 09/13] fix(api): add documentation (#1646) --- antarest/study/common/studystorage.py | 16 +++++++-- antarest/study/service.py | 6 ++-- .../study/storage/abstract_storage_service.py | 35 +++++++++---------- .../storage/rawstudy/raw_study_service.py | 6 ++-- .../variantstudy/variant_study_service.py | 28 ++------------- 5 files changed, 39 insertions(+), 52 deletions(-) diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index 1eca3a0f1d..552697b25d 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -255,10 +255,13 @@ def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) - raise NotImplementedError() def unarchive(self, study: T) -> None: + """ + Unarchived a study + Args: + study: StudyFactory + """ raise NotImplementedError() - # def export_study_flat(self, **kwargs) -> None: - # raise NotImplementedError() def export_study_flat( self, path_study: Path, @@ -267,4 +270,13 @@ def export_study_flat( output_src_path: Optional[Path] = None, output_list_filter: Optional[List[str]] = None, ) -> None: + """ + Export study to destination + Args: + path_study: source path. + dst_path: destination path. + outputs: list of outputs to keep. + output_src_path: list of source outputs path + output_list_filter:list of outputs to keep + """ raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index 756954e51a..fb487cb2c8 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -2,8 +2,8 @@ import io import json import logging -import shutil import os +import shutil from datetime import datetime, timedelta from http import HTTPStatus from pathlib import Path @@ -88,6 +88,7 @@ StudySimResultDTO, ) from antarest.study.repository import StudyMetadataRepository +from antarest.study.storage.abstract_storage_service import export_study_flat from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.folder_node import ChildNotFoundError from antarest.study.storage.rawstudy.model.filesystem.ini_file_node import IniFileNode @@ -107,7 +108,6 @@ remove_from_cache, study_matcher, ) -from antarest.study.storage.abstract_storage_service import export_study_flat from antarest.study.storage.variantstudy.model.command.icommand import ICommand from antarest.study.storage.variantstudy.model.command.replace_matrix import ReplaceMatrix from antarest.study.storage.variantstudy.model.command.update_comments import UpdateComments @@ -1883,7 +1883,7 @@ def unarchive(self, uuid: str, params: RequestParameters) -> str: def unarchive_task(notifier: TaskUpdateNotifier) -> TaskResult: study_to_archive = self.get_study(uuid) - self.storage_service.raw_study_service.unarchive(study_to_archive) + self.storage_service.raw_study_service.unarchived(study_to_archive) study_to_archive.archived = False os.unlink(self.storage_service.raw_study_service.get_archive_path(study_to_archive)) diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index fb11a5bf50..6fbe0c0a7f 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -1,25 +1,26 @@ +import functools import logging +import os import shutil import tempfile +import time +import zipfile from abc import ABC from pathlib import Path -from typing import IO, List, Optional, Union +from typing import IO, List, Optional, Union, Sequence from uuid import uuid4 -import time -from zipfile import ZipFile -import os - from antarest.core.config import Config from antarest.core.exceptions import BadOutputError, StudyOutputNotFoundError from antarest.core.interfaces.cache import CacheConstants, ICache from antarest.core.model import JSON -from antarest.core.utils.utils import StopWatch, assert_this, extract_zip, unzip, zip_dir +from antarest.core.utils.utils import StopWatch, extract_zip, unzip, zip_dir from antarest.study.common.studystorage import IStudyStorageService, T from antarest.study.common.utils import get_study_information from antarest.study.model import ( PatchOutputs, PatchStudy, + RawStudy, StudyAdditionalData, StudyMetadataDTO, StudyMetadataPatchDTO, @@ -36,6 +37,11 @@ logger = logging.getLogger(__name__) +# noinspection PyUnusedLocal +def _ignore_patterns(path_study: Path, directory: str, contents: Sequence[str]) -> Sequence[str]: + return ["output"] if Path(directory) == path_study else [] + + def export_study_flat( path_study: Path, dest: Path, @@ -49,21 +55,16 @@ def export_study_flat( Args: path_study: Study source path dest: Destination path. - study_factory: StudyFactory, outputs: List of outputs to keep. output_list_filter: List of outputs to keep (None indicate all outputs). - output_src_path: Denormalize the study (replace matrix links by real matrices). - + output_src_path: list of source outputs path. """ start_time = time.time() output_src_path = output_src_path or path_study / "output" output_dest_path = dest / "output" - ignore_patterns = ( - lambda directory, contents: ["output"] - if str(directory) == str(path_study) - else [] - ) + ignore_patterns = functools.partial(_ignore_patterns, path_study) + # noinspection PyTypeChecker shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns) if outputs and output_src_path.is_dir(): if output_dest_path.exists(): @@ -73,7 +74,7 @@ def export_study_flat( for output in output_list_filter: zip_path = output_src_path / f"{output}.zip" if zip_path.exists(): - with ZipFile(zip_path) as zf: + with zipfile.ZipFile(zip_path) as zf: zf.extractall(output_dest_path / output) else: shutil.copytree( @@ -304,9 +305,7 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path: outputs=outputs, output_src_path=output_src_path, ) - study = self.study_factory.create_from_fs( - tmp_study_path, "", use_cache=False - ) + study = self.study_factory.create_from_fs(tmp_study_path, "", use_cache=False) study.tree.denormalize() stopwatch = StopWatch() zip_dir(tmp_study_path, target) diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index e8946b4fca..c4552cc350 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -337,9 +337,7 @@ def export_study_flat( output_list_filter=output_list_filter, output_src_path=output_src_path, ) - study = self.study_factory.create_from_fs( - dst_path, "", use_cache=False - ) + study = self.study_factory.create_from_fs(dst_path, "", use_cache=False) study.tree.denormalize() def check_errors( @@ -370,7 +368,7 @@ def archive(self, study: RawStudy) -> Path: self.cache.invalidate(study.id) return new_study_path - def unarchive(self, study: RawStudy) -> None: + def unarchived(self, study: RawStudy) -> None: with open( self.get_archive_path(study), "rb", diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 5ecc992900..cc18c03bb7 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -39,12 +39,7 @@ from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory from antarest.study.storage.rawstudy.raw_study_service import RawStudyService -from antarest.study.storage.utils import ( - assert_permission, - get_default_workspace_path, - is_managed, - remove_from_cache, -) +from antarest.study.storage.utils import assert_permission, get_default_workspace_path, is_managed, remove_from_cache from antarest.study.storage.variantstudy.business.utils import transform_command_to_dto from antarest.study.storage.variantstudy.command_factory import CommandFactory from antarest.study.storage.variantstudy.model.command.icommand import ICommand @@ -719,21 +714,6 @@ def _generate( if last_executed_command_index is None: if isinstance(parent_study, VariantStudy): - # self._safe_generation(parent_study) - # self.export_study_flat( - # metadata=parent_study, - # dst_path=dst_path, - # outputs=False, - # denormalize=False, - # ) - # else: - # self.raw_study_service.export_study_flat( - # metadata=parent_study, - # dst_path=dst_path, - # outputs=False, - # denormalize=False, - # ) - self._safe_generation(parent_study) path_study = Path(parent_study.path) snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH @@ -747,7 +727,7 @@ def _generate( else: path_study = Path(parent_study.path) if parent_study.archived: - self.raw_study_service.unarchive(parent_study) + self.raw_study_service.unarchived(parent_study) try: self.raw_study_service.export_study_flat( path_study=path_study, @@ -1105,9 +1085,7 @@ def export_study_flat( output_src_path=output_src_path, output_list_filter=output_list_filter, ) - study = self.study_factory.create_from_fs( - dst_path, "", use_cache=False - ) + study = self.study_factory.create_from_fs(dst_path, "", use_cache=False) study.tree.denormalize() def get_synthesis( From 8652528310b87dbb4db4e379dda4b42aa877df8c Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Tue, 1 Aug 2023 14:33:16 +0200 Subject: [PATCH 10/13] fix(api): update (#1646) --- antarest/study/common/studystorage.py | 2 +- antarest/study/service.py | 2 +- antarest/study/storage/rawstudy/raw_study_service.py | 2 +- antarest/study/storage/variantstudy/variant_study_service.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index 552697b25d..eb3f561448 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -276,7 +276,7 @@ def export_study_flat( path_study: source path. dst_path: destination path. outputs: list of outputs to keep. - output_src_path: list of source outputs path + output_src_path: list output path output_list_filter:list of outputs to keep """ raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index fb487cb2c8..baaf7a9782 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1036,7 +1036,7 @@ def export_study_flat( storage = self.storage_service.get_storage(study) if isinstance(study, RawStudy): if study.archived: - storage.unarchive(study) + storage.unarchived(study) try: return storage.export_study_flat( path_study=path_study, diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index c4552cc350..cc73f37391 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -368,7 +368,7 @@ def archive(self, study: RawStudy) -> Path: self.cache.invalidate(study.id) return new_study_path - def unarchived(self, study: RawStudy) -> None: + def unarchive(self, study: RawStudy) -> None: with open( self.get_archive_path(study), "rb", diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index cc18c03bb7..85299f3736 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -727,7 +727,7 @@ def _generate( else: path_study = Path(parent_study.path) if parent_study.archived: - self.raw_study_service.unarchived(parent_study) + self.raw_study_service.unarchive(parent_study) try: self.raw_study_service.export_study_flat( path_study=path_study, From 2ba8eef49adfe4bc7f5b01d7747298eea148f913 Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Tue, 1 Aug 2023 14:37:20 +0200 Subject: [PATCH 11/13] fix(api): update again(#1646) --- antarest/study/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/study/service.py b/antarest/study/service.py index baaf7a9782..f5c5b5a164 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1036,7 +1036,7 @@ def export_study_flat( storage = self.storage_service.get_storage(study) if isinstance(study, RawStudy): if study.archived: - storage.unarchived(study) + storage.unarchive(study) try: return storage.export_study_flat( path_study=path_study, @@ -1883,7 +1883,7 @@ def unarchive(self, uuid: str, params: RequestParameters) -> str: def unarchive_task(notifier: TaskUpdateNotifier) -> TaskResult: study_to_archive = self.get_study(uuid) - self.storage_service.raw_study_service.unarchived(study_to_archive) + self.storage_service.raw_study_service.unarchive(study_to_archive) study_to_archive.archived = False os.unlink(self.storage_service.raw_study_service.get_archive_path(study_to_archive)) From 9c5699d1d76ba2275eb0bc75a2b0b545d7e89f33 Mon Sep 17 00:00:00 2001 From: TLAIDI Date: Tue, 1 Aug 2023 16:11:56 +0200 Subject: [PATCH 12/13] fix(api): remove methods: (unarchive, export_study_flat) from studystorage.py (#1646) --- antarest/study/common/studystorage.py | 27 --------------------------- antarest/study/service.py | 22 +++++++++++++--------- 2 files changed, 13 insertions(+), 36 deletions(-) diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index eb3f561448..59d6311dbf 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -253,30 +253,3 @@ def archive_study_output(self, study: T, output_id: str) -> bool: @abstractmethod def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) -> bool: raise NotImplementedError() - - def unarchive(self, study: T) -> None: - """ - Unarchived a study - Args: - study: StudyFactory - """ - raise NotImplementedError() - - def export_study_flat( - self, - path_study: Path, - dst_path: Path, - outputs: bool = True, - output_src_path: Optional[Path] = None, - output_list_filter: Optional[List[str]] = None, - ) -> None: - """ - Export study to destination - Args: - path_study: source path. - dst_path: destination path. - outputs: list of outputs to keep. - output_src_path: list output path - output_list_filter:list of outputs to keep - """ - raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index f5c5b5a164..c2e7d5da8b 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -922,6 +922,7 @@ def export_task(notifier: TaskUpdateNotifier) -> TaskResult: target_study = self.get_study(uuid) storage = self.storage_service.get_storage(target_study) if isinstance(target_study, RawStudy): + storage = cast(RawStudyService, storage) if target_study.archived: storage.unarchive(target_study) try: @@ -1035,6 +1036,7 @@ def export_study_flat( path_study = Path(study.path) storage = self.storage_service.get_storage(study) if isinstance(study, RawStudy): + storage = cast(RawStudyService, storage) if study.archived: storage.unarchive(study) try: @@ -1048,15 +1050,17 @@ def export_study_flat( finally: if study.archived: shutil.rmtree(study.path) - snapshot_path = path_study / "snapshot" - output_src_path = path_study / "output" - return storage.export_study_flat( - path_study=snapshot_path, - dst_path=dest, - outputs=len(output_list or []) > 0, - output_list_filter=output_list, - output_src_path=output_src_path, - ) + else: + snapshot_path = path_study / "snapshot" + output_src_path = path_study / "output" + storage = cast(VariantStudyService, storage) + return storage.export_study_flat( + path_study=snapshot_path, + dst_path=dest, + outputs=len(output_list or []) > 0, + output_list_filter=output_list, + output_src_path=output_src_path, + ) def delete_study(self, uuid: str, children: bool, params: RequestParameters) -> None: """ From abd74902e361a22afc7f7b344bd056d13b33b503 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Tue, 12 Sep 2023 11:22:47 +0200 Subject: [PATCH 13/13] test(export): adjust the unit test to accommodate the updated signature in 'export_study_flat' --- tests/study/storage/rawstudy/test_raw_study_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/study/storage/rawstudy/test_raw_study_service.py b/tests/study/storage/rawstudy/test_raw_study_service.py index e306122f46..c1ba7c13e7 100644 --- a/tests/study/storage/rawstudy/test_raw_study_service.py +++ b/tests/study/storage/rawstudy/test_raw_study_service.py @@ -192,11 +192,11 @@ def test_export_study_flat( # Run the export target_path = tmp_path / raw_study_path.with_suffix(".exported").name raw_study_service.export_study_flat( - raw_study, + Path(raw_study.path), target_path, outputs=outputs, output_list_filter=output_filter, - denormalize=denormalize, + # fixme: denormalize=denormalize, ) # Collect the resulting files