diff --git a/antarest/study/common/studystorage.py b/antarest/study/common/studystorage.py index f795b46d6c..1eca3a0f1d 100644 --- a/antarest/study/common/studystorage.py +++ b/antarest/study/common/studystorage.py @@ -256,3 +256,15 @@ def unarchive_study_output(self, study: T, output_id: str, keep_src_zip: bool) - def unarchive(self, study: T) -> None: raise NotImplementedError() + + # def export_study_flat(self, **kwargs) -> None: + # raise NotImplementedError() + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + raise NotImplementedError() diff --git a/antarest/study/service.py b/antarest/study/service.py index f4bb7304dd..756954e51a 100644 --- a/antarest/study/service.py +++ b/antarest/study/service.py @@ -1033,26 +1033,26 @@ def export_study_flat( assert_permission(params.user, study, StudyPermissionType.READ) self._assert_study_unarchived(study) path_study = Path(study.path) + storage = self.storage_service.get_storage(study) if isinstance(study, RawStudy): if study.archived: - self.storage_service.get_storage(study).unarchive(study) + storage.unarchive(study) try: - return export_study_flat( + return storage.export_study_flat( path_study=path_study, - study_factory=study, - dest=dest, + dst_path=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, + output_src_path=None, ) finally: if study.archived: shutil.rmtree(study.path) snapshot_path = path_study / "snapshot" output_src_path = path_study / "output" - export_study_flat( + return storage.export_study_flat( path_study=snapshot_path, - study_factory=study, - dest=dest, + dst_path=dest, outputs=len(output_list or []) > 0, output_list_filter=output_list, output_src_path=output_src_path, diff --git a/antarest/study/storage/abstract_storage_service.py b/antarest/study/storage/abstract_storage_service.py index be565c6127..fb11a5bf50 100644 --- a/antarest/study/storage/abstract_storage_service.py +++ b/antarest/study/storage/abstract_storage_service.py @@ -39,7 +39,6 @@ def export_study_flat( path_study: Path, dest: Path, - study_factory: StudyFactory, outputs: bool = True, output_list_filter: Optional[List[str]] = None, output_src_path: Optional[Path] = None, @@ -91,11 +90,6 @@ def export_study_flat( duration = "{:.3f}".format(stop_time - start_time) logger.info(f"Study {path_study} exported (flat mode) in {duration}s") - study = study_factory.create_from_fs(dest, "", use_cache=False) - study.tree.denormalize() - duration = "{:.3f}".format(time.time() - stop_time) - logger.info(f"Study {path_study} denormalized in {duration}s") - class AbstractStorageService(IStudyStorageService[T], ABC): def __init__( @@ -295,19 +289,25 @@ def export_study(self, metadata: T, target: Path, outputs: bool = True) -> Path: with tempfile.TemporaryDirectory(dir=self.config.storage.tmp_dir) as tmpdir: logger.info(f"Exporting study {metadata.id} to temporary path {tmpdir}") tmp_study_path = Path(tmpdir) / "tmp_copy" - if not isinstance(metadata, RawStudy): + if isinstance(metadata, RawStudy): + export_study_flat( + path_study=path_study, + dest=tmp_study_path, + outputs=outputs, + ) + else: snapshot_path = path_study / "snapshot" output_src_path = path_study / "output" export_study_flat( path_study=snapshot_path, - study_factory=self.study_factory, dest=tmp_study_path, outputs=outputs, output_src_path=output_src_path, ) - export_study_flat( - path_study, tmp_study_path, self.study_factory, outputs + study = self.study_factory.create_from_fs( + tmp_study_path, "", use_cache=False ) + study.tree.denormalize() stopwatch = StopWatch() zip_dir(tmp_study_path, target) stopwatch.log_elapsed(lambda x: logger.info(f"Study {path_study} exported (zipped mode) in {x}s")) diff --git a/antarest/study/storage/rawstudy/raw_study_service.py b/antarest/study/storage/rawstudy/raw_study_service.py index c9ce8fd0d1..e8946b4fca 100644 --- a/antarest/study/storage/rawstudy/raw_study_service.py +++ b/antarest/study/storage/rawstudy/raw_study_service.py @@ -16,7 +16,7 @@ from antarest.core.requests import RequestParameters from antarest.core.utils.utils import extract_zip from antarest.study.model import DEFAULT_WORKSPACE_NAME, Patch, RawStudy, Study, StudyAdditionalData -from antarest.study.storage.abstract_storage_service import AbstractStorageService +from antarest.study.storage.abstract_storage_service import AbstractStorageService, export_study_flat from antarest.study.storage.patch_service import PatchService from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig, FileStudyTreeConfigDTO from antarest.study.storage.rawstudy.model.filesystem.factory import FileStudy, StudyFactory @@ -322,6 +322,26 @@ def import_study(self, metadata: RawStudy, stream: IO[bytes]) -> Study: metadata.path = str(path_study) return metadata + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + export_study_flat( + path_study=path_study, + dest=dst_path, + outputs=outputs, + output_list_filter=output_list_filter, + output_src_path=output_src_path, + ) + study = self.study_factory.create_from_fs( + dst_path, "", use_cache=False + ) + study.tree.denormalize() + def check_errors( self, metadata: RawStudy, diff --git a/antarest/study/storage/variantstudy/variant_study_service.py b/antarest/study/storage/variantstudy/variant_study_service.py index 765c9b053a..5ecc992900 100644 --- a/antarest/study/storage/variantstudy/variant_study_service.py +++ b/antarest/study/storage/variantstudy/variant_study_service.py @@ -719,14 +719,28 @@ def _generate( if last_executed_command_index is None: if isinstance(parent_study, VariantStudy): + # self._safe_generation(parent_study) + # self.export_study_flat( + # metadata=parent_study, + # dst_path=dst_path, + # outputs=False, + # denormalize=False, + # ) + # else: + # self.raw_study_service.export_study_flat( + # metadata=parent_study, + # dst_path=dst_path, + # outputs=False, + # denormalize=False, + # ) + self._safe_generation(parent_study) path_study = Path(parent_study.path) snapshot_path = path_study / SNAPSHOT_RELATIVE_PATH output_src_path = path_study / "output" - export_study_flat( - snapshot_path, - dst_path, - self.study_factory, + self.export_study_flat( + path_study=snapshot_path, + dst_path=dst_path, outputs=False, output_src_path=output_src_path, ) @@ -735,10 +749,9 @@ def _generate( if parent_study.archived: self.raw_study_service.unarchive(parent_study) try: - export_study_flat( + self.raw_study_service.export_study_flat( path_study=path_study, - dest=dst_path, - study_factory=self.study_factory, + dst_path=dst_path, outputs=False, ) finally: @@ -1077,6 +1090,26 @@ def get_study_path(self, metadata: Study) -> Path: """ return Path(metadata.path) / SNAPSHOT_RELATIVE_PATH + def export_study_flat( + self, + path_study: Path, + dst_path: Path, + outputs: bool = True, + output_src_path: Optional[Path] = None, + output_list_filter: Optional[List[str]] = None, + ) -> None: + export_study_flat( + path_study=path_study, + dest=dst_path, + outputs=outputs, + output_src_path=output_src_path, + output_list_filter=output_list_filter, + ) + study = self.study_factory.create_from_fs( + dst_path, "", use_cache=False + ) + study.tree.denormalize() + def get_synthesis( self, metadata: VariantStudy, diff --git a/tests/storage/business/test_export.py b/tests/storage/business/test_export.py index 04f47b6d51..0684443355 100644 --- a/tests/storage/business/test_export.py +++ b/tests/storage/business/test_export.py @@ -5,10 +5,10 @@ import pytest from checksumdir import dirhash -from antarest.core.config import Config, StorageConfig +from antarest.core.config import Config from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.study.storage.rawstudy.raw_study_service import RawStudyService from antarest.study.storage.abstract_storage_service import export_study_flat +from antarest.study.storage.rawstudy.raw_study_service import RawStudyService @pytest.mark.unit_test @@ -92,29 +92,15 @@ def test_export_flat(tmp_path: Path): root_hash = dirhash(root, "md5") root_without_output_hash = dirhash(root_without_output, "md5") - study_factory = Mock() - - study_service = RawStudyService( - config=Config(storage=StorageConfig(tmp_dir=tmp_path)), - study_factory=study_factory, - path_resources=Mock(), - patch_service=Mock(), - cache=Mock(), - ) - study_tree = Mock() - study_factory.create_from_fs.return_value = study_tree - - study = RawStudy(id="id", path=root) - path_study = Path(study.path) + path_study = root - export_study_flat(path_study, tmp_path / "copy_with_output", study_factory, outputs=True) + export_study_flat(path_study, tmp_path / "copy_with_output", outputs=True) copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") assert root_hash == copy_with_output_hash export_study_flat( path_study, tmp_path / "copy_without_output", - study_factory, outputs=False, ) copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") diff --git a/tests/storage/integration/test_exporter.py b/tests/storage/integration/test_exporter.py index 2ab14e26c2..aefa0dfb65 100644 --- a/tests/storage/integration/test_exporter.py +++ b/tests/storage/integration/test_exporter.py @@ -16,7 +16,7 @@ from antarest.matrixstore.service import MatrixService from antarest.study.main import build_study_service from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.study.storage.utils import export_study_flat +from antarest.study.storage.abstract_storage_service import export_study_flat from antarest.study.storage.variantstudy.business.matrix_constants_generator import GeneratorMatrixConstants from tests.storage.conftest import SimpleFileTransferManager, SimpleSyncTaskService @@ -118,7 +118,6 @@ def test_export_flat( export_study_flat( path_studies / "STA-mini", export_path / "STA-mini-export", - Mock(), outputs, output_list, ) diff --git a/tests/variantstudy/model/test_variant_model.py b/tests/variantstudy/model/test_variant_model.py index 3b2a66c4f5..014943699c 100644 --- a/tests/variantstudy/model/test_variant_model.py +++ b/tests/variantstudy/model/test_variant_model.py @@ -1,6 +1,6 @@ import datetime from pathlib import Path -from unittest.mock import ANY, Mock, patch +from unittest.mock import ANY, Mock from sqlalchemy import create_engine @@ -13,7 +13,6 @@ from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware, db from antarest.study.model import DEFAULT_WORKSPACE_NAME, RawStudy, StudyAdditionalData from antarest.study.storage.variantstudy.command_factory import CommandFactory -from antarest.study.storage.variantstudy.model.dbmodel import VariantStudy from antarest.study.storage.variantstudy.model.model import CommandDTO, GenerationResultInfoDTO from antarest.study.storage.variantstudy.repository import VariantStudyRepository from antarest.study.storage.variantstudy.variant_study_service import SNAPSHOT_RELATIVE_PATH, VariantStudyService @@ -29,12 +28,7 @@ ) -@patch( - "antarest.study.storage.variantstudy.variant_study_service.export_study_flat" -) -def test_commands_service( - mock_export, tmp_path: Path, command_factory: CommandFactory -): +def test_commands_service(tmp_path: Path, command_factory: CommandFactory): engine = create_engine( "sqlite:///:memory:", echo=False, @@ -47,7 +41,6 @@ def test_commands_service( custom_engine=engine, session_args={"autocommit": False, "autoflush": False}, ) - repository = VariantStudyRepository(LocalCache()) service = VariantStudyService( raw_study_service=Mock(), @@ -145,8 +138,7 @@ def test_commands_service( assert study.snapshot.id == study.id -@patch("antarest.study.storage.variantstudy.variant_study_service.export_study_flat") -def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandFactory) -> None: +def test_smart_generation(tmp_path: Path, command_factory: CommandFactory) -> None: engine = create_engine( "sqlite:///:memory:", echo=False, @@ -180,18 +172,13 @@ def test_smart_generation(mock_export, tmp_path: Path, command_factory: CommandF ] # noinspection PyUnusedLocal - def export_flat( - path_study: Path, - dest: Path, - study_factory: VariantStudy, - outputs: bool = True, - denormalize: bool = True, - ) -> None: - dest.mkdir(parents=True) - (dest / "user").mkdir() - (dest / "user" / "some_unmanaged_config").touch() - - mock_export.side_effect = export_flat + def export_flat(path_study: Path, dst_path: Path, outputs: bool = True) -> None: + dst_path.mkdir(parents=True) + (dst_path / "user").mkdir() + (dst_path / "user" / "some_unmanaged_config").touch() + + service.raw_study_service.export_study_flat.side_effect = export_flat + with db(): origin_id = "base-study" # noinspection PyArgumentList