From 2ec766f53c3d99278dc43e62e193fdc78b28e5d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Jolain?= <4466185+FrancoisJ@users.noreply.github.com> Date: Thu, 8 Jul 2021 16:37:03 +0200 Subject: [PATCH] Issue 400 fix exporter service for studies with matrix link (#405) --- antarest/storage/business/exporter_service.py | 72 ++++++++++++-- antarest/storage/main.py | 10 +- .../antares_io/exporter/__init__.py | 0 .../antares_io/exporter/export_file.py | 59 ----------- .../repository/filesystem/exceptions.py | 3 + .../repository/filesystem/folder_node.py | 8 ++ .../repository/filesystem/ini_file_node.py | 6 ++ .../storage/repository/filesystem/inode.py | 26 ++++- .../repository/filesystem/matrix/matrix.py | 35 +++++++ .../repository/filesystem/raw_file_node.py | 6 ++ .../filesystem/root/input/areas/list.py | 6 ++ antarest/storage/web/studies_blueprint.py | 2 +- requirements.txt | 17 ++-- resources/application.yaml | 2 + .../exporter => business}/folder/file.txt | 0 .../storage/business/test_exporter_service.py | 97 +++++++++++++++++-- tests/storage/integration/conftest.py | 3 - tests/storage/integration/test_exporter.py | 4 - .../antares_io/exporter/__init__.py | 0 .../antares_io/exporter/test_export_file.py | 70 ------------- .../filesystem/matrix/test_matrix_node.py | 91 +++++++++++++++++ .../repository/filesystem/test_lazy_node.py | 6 ++ tests/storage/repository/filesystem/utils.py | 6 ++ 23 files changed, 358 insertions(+), 171 deletions(-) delete mode 100644 antarest/storage/repository/antares_io/exporter/__init__.py delete mode 100644 antarest/storage/repository/antares_io/exporter/export_file.py create mode 100644 antarest/storage/repository/filesystem/exceptions.py rename tests/storage/{repository/antares_io/exporter => business}/folder/file.txt (100%) delete mode 100644 tests/storage/repository/antares_io/exporter/__init__.py delete mode 100644 tests/storage/repository/antares_io/exporter/test_export_file.py create mode 100644 tests/storage/repository/filesystem/matrix/test_matrix_node.py diff --git a/antarest/storage/business/exporter_service.py b/antarest/storage/business/exporter_service.py index 42aa9f9a6d..603df7a6b9 100644 --- a/antarest/storage/business/exporter_service.py +++ b/antarest/storage/business/exporter_service.py @@ -1,13 +1,20 @@ +import glob +import logging +import os +import shutil +import tempfile +import time from io import BytesIO from pathlib import Path +from zipfile import ZipFile, ZIP_DEFLATED +from antarest.common.config import Config from antarest.storage.business.raw_study_service import RawStudyService from antarest.storage.model import Study -from antarest.storage.repository.antares_io.exporter.export_file import ( - Exporter, -) from antarest.storage.repository.filesystem.factory import StudyFactory +logger = logging.getLogger(__name__) + class ExporterService: """ @@ -18,11 +25,11 @@ def __init__( self, study_service: RawStudyService, study_factory: StudyFactory, - exporter: Exporter, + config: Config, ): self.study_service = study_service self.study_factory = study_factory - self.exporter = exporter + self.config = config def export_study( self, metadata: Study, target: Path, outputs: bool = True @@ -39,15 +46,60 @@ def export_study( """ path_study = self.study_service.get_study_path(metadata) - self.study_service.check_study_exists(metadata) - - return self.exporter.export_file(path_study, target, outputs) + return self.export_file(path_study, target, outputs) def export_study_flat( self, metadata: Study, dest: Path, outputs: bool = True ) -> None: path_study = self.study_service.get_study_path(metadata) - self.study_service.check_study_exists(metadata) + self.export_flat(path_study, dest, outputs) + + def export_file( + self, path_study: Path, export_path: Path, outputs: bool = True + ) -> Path: + with tempfile.TemporaryDirectory(dir=self.config.tmp_dir) as tmpdir: + tmp_study_path = Path(tmpdir) / "tmp_copy" + self.export_flat(path_study, tmp_study_path, outputs) + start_time = time.time() + with ZipFile(export_path, "w", ZIP_DEFLATED) as zipf: + current_dir = os.getcwd() + os.chdir(tmp_study_path) + + for path in glob.glob("**", recursive=True): + if outputs or path.split(os.sep)[0] != "output": + zipf.write(path, path) - self.exporter.export_flat(path_study, dest, outputs) + zipf.close() + + os.chdir(current_dir) + duration = "{:.3f}".format(time.time() - start_time) + logger.info( + f"Study {path_study} exported (zipped mode) in {duration}s" + ) + return export_path + + def export_flat( + self, + path_study: Path, + dest: Path, + outputs: bool = False, + ) -> None: + start_time = time.time() + ignore_patterns = ( + ( + lambda directory, contents: ["output"] + if str(directory) == str(path_study) + else [] + ) + if not outputs + else None + ) + shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns) + stop_time = time.time() + duration = "{:.3f}".format(stop_time - start_time) + logger.info(f"Study {path_study} exported (flat mode) in {duration}s") + _, study = self.study_factory.create_from_fs(dest, "") + study.denormalize() + duration = "{:.3f}".format(time.time() - stop_time) + logger.info(f"Study {path_study} denormalized in {duration}s") diff --git a/antarest/storage/main.py b/antarest/storage/main.py index f1719fe6a7..94a4ec775e 100644 --- a/antarest/storage/main.py +++ b/antarest/storage/main.py @@ -12,9 +12,6 @@ from antarest.storage.business.raw_study_service import RawStudyService from antarest.storage.business.uri_resolver_service import UriResolverService from antarest.storage.business.watcher import Watcher -from antarest.storage.repository.antares_io.exporter.export_file import ( - Exporter, -) from antarest.storage.repository.filesystem.factory import StudyFactory from antarest.storage.repository.patch_repository import PatchRepository from antarest.storage.repository.study import StudyMetadataRepository @@ -30,7 +27,6 @@ def build_storage( user_service: LoginService, matrix_service: MatrixService, metadata_repository: Optional[StudyMetadataRepository] = None, - exporter: Optional[Exporter] = None, storage_service: Optional[StorageService] = None, patch_service: Optional[PatchService] = None, event_bus: IEventBus = DummyEventBusService(), @@ -42,9 +38,8 @@ def build_storage( application: flask application config: server config user_service: user service facade + matrix_service: matrix store service metadata_repository: used by testing to inject mock. Let None to use true instantiation - study_factory: used by testing to inject mock. Let None to use true instantiation - exporter: used by testing to inject mock. Let None to use true instantiation storage_service: used by testing to inject mock. Let None to use true instantiation patch_service: used by testing to inject mock. Let None to use true instantiation event_bus: used by testing to inject mock. Let None to use true instantiation @@ -57,7 +52,6 @@ def build_storage( resolver = UriResolverService(config, matrix_service=matrix_service) study_factory = StudyFactory(matrix=matrix_service, resolver=resolver) - exporter = exporter or Exporter() metadata_repository = metadata_repository or StudyMetadataRepository() patch_service = patch_service or PatchService(PatchRepository()) @@ -75,7 +69,7 @@ def build_storage( exporter_service = ExporterService( study_service=study_service, study_factory=study_factory, - exporter=exporter, + config=config, ) storage_service = storage_service or StorageService( diff --git a/antarest/storage/repository/antares_io/exporter/__init__.py b/antarest/storage/repository/antares_io/exporter/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/antarest/storage/repository/antares_io/exporter/export_file.py b/antarest/storage/repository/antares_io/exporter/export_file.py deleted file mode 100644 index 6da8f13a91..0000000000 --- a/antarest/storage/repository/antares_io/exporter/export_file.py +++ /dev/null @@ -1,59 +0,0 @@ -import glob -import json -import logging -import os -import re -import shutil -import time -import uuid -from io import BytesIO -from pathlib import Path -from zipfile import ZIP_DEFLATED, ZipFile - -from antarest.common.custom_types import JSON - - -logger = logging.getLogger(__name__) - -# TODO merge with exporter service ? -class Exporter: - def export_file( - self, path_study: Path, export_path: Path, outputs: bool = True - ) -> Path: - start_time = time.time() - with ZipFile(export_path, "w", ZIP_DEFLATED) as zipf: - current_dir = os.getcwd() - os.chdir(path_study) - - for path in glob.glob("**", recursive=True): - if outputs or path.split(os.sep)[0] != "output": - zipf.write(path, path) - - zipf.close() - - os.chdir(current_dir) - duration = "{:.3f}".format(time.time() - start_time) - logger.info( - f"Study {path_study} exported (zipped mode) in {duration}s" - ) - return export_path - - def export_flat( - self, - path_study: Path, - dest: Path, - outputs: bool = False, - ) -> None: - start_time = time.time() - ignore_patterns = ( - ( - lambda directory, contents: ["output"] - if str(directory) == str(path_study) - else [] - ) - if not outputs - else None - ) - shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns) - duration = "{:.3f}".format(time.time() - start_time) - logger.info(f"Study {path_study} exported (flat mode) in {duration}s") diff --git a/antarest/storage/repository/filesystem/exceptions.py b/antarest/storage/repository/filesystem/exceptions.py new file mode 100644 index 0000000000..4a0cdd49cd --- /dev/null +++ b/antarest/storage/repository/filesystem/exceptions.py @@ -0,0 +1,3 @@ +class DenormalizationException(Exception): + def __init__(self, msg: str): + super(DenormalizationException, self).__init__(msg) diff --git a/antarest/storage/repository/filesystem/folder_node.py b/antarest/storage/repository/filesystem/folder_node.py index d8215de996..125e115f0a 100644 --- a/antarest/storage/repository/filesystem/folder_node.py +++ b/antarest/storage/repository/filesystem/folder_node.py @@ -117,6 +117,14 @@ def check_errors( ) return errors + def normalize(self) -> None: + for child in self.build(self.config).values(): + child.normalize() + + def denormalize(self) -> None: + for child in self.build(self.config).values(): + child.denormalize() + def extract_child( self, children: TREE, url: List[str] ) -> Tuple[List[str], List[str]]: diff --git a/antarest/storage/repository/filesystem/ini_file_node.py b/antarest/storage/repository/filesystem/ini_file_node.py index 7a60129d4f..dc229e4676 100644 --- a/antarest/storage/repository/filesystem/ini_file_node.py +++ b/antarest/storage/repository/filesystem/ini_file_node.py @@ -87,6 +87,12 @@ def check_errors( return errors + def normalize(self) -> None: + pass # no external store in this node + + def denormalize(self) -> None: + pass # no external store in this node + def _validate_param( self, section: str, diff --git a/antarest/storage/repository/filesystem/inode.py b/antarest/storage/repository/filesystem/inode.py index 33bd9fd2a7..047a12a47c 100644 --- a/antarest/storage/repository/filesystem/inode.py +++ b/antarest/storage/repository/filesystem/inode.py @@ -23,7 +23,7 @@ def build(self, config: StudyConfig) -> "TREE": Returns: children of current node """ - pass + raise NotImplementedError() @abstractmethod def get( @@ -43,7 +43,7 @@ def get( Returns: json """ - pass + raise NotImplementedError() @abstractmethod def save(self, data: S, url: Optional[List[str]] = None) -> None: @@ -57,7 +57,7 @@ def save(self, data: S, url: Optional[List[str]] = None) -> None: Returns: """ - pass + raise NotImplementedError() @abstractmethod def check_errors( @@ -73,7 +73,25 @@ def check_errors( Returns: list of errors belongs to this node or children """ - pass + raise NotImplementedError() + + @abstractmethod + def normalize(self) -> None: + """ + Scan tree to send matrix in matrix store and replace by its links + Returns: + + """ + raise NotImplementedError() + + @abstractmethod + def denormalize(self) -> None: + """ + Scan tree to fetch matrix by its links + Returns: + + """ + raise NotImplementedError() def _assert_url_end(self, url: Optional[List[str]] = None) -> None: """ diff --git a/antarest/storage/repository/filesystem/matrix/matrix.py b/antarest/storage/repository/filesystem/matrix/matrix.py index 666f8c9b33..cf81e39e45 100644 --- a/antarest/storage/repository/filesystem/matrix/matrix.py +++ b/antarest/storage/repository/filesystem/matrix/matrix.py @@ -8,6 +8,9 @@ from antarest.matrixstore.model import MatrixDTO, MatrixFreq from antarest.storage.repository.filesystem.config.model import StudyConfig from antarest.storage.repository.filesystem.context import ContextServer +from antarest.storage.repository.filesystem.exceptions import ( + DenormalizationException, +) from antarest.storage.repository.filesystem.lazy_node import LazyNode @@ -26,6 +29,38 @@ def get_lazy_content( ) -> str: return f"matrixfile://{self.config.path.name}" + def normalize(self) -> None: + if self.get_link_path().exists(): + return + + matrix = self.load() + dto = MatrixDTO( + freq=MatrixFreq.from_str(self.freq), + index=matrix["index"], + columns=matrix["columns"], + data=matrix["data"], + ) + + uuid = self.context.matrix.create(dto) + self.get_link_path().write_text( + self.context.resolver.build_matrix_uri(uuid) + ) + self.config.path.unlink() + + def denormalize(self) -> None: + if self.config.path.exists(): + return + + uuid = self.get_link_path().read_text() + matrix = self.context.resolver.resolve(uuid) + if not matrix or not isinstance(matrix, dict): + raise DenormalizationException( + f"Failed to retrieve original matrix for {self.config.path}" + ) + + self.dump(matrix) + self.get_link_path().unlink() + @abstractmethod def load( self, diff --git a/antarest/storage/repository/filesystem/raw_file_node.py b/antarest/storage/repository/filesystem/raw_file_node.py index 782c50d5f6..c6324d3a0b 100644 --- a/antarest/storage/repository/filesystem/raw_file_node.py +++ b/antarest/storage/repository/filesystem/raw_file_node.py @@ -46,3 +46,9 @@ def check_errors( raise ValueError(msg) return [msg] return [] + + def normalize(self) -> None: + pass # no external store in this node + + def denormalize(self) -> None: + pass # no external store in this node diff --git a/antarest/storage/repository/filesystem/root/input/areas/list.py b/antarest/storage/repository/filesystem/root/input/areas/list.py index 4af58638bb..0ea552e654 100644 --- a/antarest/storage/repository/filesystem/root/input/areas/list.py +++ b/antarest/storage/repository/filesystem/root/input/areas/list.py @@ -6,6 +6,12 @@ class InputAreasList(INode[List[str], List[str], List[str]]): + def normalize(self) -> None: + pass # no external store in this node + + def denormalize(self) -> None: + pass # no external store in this node + def __init__(self, context: ContextServer, config: StudyConfig): self.config = config self.context = context diff --git a/antarest/storage/web/studies_blueprint.py b/antarest/storage/web/studies_blueprint.py index 5168f6039d..9ff40ca08e 100644 --- a/antarest/storage/web/studies_blueprint.py +++ b/antarest/storage/web/studies_blueprint.py @@ -10,7 +10,7 @@ from starlette.responses import StreamingResponse, Response, FileResponse from antarest.common.config import Config -from antarest.common.custom_types import JSON +from antarest.common.custom_types import JSON, SUB_JSON from antarest.common.jwt import JWTUser from antarest.common.requests import ( RequestParameters, diff --git a/requirements.txt b/requirements.txt index 4c8882284c..9b64f12db6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,12 +9,17 @@ requests~=2.25.1 pandas~=1.1.5 numpy~=1.19.5 dataclasses-json -fastapi[all] +fastapi[all]~=0.65.2 fastapi-jwt-auth python-multipart aiofiles -jinja2 -uvicorn[standard] -bcrypt -contextvars -scandir \ No newline at end of file +jinja2~=2.11.3 +uvicorn[standard]~=0.13.4 +bcrypt~=3.2.0 +contextvars~=2.4 +scandir +starlette~=0.14.2 +locust~=1.5.1 +MarkupSafe~=1.1.1 +checksumdir~=1.2.0 +pydantic~=1.8.2 \ No newline at end of file diff --git a/resources/application.yaml b/resources/application.yaml index d725403111..0cc4fefd5f 100644 --- a/resources/application.yaml +++ b/resources/application.yaml @@ -15,6 +15,8 @@ db: matrixstore: bucket: ./bucket +#tmp_dir: /tmp + storage: workspaces: tmp: diff --git a/tests/storage/repository/antares_io/exporter/folder/file.txt b/tests/storage/business/folder/file.txt similarity index 100% rename from tests/storage/repository/antares_io/exporter/folder/file.txt rename to tests/storage/business/folder/file.txt diff --git a/tests/storage/business/test_exporter_service.py b/tests/storage/business/test_exporter_service.py index de15864fe5..9b2ce68b35 100644 --- a/tests/storage/business/test_exporter_service.py +++ b/tests/storage/business/test_exporter_service.py @@ -1,12 +1,15 @@ from pathlib import Path from unittest.mock import Mock +from zipfile import ZipFile + +from checksumdir import dirhash import pytest +from antarest.common.config import Config from antarest.storage.business.exporter_service import ExporterService from antarest.storage.business.raw_study_service import RawStudyService from antarest.storage.model import Study, DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.storage.repository.filesystem.config.model import StudyConfig def build_storage_service(workspace: Path, uuid: str) -> RawStudyService: @@ -23,20 +26,102 @@ def test_export_file(tmp_path: Path): study_path.mkdir() (study_path / "study.antares").touch() - exporter = Mock() - exporter.export_file.return_value = b"Hello" - study_service = Mock() study_service.check_study_exist.return_value = None exporter_service = ExporterService( study_service=build_storage_service(tmp_path, name), study_factory=Mock(), - exporter=exporter, + config=Config(), ) + exporter_service.export_file = Mock() + exporter_service.export_file.return_value = b"Hello" # Test good study md = RawStudy(id=name, workspace=DEFAULT_WORKSPACE_NAME) export_path = tmp_path / "export.zip" exporter_service.export_study(md, export_path) - exporter.export_file.assert_called_once_with(study_path, export_path, True) + exporter_service.export_file.assert_called_once_with( + study_path, export_path, True + ) + + +@pytest.mark.unit_test +@pytest.mark.parametrize("outputs", [True, False]) +def test_export_file(tmp_path: Path, outputs: bool): + root = tmp_path / "folder" + root.mkdir() + (root / "test").mkdir() + (root / "test/file.txt").write_text("Bonjour") + (root / "file.txt").write_text("Hello, World") + (root / "output").mkdir() + (root / "output/file.txt").write_text("42") + + export_path = tmp_path / "study.zip" + + study_factory = Mock() + exporter_service = ExporterService( + study_service=Mock(), + study_factory=study_factory, + config=Config(), + ) + study_tree = Mock() + study_factory.create_from_fs.return_value = (None, study_tree) + + exporter_service.export_file(root, export_path, outputs) + zipf = ZipFile(export_path) + + assert "file.txt" in zipf.namelist() + assert "test/" in zipf.namelist() + assert "test/file.txt" in zipf.namelist() + assert ("output/" in zipf.namelist()) == outputs + assert ("output/file.txt" in zipf.namelist()) == outputs + + +@pytest.mark.unit_test +def test_export_flat(tmp_path: Path): + root = tmp_path / "folder-with-output" + root.mkdir() + (root / "test").mkdir() + (root / "test/file.txt").write_text("Bonjour") + (root / "test/output").mkdir() + (root / "test/output/file.txt").write_text("Test") + (root / "file.txt").write_text("Hello, World") + (root / "output").mkdir() + (root / "output/file.txt").write_text("42") + + root_without_output = tmp_path / "folder-without-output" + root_without_output.mkdir() + (root_without_output / "test").mkdir() + (root_without_output / "test/file.txt").write_text("Bonjour") + (root_without_output / "test/output").mkdir() + (root_without_output / "test/output/file.txt").write_text("Test") + (root_without_output / "file.txt").write_text("Hello, World") + + root_hash = dirhash(root, "md5") + root_without_output_hash = dirhash(root_without_output, "md5") + + study_factory = Mock() + exporter_service = ExporterService( + study_service=Mock(), + study_factory=study_factory, + config=Config(tmp_dir=tmp_path), + ) + study_tree = Mock() + study_factory.create_from_fs.return_value = (None, study_tree) + + exporter_service.export_flat( + root, tmp_path / "copy_with_output", outputs=True + ) + + copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") + + assert root_hash == copy_with_output_hash + + exporter_service.export_flat( + root, tmp_path / "copy_without_output", outputs=False + ) + + copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") + + assert root_without_output_hash == copy_without_output_hash diff --git a/tests/storage/integration/conftest.py b/tests/storage/integration/conftest.py index e0144e5ea4..f0a564c9b3 100644 --- a/tests/storage/integration/conftest.py +++ b/tests/storage/integration/conftest.py @@ -16,9 +16,6 @@ from antarest.storage.business.raw_study_service import RawStudyService from antarest.storage.main import build_storage from antarest.storage.model import Study, DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.storage.repository.antares_io.exporter.export_file import ( - Exporter, -) from antarest.storage.repository.filesystem.factory import StudyFactory from antarest.storage.service import StorageService diff --git a/tests/storage/integration/test_exporter.py b/tests/storage/integration/test_exporter.py index bd65282a2c..146580622e 100644 --- a/tests/storage/integration/test_exporter.py +++ b/tests/storage/integration/test_exporter.py @@ -12,10 +12,6 @@ WorkspaceConfig, ) from antarest.storage.model import Study, DEFAULT_WORKSPACE_NAME, RawStudy -from antarest.storage.repository.antares_io.exporter.export_file import ( - Exporter, -) -from antarest.storage.repository.filesystem.factory import StudyFactory from antarest.storage.main import build_storage from antarest.storage.service import StorageService diff --git a/tests/storage/repository/antares_io/exporter/__init__.py b/tests/storage/repository/antares_io/exporter/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/storage/repository/antares_io/exporter/test_export_file.py b/tests/storage/repository/antares_io/exporter/test_export_file.py deleted file mode 100644 index 1089c768a9..0000000000 --- a/tests/storage/repository/antares_io/exporter/test_export_file.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -from pathlib import Path -from zipfile import ZipFile - -import pytest -from checksumdir import dirhash - -from antarest.storage.repository.antares_io.exporter.export_file import ( - Exporter, -) - - -@pytest.mark.unit_test -@pytest.mark.parametrize("outputs", [True, False]) -def test_export_file(tmp_path: Path, outputs: bool): - root = tmp_path / "folder" - root.mkdir() - (root / "test").mkdir() - (root / "test/file.txt").write_text("Bonjour") - (root / "file.txt").write_text("Hello, World") - (root / "output").mkdir() - (root / "output/file.txt").write_text("42") - - export_path = tmp_path / "study.zip" - - Exporter().export_file(root, export_path, outputs) - zipf = ZipFile(export_path) - - assert "file.txt" in zipf.namelist() - assert "test/" in zipf.namelist() - assert "test/file.txt" in zipf.namelist() - assert ("output/" in zipf.namelist()) == outputs - assert ("output/file.txt" in zipf.namelist()) == outputs - - -@pytest.mark.unit_test -def test_export_flat(tmp_path: Path): - root = tmp_path / "folder-with-output" - root.mkdir() - (root / "test").mkdir() - (root / "test/file.txt").write_text("Bonjour") - (root / "test/output").mkdir() - (root / "test/output/file.txt").write_text("Test") - (root / "file.txt").write_text("Hello, World") - (root / "output").mkdir() - (root / "output/file.txt").write_text("42") - - root_without_output = tmp_path / "folder-without-output" - root_without_output.mkdir() - (root_without_output / "test").mkdir() - (root_without_output / "test/file.txt").write_text("Bonjour") - (root_without_output / "test/output").mkdir() - (root_without_output / "test/output/file.txt").write_text("Test") - (root_without_output / "file.txt").write_text("Hello, World") - - root_hash = dirhash(root, "md5") - root_without_output_hash = dirhash(root_without_output, "md5") - Exporter().export_flat(root, tmp_path / "copy_with_output", outputs=True) - - copy_with_output_hash = dirhash(tmp_path / "copy_with_output", "md5") - - assert root_hash == copy_with_output_hash - - Exporter().export_flat( - root, tmp_path / "copy_without_output", outputs=False - ) - - copy_without_output_hash = dirhash(tmp_path / "copy_without_output", "md5") - - assert root_without_output_hash == copy_without_output_hash diff --git a/tests/storage/repository/filesystem/matrix/test_matrix_node.py b/tests/storage/repository/filesystem/matrix/test_matrix_node.py new file mode 100644 index 0000000000..464f454d45 --- /dev/null +++ b/tests/storage/repository/filesystem/matrix/test_matrix_node.py @@ -0,0 +1,91 @@ +import json +from pathlib import Path +from typing import Optional, List +from unittest.mock import Mock + +from antarest.common.custom_types import JSON +from antarest.matrixstore.model import MatrixDTO, MatrixFreq +from antarest.storage.repository.filesystem.config.model import StudyConfig +from antarest.storage.repository.filesystem.context import ContextServer +from antarest.storage.repository.filesystem.inode import TREE +from antarest.storage.repository.filesystem.matrix.matrix import MatrixNode + + +MOCK_MATRIX_JSON = { + "index": ["1", "2"], + "columns": ["a", "b"], + "data": [[1, 2], [3, 4]], +} + + +MOCK_MATRIX_DTO = MatrixDTO( + freq=MatrixFreq.ANNUAL, + index=["1", "2"], + columns=["a", "b"], + data=[[1, 2], [3, 4]], +) + + +class MockMatrixNode(MatrixNode): + def __init__(self, context: ContextServer, config: StudyConfig) -> None: + super().__init__(config=config, context=context, freq="annual") + + def load( + self, + url: Optional[List[str]] = None, + depth: int = -1, + expanded: bool = False, + ) -> JSON: + return MOCK_MATRIX_JSON + + def _dump_json(self, data: JSON) -> None: + json.dump(data, self.config.path.open("w")) + + def build(self, config: StudyConfig) -> TREE: + pass # not used + + def check_errors( + self, data: str, url: Optional[List[str]] = None, raising: bool = False + ) -> List[str]: + pass # not used + + +def test_normalize(tmp_path: Path): + file = tmp_path / "matrix.txt" + file.touch() + + matrix_service = Mock() + matrix_service.create.return_value = "my-id" + + resolver = Mock() + resolver.build_matrix_uri.return_value = "matrix://my-id" + + node = MockMatrixNode( + context=ContextServer(matrix=matrix_service, resolver=resolver), + config=StudyConfig(study_path=file, study_id="mi-id"), + ) + + node.normalize() + assert node.get_link_path().read_text() == "matrix://my-id" + assert not file.exists() + matrix_service.create.assert_called_once_with(MOCK_MATRIX_DTO) + resolver.build_matrix_uri.assert_called_once_with("my-id") + + +def test_denormalize(tmp_path: Path): + file = tmp_path / "matrix.txt" + + link = file.parent / f"{file.name}.link" + link.write_text("my-id") + + resolver = Mock() + resolver.resolve.return_value = MOCK_MATRIX_JSON + + node = MockMatrixNode( + context=ContextServer(matrix=Mock(), resolver=resolver), + config=StudyConfig(study_path=file, study_id="mi-id"), + ) + + node.denormalize() + assert not link.exists() + assert json.loads(file.read_text()) == MOCK_MATRIX_JSON diff --git a/tests/storage/repository/filesystem/test_lazy_node.py b/tests/storage/repository/filesystem/test_lazy_node.py index 997af2ae81..107a627602 100644 --- a/tests/storage/repository/filesystem/test_lazy_node.py +++ b/tests/storage/repository/filesystem/test_lazy_node.py @@ -9,6 +9,12 @@ class MockLazyNode(LazyNode[str, str, str]): + def normalize(self) -> None: + pass # no external store in this node + + def denormalize(self) -> None: + pass # no external store in this node + def __init__(self, context: ContextServer, config: StudyConfig) -> None: super().__init__( config=config, diff --git a/tests/storage/repository/filesystem/utils.py b/tests/storage/repository/filesystem/utils.py index 1f63768334..12425f9d3f 100644 --- a/tests/storage/repository/filesystem/utils.py +++ b/tests/storage/repository/filesystem/utils.py @@ -9,6 +9,12 @@ class TestSubNode(INode[int, int, int]): + def normalize(self) -> None: + pass + + def denormalize(self) -> None: + pass + def build(self, config: StudyConfig) -> "TREE": pass