Issue 400 fix exporter service for studies with matrix link (#405)

antarest/storage/business/exporter_service.py

@@ -1,13 +1,20 @@
+import glob
+import logging
+import os
+import shutil
+import tempfile
+import time
 from io import BytesIO
 from pathlib import Path
+from zipfile import ZipFile, ZIP_DEFLATED
 
+from antarest.common.config import Config
 from antarest.storage.business.raw_study_service import RawStudyService
 from antarest.storage.model import Study
-from antarest.storage.repository.antares_io.exporter.export_file import (
-    Exporter,
-)
 from antarest.storage.repository.filesystem.factory import StudyFactory
 
+logger = logging.getLogger(__name__)
+
 
 class ExporterService:
     """
@@ -18,11 +25,11 @@ def __init__(
         self,
         study_service: RawStudyService,
         study_factory: StudyFactory,
-        exporter: Exporter,
+        config: Config,
     ):
         self.study_service = study_service
         self.study_factory = study_factory
-        self.exporter = exporter
+        self.config = config
 
     def export_study(
         self, metadata: Study, target: Path, outputs: bool = True
@@ -39,15 +46,60 @@ def export_study(
         """
         path_study = self.study_service.get_study_path(metadata)
 
-        self.study_service.check_study_exists(metadata)
-
-        return self.exporter.export_file(path_study, target, outputs)
+        return self.export_file(path_study, target, outputs)
 
     def export_study_flat(
         self, metadata: Study, dest: Path, outputs: bool = True
     ) -> None:
         path_study = self.study_service.get_study_path(metadata)
 
-        self.study_service.check_study_exists(metadata)
+        self.export_flat(path_study, dest, outputs)
+
+    def export_file(
+        self, path_study: Path, export_path: Path, outputs: bool = True
+    ) -> Path:
+        with tempfile.TemporaryDirectory(dir=self.config.tmp_dir) as tmpdir:
+            tmp_study_path = Path(tmpdir) / "tmp_copy"
+            self.export_flat(path_study, tmp_study_path, outputs)
+            start_time = time.time()
+            with ZipFile(export_path, "w", ZIP_DEFLATED) as zipf:
+                current_dir = os.getcwd()
+                os.chdir(tmp_study_path)
+
+                for path in glob.glob("**", recursive=True):
+                    if outputs or path.split(os.sep)[0] != "output":
+                        zipf.write(path, path)
 
-        self.exporter.export_flat(path_study, dest, outputs)
+                zipf.close()
+
+                os.chdir(current_dir)
+            duration = "{:.3f}".format(time.time() - start_time)
+            logger.info(
+                f"Study {path_study} exported (zipped mode) in {duration}s"
+            )
+        return export_path
+
+    def export_flat(
+        self,
+        path_study: Path,
+        dest: Path,
+        outputs: bool = False,
+    ) -> None:
+        start_time = time.time()
+        ignore_patterns = (
+            (
+                lambda directory, contents: ["output"]
+                if str(directory) == str(path_study)
+                else []
+            )
+            if not outputs
+            else None
+        )
+        shutil.copytree(src=path_study, dst=dest, ignore=ignore_patterns)
+        stop_time = time.time()
+        duration = "{:.3f}".format(stop_time - start_time)
+        logger.info(f"Study {path_study} exported (flat mode) in {duration}s")
+        _, study = self.study_factory.create_from_fs(dest, "")
+        study.denormalize()
+        duration = "{:.3f}".format(time.time() - stop_time)
+        logger.info(f"Study {path_study} denormalized in {duration}s")

antarest/storage/main.py

@@ -12,9 +12,6 @@
 from antarest.storage.business.raw_study_service import RawStudyService
 from antarest.storage.business.uri_resolver_service import UriResolverService
 from antarest.storage.business.watcher import Watcher
-from antarest.storage.repository.antares_io.exporter.export_file import (
-    Exporter,
-)
 from antarest.storage.repository.filesystem.factory import StudyFactory
 from antarest.storage.repository.patch_repository import PatchRepository
 from antarest.storage.repository.study import StudyMetadataRepository
@@ -30,7 +27,6 @@ def build_storage(
     user_service: LoginService,
     matrix_service: MatrixService,
     metadata_repository: Optional[StudyMetadataRepository] = None,
-    exporter: Optional[Exporter] = None,
     storage_service: Optional[StorageService] = None,
     patch_service: Optional[PatchService] = None,
     event_bus: IEventBus = DummyEventBusService(),
@@ -42,9 +38,8 @@ def build_storage(
         application: flask application
         config: server config
         user_service: user service facade
+        matrix_service: matrix store service
         metadata_repository: used by testing to inject mock. Let None to use true instantiation
-        study_factory: used by testing to inject mock. Let None to use true instantiation
-        exporter: used by testing to inject mock. Let None to use true instantiation
         storage_service: used by testing to inject mock. Let None to use true instantiation
         patch_service: used by testing to inject mock. Let None to use true instantiation
         event_bus: used by testing to inject mock. Let None to use true instantiation
@@ -57,7 +52,6 @@ def build_storage(
 
     resolver = UriResolverService(config, matrix_service=matrix_service)
     study_factory = StudyFactory(matrix=matrix_service, resolver=resolver)
-    exporter = exporter or Exporter()
     metadata_repository = metadata_repository or StudyMetadataRepository()
 
     patch_service = patch_service or PatchService(PatchRepository())
@@ -75,7 +69,7 @@ def build_storage(
     exporter_service = ExporterService(
         study_service=study_service,
         study_factory=study_factory,
-        exporter=exporter,
+        config=config,
     )
 
     storage_service = storage_service or StorageService(

antarest/storage/repository/filesystem/exceptions.py

@@ -0,0 +1,3 @@
+class DenormalizationException(Exception):
+    def __init__(self, msg: str):
+        super(DenormalizationException, self).__init__(msg)

antarest/storage/repository/filesystem/folder_node.py

@@ -117,6 +117,14 @@ def check_errors(
                     )
             return errors
 
+    def normalize(self) -> None:
+        for child in self.build(self.config).values():
+            child.normalize()
+
+    def denormalize(self) -> None:
+        for child in self.build(self.config).values():
+            child.denormalize()
+
     def extract_child(
         self, children: TREE, url: List[str]
     ) -> Tuple[List[str], List[str]]:

antarest/storage/repository/filesystem/ini_file_node.py

@@ -87,6 +87,12 @@ def check_errors(
 
         return errors
 
+    def normalize(self) -> None:
+        pass  # no external store in this node
+
+    def denormalize(self) -> None:
+        pass  # no external store in this node
+
     def _validate_param(
         self,
         section: str,

antarest/storage/repository/filesystem/inode.py

@@ -23,7 +23,7 @@ def build(self, config: StudyConfig) -> "TREE":
         Returns: children of current node
 
         """
-        pass
+        raise NotImplementedError()
 
     @abstractmethod
     def get(
@@ -43,7 +43,7 @@ def get(
         Returns: json
 
         """
-        pass
+        raise NotImplementedError()
 
     @abstractmethod
     def save(self, data: S, url: Optional[List[str]] = None) -> None:
@@ -57,7 +57,7 @@ def save(self, data: S, url: Optional[List[str]] = None) -> None:
         Returns:
 
         """
-        pass
+        raise NotImplementedError()
 
     @abstractmethod
     def check_errors(
@@ -73,7 +73,25 @@ def check_errors(
         Returns: list of errors belongs to this node or children
 
         """
-        pass
+        raise NotImplementedError()
+
+    @abstractmethod
+    def normalize(self) -> None:
+        """
+        Scan tree to send matrix in matrix store and replace by its links
+        Returns:
+
+        """
+        raise NotImplementedError()
+
+    @abstractmethod
+    def denormalize(self) -> None:
+        """
+        Scan tree to fetch matrix by its links
+        Returns:
+
+        """
+        raise NotImplementedError()
 
     def _assert_url_end(self, url: Optional[List[str]] = None) -> None:
         """

antarest/storage/repository/filesystem/matrix/matrix.py

@@ -8,6 +8,9 @@
 from antarest.matrixstore.model import MatrixDTO, MatrixFreq
 from antarest.storage.repository.filesystem.config.model import StudyConfig
 from antarest.storage.repository.filesystem.context import ContextServer
+from antarest.storage.repository.filesystem.exceptions import (
+    DenormalizationException,
+)
 from antarest.storage.repository.filesystem.lazy_node import LazyNode
 
 
@@ -26,6 +29,38 @@ def get_lazy_content(
     ) -> str:
         return f"matrixfile://{self.config.path.name}"
 
+    def normalize(self) -> None:
+        if self.get_link_path().exists():
+            return
+
+        matrix = self.load()
+        dto = MatrixDTO(
+            freq=MatrixFreq.from_str(self.freq),
+            index=matrix["index"],
+            columns=matrix["columns"],
+            data=matrix["data"],
+        )
+
+        uuid = self.context.matrix.create(dto)
+        self.get_link_path().write_text(
+            self.context.resolver.build_matrix_uri(uuid)
+        )
+        self.config.path.unlink()
+
+    def denormalize(self) -> None:
+        if self.config.path.exists():
+            return
+
+        uuid = self.get_link_path().read_text()
+        matrix = self.context.resolver.resolve(uuid)
+        if not matrix or not isinstance(matrix, dict):
+            raise DenormalizationException(
+                f"Failed to retrieve original matrix for {self.config.path}"
+            )
+
+        self.dump(matrix)
+        self.get_link_path().unlink()
+
     @abstractmethod
     def load(
         self,

antarest/storage/repository/filesystem/raw_file_node.py

@@ -46,3 +46,9 @@ def check_errors(
                 raise ValueError(msg)
             return [msg]
         return []
+
+    def normalize(self) -> None:
+        pass  # no external store in this node
+
+    def denormalize(self) -> None:
+        pass  # no external store in this node

antarest/storage/repository/filesystem/root/input/areas/list.py

@@ -6,6 +6,12 @@
 
 
 class InputAreasList(INode[List[str], List[str], List[str]]):
+    def normalize(self) -> None:
+        pass  # no external store in this node
+
+    def denormalize(self) -> None:
+        pass  # no external store in this node
+
     def __init__(self, context: ContextServer, config: StudyConfig):
         self.config = config
         self.context = context

antarest/storage/web/studies_blueprint.py

@@ -10,7 +10,7 @@
 from starlette.responses import StreamingResponse, Response, FileResponse
 
 from antarest.common.config import Config
-from antarest.common.custom_types import JSON
+from antarest.common.custom_types import JSON, SUB_JSON
 from antarest.common.jwt import JWTUser
 from antarest.common.requests import (
     RequestParameters,