Merge branch 'dev' into feat/store-matrices-as-hdf5

antarest/study/service.py

@@ -141,6 +141,11 @@
 from antarest.study.storage.rawstudy.model.filesystem.matrix.matrix import MatrixFrequency
 from antarest.study.storage.rawstudy.model.filesystem.matrix.output_series_matrix import OutputSeriesMatrix
 from antarest.study.storage.rawstudy.model.filesystem.raw_file_node import RawFileNode
+from antarest.study.storage.rawstudy.model.filesystem.root.output.simulation.mode.mcall.digest import (
+    DigestSynthesis,
+    DigestUI,
+)
+from antarest.study.storage.rawstudy.model.filesystem.root.user.user import User
 from antarest.study.storage.rawstudy.raw_study_service import RawStudyService
 from antarest.study.storage.storage_service import StudyStorageService
 from antarest.study.storage.study_download_utils import StudyDownloader, get_output_variables_information
@@ -2841,3 +2846,15 @@ def _alter_user_folder(
         cache_id = f"{CacheConstants.RAW_STUDY}/{study.id}"
         updated_tree = file_study.tree.get()
         self.storage_service.get_storage(study).cache.put(cache_id, updated_tree)  # type: ignore
+
+    def get_digest_file(self, study_id: str, output_id: str, params: RequestParameters) -> DigestUI:
+        """
+        Returns the digest file as 4 separated intelligible matrices.
+        Raises ChildNotFoundError if the output_id doesn't exist or if the digest file wasn't generated
+        """
+        study = self.get_study(study_id)
+        assert_permission(params.user, study, StudyPermissionType.READ)
+        file_study = self.storage_service.get_storage(study).get_raw(study)
+        digest_node = file_study.tree.get_node(url=["output", output_id, "economy", "mc-all", "grid", "digest"])
+        assert isinstance(digest_node, DigestSynthesis)
+        return digest_node.get_ui()

antarest/study/storage/rawstudy/model/filesystem/root/output/simulation/mode/mcall/digest.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+import typing as t
+
+import pandas as pd
+from pydantic import Field
+from typing_extensions import override
+
+from antarest.core.model import JSON
+from antarest.core.serialization import AntaresBaseModel
+from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
+from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
+from antarest.study.storage.rawstudy.model.filesystem.root.output.simulation.mode.mcall.synthesis import OutputSynthesis
+
+
+class DigestMatrixUI(AntaresBaseModel):
+    columns: t.List[t.Union[str, t.List[str]]]
+    data: t.List[t.List[str]]
+    grouped_columns: bool = Field(alias="groupedColumns")
+
+
+class DigestUI(AntaresBaseModel):
+    area: DigestMatrixUI
+    districts: DigestMatrixUI
+    flow_linear: DigestMatrixUI = Field(alias="flowLinear")
+    flow_quadratic: DigestMatrixUI = Field(alias="flowQuadratic")
+
+
+def _get_flow_linear(df: pd.DataFrame) -> DigestMatrixUI:
+    return _get_flow(df, "Links (FLOW LIN.)")
+
+
+def _get_flow_quadratic(df: pd.DataFrame) -> DigestMatrixUI:
+    return _get_flow(df, "Links (FLOW QUAD.)")
+
+
+def _get_flow(df: pd.DataFrame, keyword: str) -> DigestMatrixUI:
+    first_column = df["1"].tolist()
+    index = next((k for k, v in enumerate(first_column) if v == keyword), None)
+    if not index:
+        return DigestMatrixUI(columns=[], data=[], groupedColumns=False)
+    index_start = index + 2
+    df_col_start = 1
+    df_size = next((k for k, v in enumerate(first_column[index_start:]) if v == ""), len(first_column) - index_start)
+    flow_df = df.iloc[index_start : index_start + df_size, df_col_start : df_col_start + df_size]
+    data = flow_df.iloc[1:, :].to_numpy().tolist()
+    cols = [""] + flow_df.iloc[0, 1:].tolist()
+    return DigestMatrixUI(columns=cols, data=data, groupedColumns=False)
+
+
+def _build_areas_and_districts(df: pd.DataFrame, first_row: int) -> DigestMatrixUI:
+    first_column = df["1"].tolist()
+    first_area_row = df.iloc[first_row, 2:].tolist()
+    col_number = next((k for k, v in enumerate(first_area_row) if v == ""), df.shape[1])
+    final_index = first_column[first_row:].index("") + first_row
+    data = df.iloc[first_row:final_index, 1 : col_number + 1].to_numpy().tolist()
+    cols_raw = df.iloc[first_row - 3 : first_row, 2 : col_number + 1].to_numpy().tolist()
+    columns = [[""]] + [[a, b, c] for a, b, c in zip(cols_raw[0], cols_raw[1], cols_raw[2])]
+    return DigestMatrixUI(columns=columns, data=data, groupedColumns=True)
+
+
+def _get_area(df: pd.DataFrame) -> DigestMatrixUI:
+    return _build_areas_and_districts(df, 7)
+
+
+def _get_district(df: pd.DataFrame) -> DigestMatrixUI:
+    first_column = df["1"].tolist()
+    first_row = next((k for k, v in enumerate(first_column) if "@" in v), None)
+    if not first_row:
+        return DigestMatrixUI(columns=[], data=[], groupedColumns=False)
+    return _build_areas_and_districts(df, first_row)
+
+
+class DigestSynthesis(OutputSynthesis):
+    def __init__(self, context: ContextServer, config: FileStudyTreeConfig):
+        super().__init__(context, config)
+
+    @override
+    def load(
+        self,
+        url: t.Optional[t.List[str]] = None,
+        depth: int = -1,
+        expanded: bool = False,
+        formatted: bool = True,
+    ) -> JSON:
+        df = self._parse_digest_file()
+
+        output = df.to_dict(orient="split")
+        del output["index"]
+        return t.cast(JSON, output)
+
+    def get_ui(self) -> DigestUI:
+        """
+        Parse a digest file and returns it as 4 separated matrices.
+        One for areas, one for the districts, one for linear flow and the last one for quadratic flow.
+        """
+        df = self._parse_digest_file()
+        flow_linear = _get_flow_linear(df)
+        flow_quadratic = _get_flow_quadratic(df)
+        area = _get_area(df)
+        districts = _get_district(df)
+        return DigestUI(area=area, districts=districts, flowLinear=flow_linear, flowQuadratic=flow_quadratic)
+
+    def _parse_digest_file(self) -> pd.DataFrame:
+        """
+        Parse a digest file as a whole and return a single DataFrame.
+
+        The `digest.txt` file is a TSV file containing synthetic results of the simulation.
+        This file contains several data tables, each being separated by empty lines
+        and preceded by a header describing the nature and dimensions of the table.
+
+        Note that rows in the file may have different number of columns.
+        """
+        with open(self.config.path, "r") as digest_file:
+            # Reads the file and find the maximum number of columns in any row
+            data = [row.split("\t") for row in digest_file.read().splitlines()]
+            max_cols = max(len(row) for row in data)
+
+            # Adjust the number of columns in each row
+            data = [row + [""] * (max_cols - len(row)) for row in data]
+
+            # Returns a DataFrame from the data (do not convert values to float)
+            df = pd.DataFrame(data=data, columns=[str(i) for i in range(max_cols)], dtype=object)
+            return df

antarest/study/storage/rawstudy/model/filesystem/root/output/simulation/mode/mcall/grid.py

@@ -22,6 +22,7 @@
 from antarest.study.storage.rawstudy.model.filesystem.folder_node import FolderNode
 from antarest.study.storage.rawstudy.model.filesystem.inode import TREE
 from antarest.study.storage.rawstudy.model.filesystem.lazy_node import LazyNode
+from antarest.study.storage.rawstudy.model.filesystem.root.output.simulation.mode.mcall.digest import DigestSynthesis
 
 
 class OutputSimulationModeMcAllGrid(FolderNode):
@@ -83,47 +84,3 @@ def normalize(self) -> None:
     @override
     def denormalize(self) -> None:
         pass  # shouldn't be denormalized as it's an output file
-
-
-class DigestSynthesis(OutputSynthesis):
-    def __init__(self, context: ContextServer, config: FileStudyTreeConfig):
-        super().__init__(context, config)
-
-    @override
-    def load(
-        self,
-        url: t.Optional[t.List[str]] = None,
-        depth: int = -1,
-        expanded: bool = False,
-        formatted: bool = True,
-    ) -> JSON:
-        file_path = self.config.path
-        with open(file_path, "r") as f:
-            df = _parse_digest_file(f)
-
-        df.fillna("", inplace=True)  # replace NaN values for the front-end
-        output = df.to_dict(orient="split")
-        del output["index"]
-        return t.cast(JSON, output)
-
-
-def _parse_digest_file(digest_file: t.TextIO) -> pd.DataFrame:
-    """
-    Parse a digest file as a whole and return a single DataFrame.
-
-    The `digest.txt` file is a TSV file containing synthetic results of the simulation.
-    This file contains several data tables, each being separated by empty lines
-    and preceded by a header describing the nature and dimensions of the table.
-
-    Note that rows in the file may have different number of columns.
-    """
-
-    # Reads the file and find the maximum number of columns in any row
-    data = [row.split("\t") for row in digest_file.read().splitlines()]
-    max_cols = max(len(row) for row in data)
-
-    # Adjust the number of columns in each row
-    data = [row + [""] * (max_cols - len(row)) for row in data]
-
-    # Returns a DataFrame from the data (do not convert values to float)
-    return pd.DataFrame(data=data, columns=[str(i) for i in range(max_cols)], dtype=object)

antarest/study/storage/rawstudy/model/filesystem/root/output/simulation/mode/mcall/synthesis.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+import typing as t
+
+import pandas as pd
+from typing_extensions import override
+
+from antarest.core.exceptions import MustNotModifyOutputException
+from antarest.core.model import JSON
+from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfig
+from antarest.study.storage.rawstudy.model.filesystem.context import ContextServer
+from antarest.study.storage.rawstudy.model.filesystem.lazy_node import LazyNode
+
+
+class OutputSynthesis(LazyNode[JSON, bytes, bytes]):
+    def __init__(self, context: ContextServer, config: FileStudyTreeConfig):
+        super().__init__(context, config)
+
+    @override
+    def get_lazy_content(
+        self,
+        url: t.Optional[t.List[str]] = None,
+        depth: int = -1,
+        expanded: bool = False,
+    ) -> str:
+        return f"matrix://{self.config.path.name}"  # prefix used by the front to parse the back-end response
+
+    @override
+    def load(
+        self,
+        url: t.Optional[t.List[str]] = None,
+        depth: int = -1,
+        expanded: bool = False,
+        formatted: bool = True,
+    ) -> JSON:
+        file_path = self.config.path
+        df = pd.read_csv(file_path, sep="\t")
+        df.fillna("", inplace=True)  # replace NaN values for the front-end
+        output = df.to_dict(orient="split")
+        del output["index"]
+        return t.cast(JSON, output)
+
+    @override
+    def dump(self, data: bytes, url: t.Optional[t.List[str]] = None) -> None:
+        raise MustNotModifyOutputException(self.config.path.name)
+
+    @override
+    def check_errors(self, data: str, url: t.Optional[t.List[str]] = None, raising: bool = False) -> t.List[str]:
+        if not self.config.path.exists():
+            msg = f"{self.config.path} not exist"
+            if raising:
+                raise ValueError(msg)
+            return [msg]
+        return []
+
+    @override
+    def normalize(self) -> None:
+        pass  # shouldn't be normalized as it's an output file
+
+    @override
+    def denormalize(self) -> None:
+        pass  # shouldn't be denormalized as it's an output file

antarest/study/web/studies_blueprint.py

@@ -43,6 +43,7 @@
 from antarest.study.repository import AccessPermissions, StudyFilter, StudyPagination, StudySortBy
 from antarest.study.service import StudyService
 from antarest.study.storage.rawstudy.model.filesystem.config.model import FileStudyTreeConfigDTO
+from antarest.study.storage.rawstudy.model.filesystem.root.output.simulation.mode.mcall.digest import DigestUI
 
 logger = logging.getLogger(__name__)
 
@@ -815,6 +816,26 @@ def unarchive_output(
         )
         return content
 
+    @bp.get(
+        "/private/studies/{study_id}/outputs/{output_id}/digest-ui",
+        tags=[APITag.study_outputs],
+        summary="Display an output digest file for the front-end",
+        response_model=DigestUI,
+    )
+    def get_digest_file(
+        study_id: str,
+        output_id: str,
+        current_user: JWTUser = Depends(auth.get_current_user),
+    ) -> DigestUI:
+        study_id = sanitize_uuid(study_id)
+        output_id = sanitize_string(output_id)
+        logger.info(
+            f"Retrieving the digest file for the output {output_id} of the study {study_id}",
+            extra={"user": current_user.id},
+        )
+        params = RequestParameters(user=current_user)
+        return study_service.get_digest_file(study_id, output_id, params)
+
     @bp.get(
         "/studies/{study_id}/outputs",
         summary="Get global information about a study simulation result",

tests/integration/studies_blueprint/test_digest.py

@@ -0,0 +1,59 @@
+# Copyright (c) 2025, RTE (https://www.rte-france.com)
+#
+# See AUTHORS.txt
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# SPDX-License-Identifier: MPL-2.0
+#
+# This file is part of the Antares project.
+
+from starlette.testclient import TestClient
+
+
+class TestDigest:
+    def test_get_digest_endpoint(self, client: TestClient, user_access_token: str, internal_study_id: str) -> None:
+        client.headers = {"Authorization": f"Bearer {user_access_token}"}
+
+        # Nominal case
+        output_id = "20201014-1422eco-hello"
+        res = client.get(f"/v1/private/studies/{internal_study_id}/outputs/{output_id}/digest-ui")
+        assert res.status_code == 200
+        digest = res.json()
+        assert list(digest.keys()) == ["area", "districts", "flowLinear", "flowQuadratic"]
+        assert digest["districts"] == {"columns": [], "data": [], "groupedColumns": False}
+        flow = {
+            "columns": ["", "de", "es", "fr", "it"],
+            "data": [
+                ["de", "X", "--", "0", "--"],
+                ["es", "--", "X", "0", "--"],
+                ["fr", "0", "0", "X", "0"],
+                ["it", "--", "--", "0", "X"],
+            ],
+            "groupedColumns": False,
+        }
+        assert digest["flowQuadratic"] == flow
+        assert digest["flowLinear"] == flow
+        area_matrix = digest["area"]
+        assert area_matrix["groupedColumns"] is True
+        assert area_matrix["columns"][:3] == [[""], ["OV. COST", "Euro", "EXP"], ["OP. COST", "Euro", "EXP"]]
+
+        # Asserts we have a 404 Exception when the output doesn't exist
+        fake_output = "fake_output"
+        res = client.get(f"/v1/private/studies/{internal_study_id}/outputs/{fake_output}/digest-ui")
+        assert res.status_code == 404
+        assert res.json() == {
+            "description": f"'{fake_output}' not a child of Output",
+            "exception": "ChildNotFoundError",
+        }
+
+        # Asserts we have a 404 Exception when the digest file doesn't exist
+        output_wo_digest = "20201014-1430adq"
+        res = client.get(f"/v1/private/studies/{internal_study_id}/outputs/{output_wo_digest}/digest-ui")
+        assert res.status_code == 404
+        assert res.json() == {
+            "description": "'economy' not a child of OutputSimulation",
+            "exception": "ChildNotFoundError",
+        }

webapp/src/components/App/Singlestudy/HomeView/InformationView/LauncherHistory/JobStepper.tsx

@@ -40,8 +40,8 @@ import {
 } from "./style";
 import ConfirmationDialog from "../../../../../common/dialogs/ConfirmationDialog";
 import LinearProgressWithLabel from "../../../../../common/LinearProgressWithLabel";
-import DigestDialog from "../../../../../common/dialogs/DigestDialog";
 import type { EmptyObject } from "../../../../../../utils/tsUtils";
+import DigestDialog from "@/components/common/dialogs/DigestDialog";
 
 export const ColorStatus = {
   running: "warning.main",