diff --git a/Dockerfile b/Dockerfile index affdc86660..975ef9b776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM python:3.8-slim-bullseye # RUN apt update && apt install -y procps gdb +# Add the `ls` alias to simplify debugging +RUN echo "alias ll='/bin/ls -l --color=auto'" >> /root/.bashrc + ENV ANTAREST_CONF /resources/application.yaml RUN mkdir -p examples/studies diff --git a/antarest/__init__.py b/antarest/__init__.py index 6a92596e63..136deb4f14 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -7,9 +7,9 @@ # Standard project metadata -__version__ = "2.16.2" +__version__ = "2.16.3" __author__ = "RTE, Antares Web Team" -__date__ = "2024-01-10" +__date__ = "2024-01-17" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py new file mode 100644 index 0000000000..3b15ebd03a --- /dev/null +++ b/antarest/core/filesystem_blueprint.py @@ -0,0 +1,531 @@ +""" +Filesystem Blueprint +""" +import asyncio +import datetime +import os +import shutil +import stat +import typing as t +from pathlib import Path + +import typing_extensions as te +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Extra, Field +from starlette.responses import PlainTextResponse, StreamingResponse + +from antarest.core.config import Config +from antarest.core.jwt import JWTUser +from antarest.core.utils.web import APITag +from antarest.login.auth import Auth + +FilesystemName = te.Annotated[str, Field(regex=r"^\w+$", description="Filesystem name")] +MountPointName = te.Annotated[str, Field(regex=r"^\w+$", description="Mount point name")] + + +class FilesystemDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "name": "ws", + "mount_dirs": { + "default": "/path/to/workspaces/internal_studies", + "common": "/path/to/workspaces/common_studies", + }, + }, + }, +): + """ + Filesystem information. + + Attributes: + + - `name`: name of the filesystem. + - `mount_dirs`: mapping of the mount point names to their full path in Antares Web Server. + """ + + name: FilesystemName + mount_dirs: t.Mapping[str, Path] = Field(description="Full path of the mount points in Antares Web Server") + + +class MountPointDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "name": "default", + "path": "/path/to/workspaces/internal_studies", + "total_bytes": 1e9, + "used_bytes": 0.6e9, + "free_bytes": 1e9 - 0.6e9, + "message": f"{0.6e9 / 1e9:%} used", + }, + }, +): + """ + Disk usage information of a filesystem. + + Attributes: + + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + """ + + name: MountPointName + path: Path = Field(description="Full path of the mount point in Antares Web Server") + total_bytes: int = Field(0, description="Total size of the mount point in bytes") + used_bytes: int = Field(0, description="Used size of the mount point in bytes") + free_bytes: int = Field(0, description="Free size of the mount point in bytes") + message: str = Field("", description="A message describing the status of the mount point") + + @classmethod + async def from_path(cls, name: str, path: Path) -> "MountPointDTO": + obj = cls(name=name, path=path) + try: + obj.total_bytes, obj.used_bytes, obj.free_bytes = shutil.disk_usage(obj.path) + except OSError as exc: + # Avoid raising an exception if the file doesn't exist + # or if the mount point is not available. + obj.message = f"N/A: {exc}" + else: + obj.message = f"{obj.used_bytes / obj.total_bytes:.1%} used" + return obj + + +class FileInfoDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "path": "/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares", + "file_type": "file", + "file_count": 1, + "size_bytes": 126, + "created": "2023-12-07T17:59:43", + "modified": "2023-12-07T17:59:43", + "accessed": "2024-01-11T17:54:09", + "message": "OK", + }, + }, +): + """ + File information of a file or directory. + + Attributes: + + - `path`: full path of the file or directory in Antares Web Server. + - `file_type`: file type: "directory", "file", "symlink", "socket", "block_device", + "character_device", "fifo", or "unknown". + - `file_count`: number of files and folders in the directory (1 for files). + - `size_bytes`: size of the file or total size of the directory in bytes. + - `created`: creation date of the file or directory (local time). + - `modified`: last modification date of the file or directory (local time). + - `accessed`: last access date of the file or directory (local time). + - `message`: a message describing the status of the file. + """ + + path: Path = Field(description="Full path of the file or directory in Antares Web Server") + file_type: str = Field(description="Type of the file or directory") + file_count: int = Field(1, description="Number of files and folders in the directory (1 for files)") + size_bytes: int = Field(0, description="Size of the file or total size of the directory in bytes") + created: datetime.datetime = Field(description="Creation date of the file or directory (local time)") + modified: datetime.datetime = Field(description="Last modification date of the file or directory (local time)") + accessed: datetime.datetime = Field(description="Last access date of the file or directory (local time)") + message: str = Field("OK", description="A message describing the status of the file") + + @classmethod + async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfoDTO": + try: + file_stat = full_path.stat() + except OSError as exc: + # Avoid raising an exception if the file doesn't exist + # or if the mount point is not available. + return cls( + path=full_path, + file_type="unknown", + file_count=0, # missing + created=datetime.datetime.min, + modified=datetime.datetime.min, + accessed=datetime.datetime.min, + message=f"N/A: {exc}", + ) + + obj = cls( + path=full_path, + file_type="unknown", + file_count=1, + size_bytes=file_stat.st_size, + created=datetime.datetime.fromtimestamp(file_stat.st_ctime), + modified=datetime.datetime.fromtimestamp(file_stat.st_mtime), + accessed=datetime.datetime.fromtimestamp(file_stat.st_atime), + ) + + if stat.S_ISDIR(file_stat.st_mode): + obj.file_type = "directory" + if details: + file_count, disk_space = await _calc_details(full_path) + obj.file_count = file_count + obj.size_bytes = disk_space + elif stat.S_ISREG(file_stat.st_mode): + obj.file_type = "file" + elif stat.S_ISLNK(file_stat.st_mode): # pragma: no cover + obj.file_type = "symlink" + elif stat.S_ISSOCK(file_stat.st_mode): # pragma: no cover + obj.file_type = "socket" + elif stat.S_ISBLK(file_stat.st_mode): # pragma: no cover + obj.file_type = "block_device" + elif stat.S_ISCHR(file_stat.st_mode): # pragma: no cover + obj.file_type = "character_device" + elif stat.S_ISFIFO(file_stat.st_mode): # pragma: no cover + obj.file_type = "fifo" + else: # pragma: no cover + obj.file_type = "unknown" + + return obj + + +async def _calc_details(full_path: t.Union[str, Path]) -> t.Tuple[int, int]: + """Calculate the number of files and the total size of a directory recursively.""" + + full_path = Path(full_path) + file_stat = full_path.stat() + file_count = 1 + total_size = file_stat.st_size + + if stat.S_ISDIR(file_stat.st_mode): + for entry in os.scandir(full_path): + sub_file_count, sub_total_size = await _calc_details(entry.path) + file_count += sub_file_count + total_size += sub_total_size + + return file_count, total_size + + +def _is_relative_to(path: Path, base_path: Path) -> bool: + """Check if the given path is relative to the specified base path.""" + try: + path = Path(path).resolve() + base_path = Path(base_path).resolve() + return bool(path.relative_to(base_path)) + except ValueError: + return False + + +def create_file_system_blueprint(config: Config) -> APIRouter: + """ + Create the blueprint for the file system API. + + This blueprint is used to diagnose the disk space consumption of the different + workspaces (especially the "default" workspace of Antares Web), and the different + storage directories (`tmp`, `matrixstore`, `archive`, etc. defined in the configuration file). + + This blueprint is also used to list files and directories, and to view or download a file. + + Reading files is allowed for authenticated users, but deleting files is reserved + for site administrators. + + Args: + config: Application configuration. + + Returns: + The blueprint. + """ + auth = Auth(config) + bp = APIRouter( + prefix="/v1/filesystem", + tags=[APITag.filesystem], + dependencies=[Depends(auth.get_current_user)], + include_in_schema=True, # but may be disabled in the future + ) + config_dirs = { + "res": config.resources_path, + "tmp": config.storage.tmp_dir, + "matrix": config.storage.matrixstore, + "archive": config.storage.archive_dir, + } + workspace_dirs = {name: ws_cfg.path for name, ws_cfg in config.storage.workspaces.items()} + filesystems = { + "cfg": config_dirs, + "ws": workspace_dirs, + } + + # Utility functions + # ================= + + def _get_mount_dirs(fs: str) -> t.Mapping[str, Path]: + try: + return filesystems[fs] + except KeyError: + raise HTTPException(status_code=404, detail=f"Filesystem not found: '{fs}'") from None + + def _get_mount_dir(fs: str, mount: str) -> Path: + try: + return filesystems[fs][mount] + except KeyError: + raise HTTPException(status_code=404, detail=f"Mount point not found: '{fs}/{mount}'") from None + + def _get_full_path(mount_dir: Path, path: str) -> Path: + if not path: + raise HTTPException(status_code=400, detail="Empty or missing path parameter") + full_path = (mount_dir / path).resolve() + if not _is_relative_to(full_path, mount_dir): + raise HTTPException(status_code=403, detail=f"Access denied to path: '{path}'") + if not full_path.exists(): + raise HTTPException(status_code=404, detail=f"Path not found: '{path}'") + return full_path + + # Endpoints + # ========= + + @bp.get( + "", + summary="Get filesystems information", + response_model=t.Sequence[FilesystemDTO], + ) + async def list_filesystems() -> t.Sequence[FilesystemDTO]: + """ + Get the list of filesystems and their mount points. + + Returns: + - `name`: name of the filesystem: "cfg" or "ws". + - `mount_dirs`: mapping of the mount point names to their full path in Antares Web Server. + """ + + fs = [FilesystemDTO(name=name, mount_dirs=mount_dirs) for name, mount_dirs in filesystems.items()] + return fs + + @bp.get( + "/{fs}", + summary="Get information of a filesystem", + response_model=t.Sequence[MountPointDTO], + ) + async def list_mount_points(fs: FilesystemName) -> t.Sequence[MountPointDTO]: + """ + Get the path and the disk usage of the mount points in a filesystem. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + + Returns: + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + + Possible error codes: + - 404 Not Found: If the specified filesystem doesn't exist. + """ + + mount_dirs = _get_mount_dirs(fs) + tasks = [MountPointDTO.from_path(name, path) for name, path in mount_dirs.items()] + ws = await asyncio.gather(*tasks) + return ws + + @bp.get( + "/{fs}/{mount}", + summary="Get information of a mount point", + response_model=MountPointDTO, + ) + async def get_mount_point(fs: FilesystemName, mount: MountPointName) -> MountPointDTO: + """ + Get the path and the disk usage of a mount point. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + + Returns: + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + + Possible error codes: + - 404 Not Found: If the specified filesystem or mount point doesn't exist. + """ + + mount_dir = _get_mount_dir(fs, mount) + return await MountPointDTO.from_path(mount, mount_dir) + + @bp.get( + "/{fs}/{mount}/ls", + summary="List files in a mount point", + response_model=t.Sequence[FileInfoDTO], + ) + async def list_files( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + details: bool = False, + ) -> t.Sequence[FileInfoDTO]: + """ + List files and directories in a mount point. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the directory to list. + - `details`: If true, return the number of files and the total size of each directory. + + > **⚠ Warning:** Using a glob pattern for the `path` parameter (for instance, "**/study.antares") + > or using the `details` parameter can slow down the response time of this endpoint. + + Returns: + - `path`: full path of the file or directory in Antares Web Server. + This path can contain glob patterns (e.g., `*.txt`). + - `file_type`: file type: "directory", "file", "symlink", "socket", "block_device", + "character_device", "fifo", or "unknown". + - `file_count`: number of files of folders in the directory (1 for files). + - `size_bytes`: size of the file or total size of the directory in bytes. + - `created`: creation date of the file or directory (local time). + - `modified`: last modification date of the file or directory (local time). + - `accessed`: last access date of the file or directory (local time). + - `message`: a message describing the status of the file. + + Possible error codes: + - 400 Bad Request: If the specified path is invalid (e.g., contains invalid glob patterns). + - 404 Not Found: If the specified filesystem or mount point doesn't exist. + - 403 Forbidden: If the user has no permission to access the directory. + """ + + mount_dir = _get_mount_dir(fs, mount) + + # The following code looks weird, but it's the only way to handle exceptions in generators. + tasks = [] + iterator = mount_dir.glob(path) if path else mount_dir.iterdir() + while True: + try: + file_path = next(iterator) + except StopIteration: + break + except (OSError, ValueError, IndexError) as exc: + # Unacceptable pattern: invalid glob pattern or access denied + raise HTTPException(status_code=400, detail=f"Invalid path: '{path}'. {exc}") from exc + except NotImplementedError as exc: + # Unacceptable pattern: non-relative glob pattern + raise HTTPException(status_code=403, detail=f"Access denied to path: '{path}'. {exc}") from exc + else: + file_info = FileInfoDTO.from_path(file_path, details=details) + tasks.append(file_info) + + file_info_list = await asyncio.gather(*tasks) + return file_info_list + + @bp.get( + "/{fs}/{mount}/cat", + summary="View a text file from a mount point", + response_class=PlainTextResponse, + response_description="File content as text", + ) + async def view_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + encoding: str = "utf-8", + ) -> str: + # noinspection SpellCheckingInspection + """ + View a text file from a mount point. + + Examples: + ```ini + [antares] + version = 820 + caption = default + created = 1701964773.965127 + lastsave = 1701964773.965131 + author = John DOE + ``` + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file to view. + - `encoding`: The encoding of the file. Defaults to "utf-8". + + Returns: + - File content as text or binary. + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to view the file. + - 404 Not Found: If the specified filesystem, mount point or file doesn't exist. + - 417 Expectation Failed: If the specified path is not a text file or if the encoding is invalid. + """ + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + raise HTTPException(status_code=417, detail=f"Path is not a file: '{path}'") + + elif full_path.is_file(): + try: + return full_path.read_text(encoding=encoding) + except LookupError as exc: + raise HTTPException(status_code=417, detail=f"Unknown encoding: '{encoding}'") from exc + except UnicodeDecodeError as exc: + raise HTTPException(status_code=417, detail=f"Failed to decode file: '{path}'") from exc + + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + + @bp.get( + "/{fs}/{mount}/download", + summary="Download a file from a mount point", + response_class=StreamingResponse, + response_description="File content as binary", + ) + async def download_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + ) -> StreamingResponse: + """ + Download a file from a mount point. + + > **Note:** Directory download is not supported yet. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file or directory to download. + + Returns: + - File content as binary. + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to download the file. + - 404 Not Found: If the specified filesystem, mount point or file doesn't exist. + - 417 Expectation Failed: If the specified path is not a regular file. + """ + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + raise HTTPException(status_code=417, detail=f"Path is not a file: '{path}'") + + elif full_path.is_file(): + + def iter_file() -> t.Iterator[bytes]: + with full_path.open(mode="rb") as file: + yield from file + + headers = {"Content-Disposition": f"attachment; filename={full_path.name}"} + return StreamingResponse(iter_file(), media_type="application/octet-stream", headers=headers) + + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + + return bp diff --git a/antarest/core/utils/web.py b/antarest/core/utils/web.py index 69a530a2b4..2990b05c41 100644 --- a/antarest/core/utils/web.py +++ b/antarest/core/utils/web.py @@ -12,6 +12,7 @@ class APITag: matrix = "Manage Matrix" tasks = "Manage tasks" misc = "Miscellaneous" + filesystem = "Filesystem Management" tags_metadata = [ @@ -54,4 +55,7 @@ class APITag: { "name": APITag.misc, }, + { + "name": APITag.filesystem, + }, ] diff --git a/antarest/main.py b/antarest/main.py index 700947a4dd..93be38ed2b 100644 --- a/antarest/main.py +++ b/antarest/main.py @@ -27,6 +27,7 @@ from antarest import __version__ from antarest.core.config import Config from antarest.core.core_blueprint import create_utils_routes +from antarest.core.filesystem_blueprint import create_file_system_blueprint from antarest.core.logging.utils import LoggingMiddleware, configure_logger from antarest.core.requests import RATE_LIMIT_CONFIG from antarest.core.swagger import customize_openapi @@ -299,6 +300,7 @@ def get_config() -> JwtSettings: allow_headers=["*"], ) application.include_router(create_utils_routes(config)) + application.include_router(create_file_system_blueprint(config)) # noinspection PyUnusedLocal @application.exception_handler(HTTPException) diff --git a/antarest/study/business/thematic_trimming_management.py b/antarest/study/business/thematic_trimming_management.py index 317b52b39f..4259046701 100644 --- a/antarest/study/business/thematic_trimming_management.py +++ b/antarest/study/business/thematic_trimming_management.py @@ -64,8 +64,8 @@ class ThematicTrimmingFormFields(FormFieldsBaseModel, metaclass=AllOptionalMetac cong_fee_alg: bool cong_fee_abs: bool marg_cost: bool - cong_prod_plus: bool - cong_prod_minus: bool + cong_prob_plus: bool + cong_prob_minus: bool hurdle_cost: bool # For study versions >= 810 res_generation_by_plant: bool @@ -132,8 +132,8 @@ class ThematicTrimmingFormFields(FormFieldsBaseModel, metaclass=AllOptionalMetac "cong_fee_alg": {"path": "CONG. FEE (ALG.)", "default_value": True}, "cong_fee_abs": {"path": "CONG. FEE (ABS.)", "default_value": True}, "marg_cost": {"path": "MARG. COST", "default_value": True}, - "cong_prod_plus": {"path": "CONG. PROD +", "default_value": True}, - "cong_prod_minus": {"path": "CONG. PROD -", "default_value": True}, + "cong_prob_plus": {"path": "CONG. PROB +", "default_value": True}, + "cong_prob_minus": {"path": "CONG. PROB -", "default_value": True}, "hurdle_cost": {"path": "HURDLE COST", "default_value": True}, "res_generation_by_plant": {"path": "RES generation by plant", "default_value": True, "start_version": 810}, "misc_dtg_2": {"path": "MISC. DTG 2", "default_value": True, "start_version": 810}, diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index b84a9bf107..41e214d1ad 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -128,7 +128,7 @@ def get_study( detail=f"Invalid plain text configuration in path '{path}': {exc}", ) from None elif content_type: - headers = {"Content-Disposition": f"attachment; filename='{resource_path.name}'"} + headers = {"Content-Disposition": f"attachment; filename={resource_path.name}"} return StreamingResponse( io.BytesIO(output), media_type=content_type, diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 15fa0dedad..02a27f5a98 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,33 @@ Antares Web Changelog ===================== +v2.16.3 (2024-01-17) +-------------------- + +### Features + +* **api-filesystem:** add new API endpoints to manage filesystem and get disk usage [`#1895`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1895) +* **ui-district:** enhance Scenario Playlist loading and remove Regional District menu [`#1897`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1897) + + +### Bug Fixes + +* **config:** use "CONG. PROB" for thematic trimming (fix typo) [`#1893`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1893) +* **ui-debug:** correct debug view for JSON configuration [`#1898`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1898) +* **ui-debug:** correct debug view for textual matrices [`#1896`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1896) +* **ui-hydro:** add areas encoding to hydro tabs path [`#1894`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1894) + + +### Refactoring + +* **ui-debug:** code review [`#1892`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1892) + + +### Continuous Integration + +* **docker:** add the `ll` alias in `.bashrc` to simplify debugging [`#1880`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1880) + + v2.16.2 (2024-01-10) -------------------- diff --git a/setup.py b/setup.py index db663998a7..d3007244b0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="AntaREST", - version="2.16.2", + version="2.16.3", description="Antares Server", long_description=Path("README.md").read_text(encoding="utf-8"), long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index 770a971f0c..384c692e84 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.8 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.16.2 +sonar.projectVersion=2.16.3 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/* \ No newline at end of file diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a88dc4f9e8..0ac1e1763f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,7 +6,7 @@ import jinja2 import pytest from fastapi import FastAPI -from sqlalchemy import create_engine +from sqlalchemy import create_engine # type: ignore from starlette.testclient import TestClient from antarest.dbmodel import Base @@ -22,7 +22,7 @@ @pytest.fixture(name="app") -def app_fixture(tmp_path: Path) -> FastAPI: +def app_fixture(tmp_path: Path) -> t.Generator[FastAPI, None, None]: # Currently, it is impossible to use a SQLite database in memory (with "sqlite:///:memory:") # because the database is created by the FastAPI application during each integration test, # which doesn't apply the migrations (migrations are done by Alembic). @@ -96,7 +96,7 @@ def admin_access_token_fixture(client: TestClient) -> str: ) res.raise_for_status() credentials = res.json() - return credentials["access_token"] + return t.cast(str, credentials["access_token"]) @pytest.fixture(name="user_access_token") @@ -117,7 +117,7 @@ def user_access_token_fixture( ) res.raise_for_status() credentials = res.json() - return credentials["access_token"] + return t.cast(str, credentials["access_token"]) @pytest.fixture(name="study_id") @@ -131,5 +131,5 @@ def study_id_fixture( headers={"Authorization": f"Bearer {user_access_token}"}, ) res.raise_for_status() - study_ids = res.json() + study_ids = t.cast(t.Iterable[str], res.json()) return next(iter(study_ids)) diff --git a/tests/integration/filesystem_blueprint/__init__.py b/tests/integration/filesystem_blueprint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py new file mode 100644 index 0000000000..a547313896 --- /dev/null +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -0,0 +1,421 @@ +import datetime +import operator +import re +import shutil +import typing as t +from pathlib import Path + +from starlette.testclient import TestClient + +from tests.integration.conftest import RESOURCES_DIR + + +class AnyDiskUsagePercent: + """A helper object that compares equal to any disk usage percentage.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return bool(re.fullmatch(r"\d+(?:\.\d+)?% used", other)) + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + return "" + + +class AnyIsoDateTime: + """A helper object that compares equal to any date time in ISO 8601 format.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + try: + return bool(datetime.datetime.fromisoformat(other)) + except ValueError: + return False + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + return "" + + +class IntegerRange: + """A helper object that compares equal to any integer in a given range.""" + + def __init__(self, start: int, stop: int) -> None: + self.start = start + self.stop = stop + + def __eq__(self, other: object) -> bool: + if not isinstance(other, int): + return NotImplemented + return self.start <= other <= self.stop + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + start = getattr(self, "start", None) + stop = getattr(self, "stop", None) + return f"" + + +# noinspection SpellCheckingInspection +class TestFilesystemEndpoints: + """Test the filesystem endpoints.""" + + def test_lifecycle( + self, + tmp_path: Path, + caplog: t.Any, + client: TestClient, + user_access_token: str, + admin_access_token: str, + ) -> None: + """ + Test the lifecycle of the filesystem endpoints. + + Args: + tmp_path: pytest tmp_path fixture. + caplog: pytest caplog fixture. + client: test client (tests.integration.conftest.client_fixture). + user_access_token: access token of a classic user (tests.integration.conftest.user_access_token_fixture). + admin_access_token: access token of an admin user (tests.integration.conftest.admin_access_token_fixture). + """ + # NOTE: all the following paths are based on the configuration defined in the app_fixture. + archive_dir = tmp_path / "archive_dir" + matrix_dir = tmp_path / "matrix_store" + resource_dir = RESOURCES_DIR + tmp_dir = tmp_path / "tmp" + default_workspace = tmp_path / "internal_workspace" + ext_workspace_path = tmp_path / "ext_workspace" + + expected: t.Union[t.Sequence[t.Mapping[str, t.Any]], t.Mapping[str, t.Any]] + + with caplog.at_level(level="ERROR", logger="antarest.main"): + # Count the number of errors in the caplog + err_count = 0 + + # ================================================== + # Get the list of filesystems and their mount points + # ================================================== + + # Without authentication + res = client.get("/v1/filesystem") + assert res.status_code == 401, res.json() + assert res.json()["detail"] == "Missing cookie access_token_cookie" + # This error generates no log entry + # err_count += 1 + + # With authentication + user_headers = {"Authorization": f"Bearer {user_access_token}"} + res = client.get("/v1/filesystem", headers=user_headers) + assert res.status_code == 200, res.json() + actual = res.json() + expected = [ + { + "name": "cfg", + "mount_dirs": { + "archive": str(archive_dir), + "matrix": str(matrix_dir), + "res": str(resource_dir), + "tmp": str(tmp_dir), + }, + }, + { + "name": "ws", + "mount_dirs": { + "default": str(default_workspace), + "ext": str(ext_workspace_path), + }, + }, + ] + assert actual == expected + + # =================================================================== + # Get the path and the disk usage of the mount points in a filesystem + # =================================================================== + + # Unknown filesystem + res = client.get("/v1/filesystem/foo", headers=user_headers) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Filesystem not found: 'foo'" + err_count += 1 + + # Known filesystem + res = client.get("/v1/filesystem/ws", headers=user_headers) + assert res.status_code == 200, res.json() + actual = sorted(res.json(), key=operator.itemgetter("name")) + # Both mount point are in the same filesystem, which is the `tmp_path` filesystem + total_bytes, used_bytes, free_bytes = shutil.disk_usage(tmp_path) + expected = [ + { + "name": "default", + "path": str(default_workspace), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + }, + { + "name": "ext", + "path": str(ext_workspace_path), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + }, + ] + assert actual == expected + + # ================================================ + # Get the path and the disk usage of a mount point + # ================================================ + + # Unknown mount point + res = client.get("/v1/filesystem/ws/foo", headers=user_headers) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Mount point not found: 'ws/foo'" + err_count += 1 + + res = client.get("/v1/filesystem/ws/default", headers=user_headers) + assert res.status_code == 200, res.json() + actual = res.json() + expected = { + "name": "default", + "path": str(default_workspace), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + } + assert actual == expected + + # ========================================= + # List files and directories in a workspace + # ========================================= + + # Listing a workspace with and invalid glob pattern raises an error + res = client.get("/v1/filesystem/ws/default/ls", headers=user_headers, params={"path": "."}) + assert res.status_code == 400, res.json() + assert res.json()["description"].startswith("Invalid path: '.'"), res.json() + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/v1/filesystem/ws/ext/ls", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"].startswith("Access denied to path: '/foo'"), res.json() + err_count += 1 + + # Recursively search all "study.antares" files in the "default" workspace + res = client.get("/v1/filesystem/ws/default/ls", headers=user_headers, params={"path": "**/study.antares"}) + assert res.status_code == 200, res.json() + actual = res.json() + # There is no managed study in the "default" workspace + expected = [] + assert actual == expected + + # Recursively search all "study.antares" files in the "ext" workspace + res = client.get("/v1/filesystem/ws/ext/ls", headers=user_headers, params={"path": "**/study.antares"}) + assert res.status_code == 200, res.json() + actual = res.json() + # There is one external study in the "ext" workspace, which is "STA-mini" + expected = [ + { + "path": str(ext_workspace_path / "STA-mini" / "study.antares"), + "file_type": "file", + "file_count": 1, + "size_bytes": 112, + "created": AnyIsoDateTime(), + "accessed": AnyIsoDateTime(), + "modified": AnyIsoDateTime(), + "message": "OK", + } + ] + assert actual == expected + + # Get the details of the "STA-mini" study + res = client.get( + "/v1/filesystem/ws/ext/ls", + headers=user_headers, + params={"path": "STA-mini", "details": True}, # type: ignore + ) + assert res.status_code == 200, res.json() + actual = res.json() + expected = [ + { + "path": str(ext_workspace_path / "STA-mini"), + "file_type": "directory", + "file_count": IntegerRange(900, 1000), # 918 + "size_bytes": IntegerRange(7_000_000, 9_000_000), # nt: 7_741_619, posix: 8_597_683 + "created": AnyIsoDateTime(), + "accessed": AnyIsoDateTime(), + "modified": AnyIsoDateTime(), + "message": "OK", + } + ] + assert actual == expected + + # ================================= + # View a text file from a workspace + # ================================= + + # Providing an empty path is not allowed + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": ""}) + assert res.status_code == 400, res.json() + assert res.json()["description"] == "Empty or missing path parameter" + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "Access denied to path: '/foo'" + err_count += 1 + + # Providing a directory path is not allowed + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # Let's create a dummy file in the "ext" workspace, and write some text in it + dummy_file = ext_workspace_path / "dummy.txt" + dummy_file.write_text("Hello, world!") + + # Authorized users can view text files + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + assert res.status_code == 200, res.json() + assert res.text == "Hello, world!" + + # If the file is missing, a 404 error is returned + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "missing.txt"}) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Path not found: 'missing.txt'" + err_count += 1 + + # If the file is not a text file, a 417 error is returned + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # If the user choose an unknown encoding, a 417 error is returned + res = client.get( + "/v1/filesystem/ws/ext/cat", + headers=user_headers, + params={"path": "dummy.txt", "encoding": "unknown"}, + ) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Unknown encoding: 'unknown'" + err_count += 1 + + # If the file is not a texte file, a 417 error may be returned + dummy_file.write_bytes(b"\x81\x82\x83") # invalid utf-8 bytes + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Failed to decode file: 'dummy.txt'" + err_count += 1 + + # ================================ + # Download a file from a workspace + # ================================ + + # Providing an empty path is not allowed + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": ""}) + assert res.status_code == 400, res.json() + assert res.json()["description"] == "Empty or missing path parameter" + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "Access denied to path: '/foo'" + err_count += 1 + + # Providing a directory path is not allowed + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # Let's create a dummy file in the "ext" workspace, and write some binary data in it + dummy_file = ext_workspace_path / "dummy.bin" + dummy_file.write_bytes(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09") + + # Authorized users can download files + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "dummy.bin"}) + assert res.status_code == 200, res.json() + assert res.content == b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" + + # If the file is missing, a 404 error is returned + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "missing.bin"}) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Path not found: 'missing.bin'" + err_count += 1 + + # Downloading a directory is not allowed + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # At the end of this unit test, the caplog should have errors + assert len(caplog.records) == err_count, caplog.records + + def test_size_of_studies( + self, + client: TestClient, + user_access_token: str, + caplog: t.Any, + ): + """ + This test demonstrates how to compute the size of all studies. + + - First, we get the list of studies using the `/v1/studies` endpoint. + - Then, we get the size of each study using the `/v1/filesystem/ws/{workspace}/ls` endpoint, + with the `details` parameter set to `True`. + """ + user_headers = {"Authorization": f"Bearer {user_access_token}"} + + # For this demo, we can disable the logs + with caplog.at_level(level="CRITICAL", logger="antarest.main"): + # Create a new study in the "default" workspace for this demo + res = client.post( + "/v1/studies", + headers=user_headers, + params={"name": "New Study", "version": "860"}, + ) + res.raise_for_status() + + # Get the list of studies from all workspaces + res = client.get("/v1/studies", headers=user_headers) + res.raise_for_status() + actual = res.json() + + # Get the size of each study + sizes = [] + for study in actual.values(): + res = client.get( + f"/v1/filesystem/ws/{study['workspace']}/ls", + headers=user_headers, + params={"path": study["folder"], "details": True}, + ) + res.raise_for_status() + actual = res.json() + sizes.append(actual[0]["size_bytes"]) + + # Check the sizes + # The size of the new study should be between 140 and 300 KB. + # The suze of 'STA-mini' should be between 7 and 9 MB. + sizes.sort() + assert sizes == [IntegerRange(140_000, 300_000), IntegerRange(7_000_000, 9_000_000)] diff --git a/tests/integration/filesystem_blueprint/test_model.py b/tests/integration/filesystem_blueprint/test_model.py new file mode 100644 index 0000000000..3c21340363 --- /dev/null +++ b/tests/integration/filesystem_blueprint/test_model.py @@ -0,0 +1,145 @@ +import asyncio +import datetime +import re +import shutil +from pathlib import Path + +from antarest.core.filesystem_blueprint import FileInfoDTO, FilesystemDTO, MountPointDTO + + +class TestFilesystemDTO: + def test_init(self) -> None: + example = { + "name": "ws", + "mount_dirs": { + "default": "/path/to/workspaces/internal_studies", + "common": "/path/to/workspaces/common_studies", + }, + } + dto = FilesystemDTO.parse_obj(example) + assert dto.name == example["name"] + assert dto.mount_dirs["default"] == Path(example["mount_dirs"]["default"]) + assert dto.mount_dirs["common"] == Path(example["mount_dirs"]["common"]) + + +class TestMountPointDTO: + def test_init(self) -> None: + example = { + "name": "default", + "path": "/path/to/workspaces/internal_studies", + "total_bytes": 1e9, + "used_bytes": 0.6e9, + "free_bytes": 1e9 - 0.6e9, + "message": f"{0.6e9 / 1e9:%} used", + } + dto = MountPointDTO.parse_obj(example) + assert dto.name == example["name"] + assert dto.path == Path(example["path"]) + assert dto.total_bytes == example["total_bytes"] + assert dto.used_bytes == example["used_bytes"] + assert dto.free_bytes == example["free_bytes"] + assert dto.message == example["message"] + + def test_from_path__missing_file(self) -> None: + name = "foo" + path = Path("/path/to/workspaces/internal_studies") + dto = asyncio.run(MountPointDTO.from_path(name, path)) + assert dto.name == name + assert dto.path == path + assert dto.total_bytes == 0 + assert dto.used_bytes == 0 + assert dto.free_bytes == 0 + assert dto.message.startswith("N/A:"), dto.message + + def test_from_path__file(self, tmp_path: Path) -> None: + name = "foo" + dto = asyncio.run(MountPointDTO.from_path(name, tmp_path)) + total_bytes, used_bytes, free_bytes = shutil.disk_usage(tmp_path) + assert dto.name == name + assert dto.path == tmp_path + assert dto.total_bytes == total_bytes + assert dto.used_bytes == used_bytes + assert dto.free_bytes == free_bytes + assert re.fullmatch(r"\d+(?:\.\d+)?% used", dto.message), dto.message + + +class TestFileInfoDTO: + def test_init(self) -> None: + example = { + "path": "/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares", + "file_type": "file", + "file_count": 1, + "size_bytes": 126, + "created": "2023-12-07T17:59:43", + "modified": "2023-12-07T17:59:43", + "accessed": "2024-01-11T17:54:09", + "message": "OK", + } + dto = FileInfoDTO.parse_obj(example) + assert dto.path == Path(example["path"]) + assert dto.file_type == example["file_type"] + assert dto.file_count == example["file_count"] + assert dto.size_bytes == example["size_bytes"] + assert dto.created == datetime.datetime.fromisoformat(example["created"]) + assert dto.modified == datetime.datetime.fromisoformat(example["modified"]) + assert dto.accessed == datetime.datetime.fromisoformat(example["accessed"]) + assert dto.message == example["message"] + + def test_from_path__missing_file(self) -> None: + path = Path("/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares") + dto = asyncio.run(FileInfoDTO.from_path(path)) + assert dto.path == path + assert dto.file_type == "unknown" + assert dto.file_count == 0 + assert dto.size_bytes == 0 + assert dto.created == datetime.datetime.min + assert dto.modified == datetime.datetime.min + assert dto.accessed == datetime.datetime.min + assert dto.message.startswith("N/A:"), dto.message + + def test_from_path__file(self, tmp_path: Path) -> None: + path = tmp_path / "foo.txt" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.write_bytes(b"1234567") # 7 bytes + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path)) + assert dto.path == path + assert dto.file_type == "file" + assert dto.file_count == 1 + assert dto.size_bytes == 7 + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" + + def test_from_path__directory(self, tmp_path: Path) -> None: + path = tmp_path / "foo" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.mkdir() + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path, details=False)) + assert dto.path == path + assert dto.file_type == "directory" + assert dto.file_count == 1 + assert dto.size_bytes == path.stat().st_size + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" + + def test_from_path__directory_with_files(self, tmp_path: Path) -> None: + path = tmp_path / "foo" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.mkdir() + (path / "bar.txt").write_bytes(b"1234567") + (path / "baz.txt").write_bytes(b"890") + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path, details=True)) + assert dto.path == path + assert dto.file_type == "directory" + assert dto.file_count == 3 + assert dto.size_bytes == path.stat().st_size + 10 + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index ed4f1ed8af..644713c535 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -921,8 +921,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, @@ -990,8 +990,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, @@ -1060,8 +1060,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, diff --git a/webapp/package-lock.json b/webapp/package-lock.json index fdeb700d83..0258965201 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "dependencies": { "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", diff --git a/webapp/package.json b/webapp/package.json index 1fd7eea24d..76c1838ef0 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "private": true, "engines": { "node": "18.16.1" diff --git a/webapp/src/common/types.ts b/webapp/src/common/types.ts index 7687987add..bc78b42d93 100644 --- a/webapp/src/common/types.ts +++ b/webapp/src/common/types.ts @@ -724,8 +724,8 @@ export interface ThematicTrimmingConfigDTO { "CONG. FEE (ALG.)": boolean; "CONG. FEE (ABS.)": boolean; "MARG. COST": boolean; - "CONG. PROD +": boolean; - "CONG. PROD -": boolean; + "CONG. PROB +": boolean; + "CONG. PROB -": boolean; "HURDLE COST": boolean; // Study version >= 810 "RES generation by plant"?: boolean; diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx index b90ba5771a..832f7cac08 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx @@ -9,7 +9,6 @@ import { StudyMetadata } from "../../../../../../../../common/types"; import usePromise from "../../../../../../../../hooks/usePromise"; import BasicDialog from "../../../../../../../common/dialogs/BasicDialog"; import TableForm from "../../../../../../../common/TableForm"; -import BackdropLoading from "../../../../../../../common/loaders/BackdropLoading"; import UsePromiseCond from "../../../../../../../common/utils/UsePromiseCond"; import { DEFAULT_WEIGHT, @@ -78,58 +77,60 @@ function ScenarioPlaylistDialog(props: Props) { //////////////////////////////////////////////////////////////// return ( - } - ifResolved={(defaultValues) => ( - {t("button.close")}} - // TODO: add `maxHeight` and `fullHeight` in BasicDialog` - PaperProps={{ sx: { height: 500 } }} - maxWidth="sm" - > - - - - - - - - - `MC Year ${row.id}`, - tableRef, - stretchH: "all", - className: "htCenter", - cells: handleCellsRender, - }} - /> - - )} - /> + {t("button.close")}} + // TODO: add `maxHeight` and `fullHeight` in BasicDialog` + PaperProps={{ sx: { height: 500 } }} + maxWidth="sm" + fullWidth + > + ( + <> + + + + + + + + + `MC Year ${row.id}`, + tableRef, + stretchH: "all", + className: "htCenter", + cells: handleCellsRender, + }} + /> + + )} + /> + ); } diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts index 26b4ebd6ef..b16491ff6c 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts @@ -48,8 +48,8 @@ export interface ThematicTrimmingFormFields { congFeeAlg: boolean; congFeeAbs: boolean; margCost: boolean; - congProdPlus: boolean; - congProdMinus: boolean; + congProbPlus: boolean; + congProbMinus: boolean; hurdleCost: boolean; // For study versions >= 810 resGenerationByPlant?: boolean; @@ -116,8 +116,8 @@ const keysMap: Record = { congFeeAlg: "CONG. FEE (ALG.)", congFeeAbs: "CONG. FEE (ABS.)", margCost: "MARG. COST", - congProdPlus: "CONG. PROD +", - congProdMinus: "CONG. PROD -", + congProbPlus: "CONG. PROB +", + congProbMinus: "CONG. PROB -", hurdleCost: "HURDLE COST", // Study version >= 810 resGenerationByPlant: "RES generation by plant", diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx deleted file mode 100644 index 025121ed72..0000000000 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import UnderConstruction from "../../../../../common/page/UnderConstruction"; - -function RegionalDistricts() { - return ; -} - -export default RegionalDistricts; diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx index bd56ed29fc..2645b4aba4 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx @@ -5,7 +5,6 @@ import { useMemo, useState } from "react"; import { useOutletContext } from "react-router"; import { useTranslation } from "react-i18next"; import { StudyMetadata } from "../../../../../common/types"; -import UnderConstruction from "../../../../common/page/UnderConstruction"; import PropertiesView from "../../../../common/PropertiesView"; import SplitLayoutView from "../../../../common/SplitLayoutView"; import ListElement from "../common/ListElement"; @@ -13,7 +12,6 @@ import AdequacyPatch from "./AdequacyPatch"; import AdvancedParameters from "./AdvancedParameters"; import General from "./General"; import Optimization from "./Optimization"; -import RegionalDistricts from "./RegionalDistricts"; import TimeSeriesManagement from "./TimeSeriesManagement"; import TableMode from "../../../../common/TableMode"; @@ -28,13 +26,12 @@ function Configuration() { [ { id: 0, name: "General" }, { id: 1, name: "Time-series management" }, - { id: 2, name: "Regional districts" }, - { id: 3, name: "Optimization preferences" }, - Number(study.version) >= 830 && { id: 4, name: "Adequacy Patch" }, - { id: 5, name: "Advanced parameters" }, - { id: 6, name: t("study.configuration.economicOpt") }, - { id: 7, name: t("study.configuration.geographicTrimmingAreas") }, - { id: 8, name: t("study.configuration.geographicTrimmingLinks") }, + { id: 2, name: "Optimization preferences" }, + Number(study.version) >= 830 && { id: 3, name: "Adequacy Patch" }, + { id: 4, name: "Advanced parameters" }, + { id: 5, name: t("study.configuration.economicOpt") }, + { id: 6, name: t("study.configuration.geographicTrimmingAreas") }, + { id: 7, name: t("study.configuration.geographicTrimmingLinks") }, ].filter(Boolean), [study.version, t], ); @@ -59,13 +56,11 @@ function Configuration() { {R.cond([ [R.equals(0), () => ], [R.equals(1), () => ], - [R.equals(1), () => ], - [R.equals(2), () => ], - [R.equals(3), () => ], - [R.equals(4), () => ], - [R.equals(5), () => ], + [R.equals(2), () => ], + [R.equals(3), () => ], + [R.equals(4), () => ], [ - R.equals(6), + R.equals(5), () => ( ( ( { setSaveAllowed(false); }, [studyId, path]); @@ -84,7 +79,6 @@ function Json({ path, studyId }: Props) { } ifResolved={(json) => ( )} - ifRejected={(error) => } /> ); diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx similarity index 79% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx index 524c6b5117..c09b8c645f 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx @@ -1,7 +1,7 @@ import { useOutletContext } from "react-router"; -import { MatrixStats, StudyMetadata } from "../../../../../../../common/types"; +import { MatrixStats, StudyMetadata } from "../../../../../../common/types"; import { Root, Content } from "./style"; -import MatrixInput from "../../../../../../common/MatrixInput"; +import MatrixInput from "../../../../../common/MatrixInput"; interface Props { path: string; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx similarity index 76% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx index d4f00ac1fc..c3ca47d603 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx @@ -4,17 +4,12 @@ import { useSnackbar } from "notistack"; import { useTranslation } from "react-i18next"; import { Button } from "@mui/material"; import UploadOutlinedIcon from "@mui/icons-material/UploadOutlined"; -import { - getStudyData, - importFile, -} from "../../../../../../../services/api/study"; +import { getStudyData, importFile } from "../../../../../../services/api/study"; import { Content, Header, Root } from "./style"; -import useEnqueueErrorSnackbar from "../../../../../../../hooks/useEnqueueErrorSnackbar"; -import SimpleLoader from "../../../../../../common/loaders/SimpleLoader"; -import ImportDialog from "../../../../../../common/dialogs/ImportDialog"; -import usePromiseWithSnackbarError from "../../../../../../../hooks/usePromiseWithSnackbarError"; -import SimpleContent from "../../../../../../common/page/SimpleContent"; -import UsePromiseCond from "../../../../../../common/utils/UsePromiseCond"; +import useEnqueueErrorSnackbar from "../../../../../../hooks/useEnqueueErrorSnackbar"; +import ImportDialog from "../../../../../common/dialogs/ImportDialog"; +import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; +import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; import { useDebugContext } from "../DebugContext"; interface Props { @@ -69,13 +64,11 @@ function Text({ studyId, path }: Props) { } ifResolved={(data) => ( {data} )} - ifRejected={(error) => } /> {openImportDialog && ( {}, + reloadTreeData: (): void => {}, +}; + +const DebugContext = createContext(initialDebugContextValue); + +export const useDebugContext = (): typeof initialDebugContextValue => + useContext(DebugContext); + +export default DebugContext; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/FileTreeItem.tsx similarity index 92% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Tree/FileTreeItem.tsx index 5ae11cff9d..b9e40f6a3d 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/FileTreeItem.tsx @@ -11,7 +11,7 @@ interface Props { function FileTreeItem({ name, content, path }: Props) { const { onFileSelect } = useDebugContext(); - const fullPath = `${path}/${name}`; + const filePath = `${path}/${name}`; const fileType = determineFileType(content); const FileIcon = getFileIcon(fileType); const isFolderEmpty = !Object.keys(content).length; @@ -22,7 +22,7 @@ function FileTreeItem({ name, content, path }: Props) { const handleClick = () => { if (fileType !== "folder") { - onFileSelect(fileType, fullPath); + onFileSelect({ fileType, filePath }); } }; @@ -32,7 +32,7 @@ function FileTreeItem({ name, content, path }: Props) { return ( ))} diff --git a/webapp/src/components/App/Singlestudy/explore/Debug/Tree/index.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/index.tsx new file mode 100644 index 0000000000..577f388efb --- /dev/null +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/index.tsx @@ -0,0 +1,26 @@ +import { TreeView } from "@mui/x-tree-view"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import ChevronRightIcon from "@mui/icons-material/ChevronRight"; +import FileTreeItem from "./FileTreeItem"; +import { TreeData } from "../utils"; + +interface Props { + data: TreeData; +} + +function Tree({ data }: Props) { + return ( + } + defaultExpandIcon={} + > + {typeof data === "object" && + Object.keys(data).map((key) => ( + + ))} + + ); +} + +export default Tree; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/index.tsx similarity index 50% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/index.tsx index 414c004093..1f6ae3cc2d 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/index.tsx @@ -1,15 +1,13 @@ -import { useCallback, useMemo, useState } from "react"; +import { useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { useOutletContext } from "react-router-dom"; import { Box } from "@mui/material"; import Tree from "./Tree"; import Data from "./Data"; -import { StudyMetadata } from "../../../../../../common/types"; -import SimpleLoader from "../../../../../common/loaders/SimpleLoader"; -import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; -import SimpleContent from "../../../../../common/page/SimpleContent"; -import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; -import { getStudyData } from "../../../../../../services/api/study"; +import { StudyMetadata } from "../../../../../common/types"; +import UsePromiseCond from "../../../../common/utils/UsePromiseCond"; +import usePromiseWithSnackbarError from "../../../../../hooks/usePromiseWithSnackbarError"; +import { getStudyData } from "../../../../../services/api/study"; import DebugContext from "./DebugContext"; import { TreeData, filterTreeData, File } from "./utils"; @@ -18,7 +16,7 @@ function Debug() { const { study } = useOutletContext<{ study: StudyMetadata }>(); const [selectedFile, setSelectedFile] = useState(); - const studyTree = usePromiseWithSnackbarError( + const res = usePromiseWithSnackbarError( async () => { const treeData = await getStudyData(study.id, "", -1); return filterTreeData(treeData); @@ -29,24 +27,12 @@ function Debug() { }, ); - const handleFileSelection = useCallback( - (fileType: File["fileType"], filePath: string) => { - setSelectedFile({ fileType, filePath }); - }, - [], - ); - - //////////////////////////////////////////////////////////////// - // Utils - //////////////////////////////////////////////////////////////// - const contextValue = useMemo( () => ({ - treeData: studyTree.data ?? {}, - onFileSelect: handleFileSelection, - reloadTreeData: studyTree.reload, + onFileSelect: setSelectedFile, + reloadTreeData: res.reload, }), - [studyTree.data, studyTree.reload, handleFileSelection], + [res.reload], ); //////////////////////////////////////////////////////////////// @@ -66,25 +52,17 @@ function Debug() { }} > } - ifResolved={() => ( + response={res} + ifResolved={(data) => ( <> - + - {selectedFile && ( - - )} + {selectedFile && } )} - ifRejected={(error) => } /> diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts b/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts similarity index 90% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts rename to webapp/src/components/App/Singlestudy/explore/Debug/utils.ts index 7fc82f087e..e97f0e00a7 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts @@ -48,10 +48,13 @@ export const getFileIcon = (type: FileType | "folder"): SvgIconComponent => { */ export const determineFileType = (treeData: TreeData): FileType | "folder" => { if (typeof treeData === "string") { - if (treeData.startsWith("matrix://")) { + if ( + treeData.startsWith("matrix://") || + treeData.startsWith("matrixfile://") + ) { return "matrix"; } - if (treeData.startsWith("json://")) { + if (treeData.startsWith("json://") || treeData.endsWith(".json")) { return "json"; } } @@ -64,7 +67,7 @@ export const determineFileType = (treeData: TreeData): FileType | "folder" => { * @returns {TreeData} The filtered tree data. */ export const filterTreeData = (data: TreeData): TreeData => { - const excludedKeys = new Set(["Desktop", "study", "output", "logs"]); + const excludedKeys = new Set(["Desktop", "study", "logs"]); return Object.fromEntries( Object.entries(data).filter(([key]) => !excludedKeys.has(key)), diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx index 8b65d57f44..81745530f3 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx @@ -12,51 +12,24 @@ function Hydro() { const { study } = useOutletContext<{ study: StudyMetadata }>(); const areaId = useAppSelector(getCurrentAreaId); - const tabList = useMemo( - () => [ - { - label: "Management options", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/management`, - }, - { - label: "Inflow structure", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/inflowstructure`, - }, - { - label: "Allocation", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/allocation`, - }, - { - label: "Correlation", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/correlation`, - }, - { - label: "Daily Power", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/dailypower`, - }, - { - label: "Energy Credits", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/energycredits`, - }, - { - label: "Reservoir levels", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/reservoirlevels`, - }, - { - label: "Water values", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/watervalues`, - }, - { - label: "Hydro Storage", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/hydrostorage`, - }, - { - label: "Run of river", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/ror`, - }, - ], - [areaId, study?.id], - ); + const tabList = useMemo(() => { + const basePath = `/studies/${study?.id}/explore/modelization/area/${encodeURI( + areaId, + )}/hydro`; + + return [ + { label: "Management options", path: `${basePath}/management` }, + { label: "Inflow structure", path: `${basePath}/inflowstructure` }, + { label: "Allocation", path: `${basePath}/allocation` }, + { label: "Correlation", path: `${basePath}/correlation` }, + { label: "Daily Power", path: `${basePath}/dailypower` }, + { label: "Energy Credits", path: `${basePath}/energycredits` }, + { label: "Reservoir levels", path: `${basePath}/reservoirlevels` }, + { label: "Water values", path: `${basePath}/watervalues` }, + { label: "Hydro Storage", path: `${basePath}/hydrostorage` }, + { label: "Run of river", path: `${basePath}/ror` }, + ]; + }, [areaId, study?.id]); //////////////////////////////////////////////////////////////// // JSX diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/DebugContext.ts b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/DebugContext.ts deleted file mode 100644 index 52e4c07d7f..0000000000 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/DebugContext.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { createContext, useContext } from "react"; -import { FileType, TreeData } from "./utils"; - -interface DebugContextProps { - treeData: TreeData; - onFileSelect: (fileType: FileType, filePath: string) => void; - reloadTreeData: () => void; -} - -const initialDebugContextValue: DebugContextProps = { - treeData: {}, - onFileSelect: () => {}, - reloadTreeData: () => {}, -}; - -const DebugContext = createContext(initialDebugContextValue); - -export const useDebugContext = (): DebugContextProps => - useContext(DebugContext); - -export default DebugContext; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx deleted file mode 100644 index faa75ab045..0000000000 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx +++ /dev/null @@ -1,28 +0,0 @@ -import { TreeView } from "@mui/x-tree-view"; -import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; -import ChevronRightIcon from "@mui/icons-material/ChevronRight"; -import FileTreeItem from "./FileTreeItem"; -import { useDebugContext } from "../DebugContext"; - -function Tree() { - const { treeData } = useDebugContext(); - - //////////////////////////////////////////////////////////////// - // JSX - //////////////////////////////////////////////////////////////// - - return ( - } - defaultExpandIcon={} - > - {typeof treeData === "object" && - Object.keys(treeData).map((key) => ( - - ))} - - ); -} - -export default Tree; diff --git a/webapp/src/components/App/index.tsx b/webapp/src/components/App/index.tsx index 1c7824cfa5..030180b2cd 100644 --- a/webapp/src/components/App/index.tsx +++ b/webapp/src/components/App/index.tsx @@ -24,7 +24,7 @@ import BindingConstraints from "./Singlestudy/explore/Modelization/BindingConstr import Links from "./Singlestudy/explore/Modelization/Links"; import Areas from "./Singlestudy/explore/Modelization/Areas"; import Map from "./Singlestudy/explore/Modelization/Map"; -import Debug from "./Singlestudy/explore/Modelization/Debug"; +import Debug from "./Singlestudy/explore/Debug"; import Xpansion from "./Singlestudy/explore/Xpansion"; import Candidates from "./Singlestudy/explore/Xpansion/Candidates"; import XpansionSettings from "./Singlestudy/explore/Xpansion/Settings"; diff --git a/webapp/src/components/common/page/UnderConstruction.tsx b/webapp/src/components/common/page/UnderConstruction.tsx deleted file mode 100644 index 74243f286e..0000000000 --- a/webapp/src/components/common/page/UnderConstruction.tsx +++ /dev/null @@ -1,66 +0,0 @@ -import ConstructionIcon from "@mui/icons-material/Construction"; -import { Box, Paper, Typography } from "@mui/material"; -import { useTranslation } from "react-i18next"; - -interface Props { - previewImage?: string; -} - -function UnderConstruction(props: Props) { - const { previewImage } = props; - const { t } = useTranslation(); - return ( - - - - {t("common.underConstruction")} - - {previewImage ? ( - - - - PREVIEW - - - preview - - - - ) : undefined} - - ); -} - -UnderConstruction.defaultProps = { - previewImage: undefined, -}; - -export default UnderConstruction;