From 5d16fc92f1959f306a0788508fac1b2354063f9f Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Wed, 10 Jan 2024 22:59:01 +0100 Subject: [PATCH 01/26] refactor(ui-debug): code review (#1892) --- .../explore/Modelization/Debug/Data/Json.tsx | 6 +-- .../explore/Modelization/Debug/Data/Text.tsx | 4 -- .../Modelization/Debug/DebugContext.ts | 19 +++------ .../Modelization/Debug/Tree/FileTreeItem.tsx | 8 ++-- .../explore/Modelization/Debug/Tree/index.tsx | 18 ++++----- .../explore/Modelization/Debug/index.tsx | 40 +++++-------------- 6 files changed, 28 insertions(+), 67 deletions(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx index 4dce33e2ed..c23cac5404 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx @@ -10,11 +10,9 @@ import { getStudyData, } from "../../../../../../../services/api/study"; import { Header, Root } from "./style"; -import SimpleLoader from "../../../../../../common/loaders/SimpleLoader"; import JSONEditor from "../../../../../../common/JSONEditor"; import usePromiseWithSnackbarError from "../../../../../../../hooks/usePromiseWithSnackbarError"; import UsePromiseCond from "../../../../../../common/utils/UsePromiseCond"; -import SimpleContent from "../../../../../../common/page/SimpleContent"; import useEnqueueErrorSnackbar from "../../../../../../../hooks/useEnqueueErrorSnackbar"; interface Props { @@ -37,7 +35,7 @@ function Json({ path, studyId }: Props) { }, ); - /* Reset save button when path changes */ + // Reset save button when path changes useUpdateEffect(() => { setSaveAllowed(false); }, [studyId, path]); @@ -84,7 +82,6 @@ function Json({ path, studyId }: Props) { } ifResolved={(json) => ( )} - ifRejected={(error) => } /> ); diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx index d4f00ac1fc..f61ecc6fad 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx @@ -10,10 +10,8 @@ import { } from "../../../../../../../services/api/study"; import { Content, Header, Root } from "./style"; import useEnqueueErrorSnackbar from "../../../../../../../hooks/useEnqueueErrorSnackbar"; -import SimpleLoader from "../../../../../../common/loaders/SimpleLoader"; import ImportDialog from "../../../../../../common/dialogs/ImportDialog"; import usePromiseWithSnackbarError from "../../../../../../../hooks/usePromiseWithSnackbarError"; -import SimpleContent from "../../../../../../common/page/SimpleContent"; import UsePromiseCond from "../../../../../../common/utils/UsePromiseCond"; import { useDebugContext } from "../DebugContext"; @@ -69,13 +67,11 @@ function Text({ studyId, path }: Props) { } ifResolved={(data) => ( {data} )} - ifRejected={(error) => } /> {openImportDialog && ( void; - reloadTreeData: () => void; -} - -const initialDebugContextValue: DebugContextProps = { - treeData: {}, - onFileSelect: () => {}, - reloadTreeData: () => {}, +const initialDebugContextValue = { + onFileSelect: (file: File): void => {}, + reloadTreeData: (): void => {}, }; -const DebugContext = createContext(initialDebugContextValue); +const DebugContext = createContext(initialDebugContextValue); -export const useDebugContext = (): DebugContextProps => +export const useDebugContext = (): typeof initialDebugContextValue => useContext(DebugContext); export default DebugContext; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx index 5ae11cff9d..b9e40f6a3d 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx @@ -11,7 +11,7 @@ interface Props { function FileTreeItem({ name, content, path }: Props) { const { onFileSelect } = useDebugContext(); - const fullPath = `${path}/${name}`; + const filePath = `${path}/${name}`; const fileType = determineFileType(content); const FileIcon = getFileIcon(fileType); const isFolderEmpty = !Object.keys(content).length; @@ -22,7 +22,7 @@ function FileTreeItem({ name, content, path }: Props) { const handleClick = () => { if (fileType !== "folder") { - onFileSelect(fileType, fullPath); + onFileSelect({ fileType, filePath }); } }; @@ -32,7 +32,7 @@ function FileTreeItem({ name, content, path }: Props) { return ( ))} diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx index faa75ab045..577f388efb 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx @@ -2,24 +2,22 @@ import { TreeView } from "@mui/x-tree-view"; import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; import ChevronRightIcon from "@mui/icons-material/ChevronRight"; import FileTreeItem from "./FileTreeItem"; -import { useDebugContext } from "../DebugContext"; +import { TreeData } from "../utils"; -function Tree() { - const { treeData } = useDebugContext(); - - //////////////////////////////////////////////////////////////// - // JSX - //////////////////////////////////////////////////////////////// +interface Props { + data: TreeData; +} +function Tree({ data }: Props) { return ( } defaultExpandIcon={} > - {typeof treeData === "object" && - Object.keys(treeData).map((key) => ( - + {typeof data === "object" && + Object.keys(data).map((key) => ( + ))} ); diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx index 414c004093..f0f5835c37 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx @@ -1,13 +1,11 @@ -import { useCallback, useMemo, useState } from "react"; +import { useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; import { useOutletContext } from "react-router-dom"; import { Box } from "@mui/material"; import Tree from "./Tree"; import Data from "./Data"; import { StudyMetadata } from "../../../../../../common/types"; -import SimpleLoader from "../../../../../common/loaders/SimpleLoader"; import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; -import SimpleContent from "../../../../../common/page/SimpleContent"; import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; import { getStudyData } from "../../../../../../services/api/study"; import DebugContext from "./DebugContext"; @@ -18,7 +16,7 @@ function Debug() { const { study } = useOutletContext<{ study: StudyMetadata }>(); const [selectedFile, setSelectedFile] = useState(); - const studyTree = usePromiseWithSnackbarError( + const res = usePromiseWithSnackbarError( async () => { const treeData = await getStudyData(study.id, "", -1); return filterTreeData(treeData); @@ -29,24 +27,12 @@ function Debug() { }, ); - const handleFileSelection = useCallback( - (fileType: File["fileType"], filePath: string) => { - setSelectedFile({ fileType, filePath }); - }, - [], - ); - - //////////////////////////////////////////////////////////////// - // Utils - //////////////////////////////////////////////////////////////// - const contextValue = useMemo( () => ({ - treeData: studyTree.data ?? {}, - onFileSelect: handleFileSelection, - reloadTreeData: studyTree.reload, + onFileSelect: setSelectedFile, + reloadTreeData: res.reload, }), - [studyTree.data, studyTree.reload, handleFileSelection], + [res.reload], ); //////////////////////////////////////////////////////////////// @@ -66,25 +52,17 @@ function Debug() { }} > } - ifResolved={() => ( + response={res} + ifResolved={(data) => ( <> - + - {selectedFile && ( - - )} + {selectedFile && } )} - ifRejected={(error) => } /> From dd10be20cf6be9924b68a156c56f4e38a0e46c2c Mon Sep 17 00:00:00 2001 From: Hatim Dinia Date: Thu, 11 Jan 2024 16:49:50 +0100 Subject: [PATCH 02/26] fix(ui-hydro): add areas encoding to hydro tabs path (#1894) --- .../Modelization/Areas/Hydro/index.tsx | 63 ++++++------------- 1 file changed, 18 insertions(+), 45 deletions(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx index 8b65d57f44..81745530f3 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Areas/Hydro/index.tsx @@ -12,51 +12,24 @@ function Hydro() { const { study } = useOutletContext<{ study: StudyMetadata }>(); const areaId = useAppSelector(getCurrentAreaId); - const tabList = useMemo( - () => [ - { - label: "Management options", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/management`, - }, - { - label: "Inflow structure", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/inflowstructure`, - }, - { - label: "Allocation", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/allocation`, - }, - { - label: "Correlation", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/correlation`, - }, - { - label: "Daily Power", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/dailypower`, - }, - { - label: "Energy Credits", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/energycredits`, - }, - { - label: "Reservoir levels", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/reservoirlevels`, - }, - { - label: "Water values", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/watervalues`, - }, - { - label: "Hydro Storage", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/hydrostorage`, - }, - { - label: "Run of river", - path: `/studies/${study?.id}/explore/modelization/area/${areaId}/hydro/ror`, - }, - ], - [areaId, study?.id], - ); + const tabList = useMemo(() => { + const basePath = `/studies/${study?.id}/explore/modelization/area/${encodeURI( + areaId, + )}/hydro`; + + return [ + { label: "Management options", path: `${basePath}/management` }, + { label: "Inflow structure", path: `${basePath}/inflowstructure` }, + { label: "Allocation", path: `${basePath}/allocation` }, + { label: "Correlation", path: `${basePath}/correlation` }, + { label: "Daily Power", path: `${basePath}/dailypower` }, + { label: "Energy Credits", path: `${basePath}/energycredits` }, + { label: "Reservoir levels", path: `${basePath}/reservoirlevels` }, + { label: "Water values", path: `${basePath}/watervalues` }, + { label: "Hydro Storage", path: `${basePath}/hydrostorage` }, + { label: "Run of river", path: `${basePath}/ror` }, + ]; + }, [areaId, study?.id]); //////////////////////////////////////////////////////////////// // JSX From 961746f8ab2c53dbf43e9ea152f3ee3a6b8087d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Omn=C3=A8s?= <26088210+flomnes@users.noreply.github.com> Date: Thu, 11 Jan 2024 19:34:31 +0100 Subject: [PATCH 03/26] fix(configuration): use "CONG. PROB" for thematic trimming (fix typo) (#1893) --- .../study/business/thematic_trimming_management.py | 8 ++++---- tests/integration/test_integration.py | 12 ++++++------ webapp/src/common/types.ts | 4 ++-- .../General/dialogs/ThematicTrimmingDialog/utils.ts | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/antarest/study/business/thematic_trimming_management.py b/antarest/study/business/thematic_trimming_management.py index 317b52b39f..4259046701 100644 --- a/antarest/study/business/thematic_trimming_management.py +++ b/antarest/study/business/thematic_trimming_management.py @@ -64,8 +64,8 @@ class ThematicTrimmingFormFields(FormFieldsBaseModel, metaclass=AllOptionalMetac cong_fee_alg: bool cong_fee_abs: bool marg_cost: bool - cong_prod_plus: bool - cong_prod_minus: bool + cong_prob_plus: bool + cong_prob_minus: bool hurdle_cost: bool # For study versions >= 810 res_generation_by_plant: bool @@ -132,8 +132,8 @@ class ThematicTrimmingFormFields(FormFieldsBaseModel, metaclass=AllOptionalMetac "cong_fee_alg": {"path": "CONG. FEE (ALG.)", "default_value": True}, "cong_fee_abs": {"path": "CONG. FEE (ABS.)", "default_value": True}, "marg_cost": {"path": "MARG. COST", "default_value": True}, - "cong_prod_plus": {"path": "CONG. PROD +", "default_value": True}, - "cong_prod_minus": {"path": "CONG. PROD -", "default_value": True}, + "cong_prob_plus": {"path": "CONG. PROB +", "default_value": True}, + "cong_prob_minus": {"path": "CONG. PROB -", "default_value": True}, "hurdle_cost": {"path": "HURDLE COST", "default_value": True}, "res_generation_by_plant": {"path": "RES generation by plant", "default_value": True, "start_version": 810}, "misc_dtg_2": {"path": "MISC. DTG 2", "default_value": True, "start_version": 810}, diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py index ed4f1ed8af..644713c535 100644 --- a/tests/integration/test_integration.py +++ b/tests/integration/test_integration.py @@ -921,8 +921,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, @@ -990,8 +990,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, @@ -1060,8 +1060,8 @@ def test_area_management(client: TestClient, admin_access_token: str, study_id: "congFeeAlg": True, "congFeeAbs": True, "margCost": True, - "congProdPlus": True, - "congProdMinus": True, + "congProbPlus": True, + "congProbMinus": True, "hurdleCost": True, "resGenerationByPlant": True, "miscDtg2": True, diff --git a/webapp/src/common/types.ts b/webapp/src/common/types.ts index 7687987add..bc78b42d93 100644 --- a/webapp/src/common/types.ts +++ b/webapp/src/common/types.ts @@ -724,8 +724,8 @@ export interface ThematicTrimmingConfigDTO { "CONG. FEE (ALG.)": boolean; "CONG. FEE (ABS.)": boolean; "MARG. COST": boolean; - "CONG. PROD +": boolean; - "CONG. PROD -": boolean; + "CONG. PROB +": boolean; + "CONG. PROB -": boolean; "HURDLE COST": boolean; // Study version >= 810 "RES generation by plant"?: boolean; diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts index 26b4ebd6ef..b16491ff6c 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ThematicTrimmingDialog/utils.ts @@ -48,8 +48,8 @@ export interface ThematicTrimmingFormFields { congFeeAlg: boolean; congFeeAbs: boolean; margCost: boolean; - congProdPlus: boolean; - congProdMinus: boolean; + congProbPlus: boolean; + congProbMinus: boolean; hurdleCost: boolean; // For study versions >= 810 resGenerationByPlant?: boolean; @@ -116,8 +116,8 @@ const keysMap: Record = { congFeeAlg: "CONG. FEE (ALG.)", congFeeAbs: "CONG. FEE (ABS.)", margCost: "MARG. COST", - congProdPlus: "CONG. PROD +", - congProdMinus: "CONG. PROD -", + congProbPlus: "CONG. PROB +", + congProbMinus: "CONG. PROB -", hurdleCost: "HURDLE COST", // Study version >= 810 resGenerationByPlant: "RES generation by plant", From 2c7b904193abfba50e9eae99aad173df2fdf1349 Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Fri, 12 Jan 2024 17:52:21 +0100 Subject: [PATCH 04/26] fix(ui-debug): handle txt matrix files --- .../App/Singlestudy/explore/Modelization/Debug/utils.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts index 7fc82f087e..aba3d0ec45 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts @@ -48,7 +48,10 @@ export const getFileIcon = (type: FileType | "folder"): SvgIconComponent => { */ export const determineFileType = (treeData: TreeData): FileType | "folder" => { if (typeof treeData === "string") { - if (treeData.startsWith("matrix://")) { + if ( + treeData.startsWith("matrix://") || + treeData.startsWith("matrixfile://") + ) { return "matrix"; } if (treeData.startsWith("json://")) { From 501c487f02209884d9b26e26a07de5d062999a46 Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Fri, 12 Jan 2024 18:13:22 +0100 Subject: [PATCH 05/26] fix(ui-debug): undo output folder hidding --- .../App/Singlestudy/explore/Modelization/Debug/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts index aba3d0ec45..75162313a1 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts @@ -67,7 +67,7 @@ export const determineFileType = (treeData: TreeData): FileType | "folder" => { * @returns {TreeData} The filtered tree data. */ export const filterTreeData = (data: TreeData): TreeData => { - const excludedKeys = new Set(["Desktop", "study", "output", "logs"]); + const excludedKeys = new Set(["Desktop", "study", "logs"]); return Object.fromEntries( Object.entries(data).filter(([key]) => !excludedKeys.has(key)), From 28efdf88baa97aa0bed038b656930f43ac598875 Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Mon, 15 Jan 2024 15:20:50 +0100 Subject: [PATCH 06/26] refactor(ui-debug): move Debug view and update imports --- .../explore/{Modelization => }/Debug/Data/Json.tsx | 13 +++++-------- .../{Modelization => }/Debug/Data/Matrix.tsx | 4 ++-- .../explore/{Modelization => }/Debug/Data/Text.tsx | 13 +++++-------- .../explore/{Modelization => }/Debug/Data/index.tsx | 0 .../explore/{Modelization => }/Debug/Data/style.ts | 0 .../{Modelization => }/Debug/DebugContext.ts | 0 .../{Modelization => }/Debug/Tree/FileTreeItem.tsx | 0 .../explore/{Modelization => }/Debug/Tree/index.tsx | 0 .../explore/{Modelization => }/Debug/index.tsx | 8 ++++---- .../explore/{Modelization => }/Debug/utils.ts | 0 webapp/src/components/App/index.tsx | 2 +- 11 files changed, 17 insertions(+), 23 deletions(-) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Data/Json.tsx (86%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Data/Matrix.tsx (79%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Data/Text.tsx (84%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Data/index.tsx (100%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Data/style.ts (100%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/DebugContext.ts (100%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Tree/FileTreeItem.tsx (100%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/Tree/index.tsx (100%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/index.tsx (86%) rename webapp/src/components/App/Singlestudy/explore/{Modelization => }/Debug/utils.ts (100%) diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx similarity index 86% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx index c23cac5404..da0c8a477a 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Json.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx @@ -5,15 +5,12 @@ import { useSnackbar } from "notistack"; import SaveIcon from "@mui/icons-material/Save"; import { Box, Button, Typography } from "@mui/material"; import { useUpdateEffect } from "react-use"; -import { - editStudy, - getStudyData, -} from "../../../../../../../services/api/study"; +import { editStudy, getStudyData } from "../../../../../../services/api/study"; import { Header, Root } from "./style"; -import JSONEditor from "../../../../../../common/JSONEditor"; -import usePromiseWithSnackbarError from "../../../../../../../hooks/usePromiseWithSnackbarError"; -import UsePromiseCond from "../../../../../../common/utils/UsePromiseCond"; -import useEnqueueErrorSnackbar from "../../../../../../../hooks/useEnqueueErrorSnackbar"; +import JSONEditor from "../../../../../common/JSONEditor"; +import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; +import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; +import useEnqueueErrorSnackbar from "../../../../../../hooks/useEnqueueErrorSnackbar"; interface Props { path: string; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx similarity index 79% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx index 524c6b5117..c09b8c645f 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Matrix.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Matrix.tsx @@ -1,7 +1,7 @@ import { useOutletContext } from "react-router"; -import { MatrixStats, StudyMetadata } from "../../../../../../../common/types"; +import { MatrixStats, StudyMetadata } from "../../../../../../common/types"; import { Root, Content } from "./style"; -import MatrixInput from "../../../../../../common/MatrixInput"; +import MatrixInput from "../../../../../common/MatrixInput"; interface Props { path: string; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx similarity index 84% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx index f61ecc6fad..c3ca47d603 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/Text.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Text.tsx @@ -4,15 +4,12 @@ import { useSnackbar } from "notistack"; import { useTranslation } from "react-i18next"; import { Button } from "@mui/material"; import UploadOutlinedIcon from "@mui/icons-material/UploadOutlined"; -import { - getStudyData, - importFile, -} from "../../../../../../../services/api/study"; +import { getStudyData, importFile } from "../../../../../../services/api/study"; import { Content, Header, Root } from "./style"; -import useEnqueueErrorSnackbar from "../../../../../../../hooks/useEnqueueErrorSnackbar"; -import ImportDialog from "../../../../../../common/dialogs/ImportDialog"; -import usePromiseWithSnackbarError from "../../../../../../../hooks/usePromiseWithSnackbarError"; -import UsePromiseCond from "../../../../../../common/utils/UsePromiseCond"; +import useEnqueueErrorSnackbar from "../../../../../../hooks/useEnqueueErrorSnackbar"; +import ImportDialog from "../../../../../common/dialogs/ImportDialog"; +import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; +import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; import { useDebugContext } from "../DebugContext"; interface Props { diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/index.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/index.tsx similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/index.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/index.tsx diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/style.ts b/webapp/src/components/App/Singlestudy/explore/Debug/Data/style.ts similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Data/style.ts rename to webapp/src/components/App/Singlestudy/explore/Debug/Data/style.ts diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/DebugContext.ts b/webapp/src/components/App/Singlestudy/explore/Debug/DebugContext.ts similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/DebugContext.ts rename to webapp/src/components/App/Singlestudy/explore/Debug/DebugContext.ts diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/FileTreeItem.tsx similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/FileTreeItem.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Tree/FileTreeItem.tsx diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Tree/index.tsx similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/Tree/index.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/Tree/index.tsx diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/index.tsx similarity index 86% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx rename to webapp/src/components/App/Singlestudy/explore/Debug/index.tsx index f0f5835c37..1f6ae3cc2d 100644 --- a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/index.tsx @@ -4,10 +4,10 @@ import { useOutletContext } from "react-router-dom"; import { Box } from "@mui/material"; import Tree from "./Tree"; import Data from "./Data"; -import { StudyMetadata } from "../../../../../../common/types"; -import UsePromiseCond from "../../../../../common/utils/UsePromiseCond"; -import usePromiseWithSnackbarError from "../../../../../../hooks/usePromiseWithSnackbarError"; -import { getStudyData } from "../../../../../../services/api/study"; +import { StudyMetadata } from "../../../../../common/types"; +import UsePromiseCond from "../../../../common/utils/UsePromiseCond"; +import usePromiseWithSnackbarError from "../../../../../hooks/usePromiseWithSnackbarError"; +import { getStudyData } from "../../../../../services/api/study"; import DebugContext from "./DebugContext"; import { TreeData, filterTreeData, File } from "./utils"; diff --git a/webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts b/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts similarity index 100% rename from webapp/src/components/App/Singlestudy/explore/Modelization/Debug/utils.ts rename to webapp/src/components/App/Singlestudy/explore/Debug/utils.ts diff --git a/webapp/src/components/App/index.tsx b/webapp/src/components/App/index.tsx index 1c7824cfa5..030180b2cd 100644 --- a/webapp/src/components/App/index.tsx +++ b/webapp/src/components/App/index.tsx @@ -24,7 +24,7 @@ import BindingConstraints from "./Singlestudy/explore/Modelization/BindingConstr import Links from "./Singlestudy/explore/Modelization/Links"; import Areas from "./Singlestudy/explore/Modelization/Areas"; import Map from "./Singlestudy/explore/Modelization/Map"; -import Debug from "./Singlestudy/explore/Modelization/Debug"; +import Debug from "./Singlestudy/explore/Debug"; import Xpansion from "./Singlestudy/explore/Xpansion"; import Candidates from "./Singlestudy/explore/Xpansion/Candidates"; import XpansionSettings from "./Singlestudy/explore/Xpansion/Settings"; From 73fbb93c3d68c5eb0a7ad0bd4dad58271bca635f Mon Sep 17 00:00:00 2001 From: hatim dinia Date: Mon, 15 Jan 2024 15:21:49 +0100 Subject: [PATCH 07/26] fix(ui-debug): handle json files extension --- webapp/src/components/App/Singlestudy/explore/Debug/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts b/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts index 75162313a1..e97f0e00a7 100644 --- a/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts +++ b/webapp/src/components/App/Singlestudy/explore/Debug/utils.ts @@ -54,7 +54,7 @@ export const determineFileType = (treeData: TreeData): FileType | "folder" => { ) { return "matrix"; } - if (treeData.startsWith("json://")) { + if (treeData.startsWith("json://") || treeData.endsWith(".json")) { return "json"; } } From 2953d2a6dda0e0360289deb31ac8339118a87efc Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:03:06 +0100 Subject: [PATCH 08/26] style(ui-debug): remove console warning --- .../src/components/App/Singlestudy/explore/Debug/Data/Json.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx index da0c8a477a..334fa84638 100644 --- a/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Debug/Data/Json.tsx @@ -88,8 +88,7 @@ function Json({ path, studyId }: Props) { > Date: Thu, 4 Jan 2024 10:51:43 +0100 Subject: [PATCH 09/26] ci(docker): Add the `ll` alias in `.bashrc` to simplify debugging --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index affdc86660..9a0a596b91 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM python:3.8-slim-bullseye # RUN apt update && apt install -y procps gdb +# Add the `ls` alias to simplify debugging +RUN echo "alias ls='/bin/ls -l --color=auto'" >> /root/.bashrc + ENV ANTAREST_CONF /resources/application.yaml RUN mkdir -p examples/studies From 9ac19b9cb6435b5ce88b77fc90f024d997a3383f Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 4 Jan 2024 10:56:14 +0100 Subject: [PATCH 10/26] ci(docker): Add the `ll` alias in `.bashrc` to simplify debugging --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9a0a596b91..975ef9b776 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ FROM python:3.8-slim-bullseye # RUN apt update && apt install -y procps gdb # Add the `ls` alias to simplify debugging -RUN echo "alias ls='/bin/ls -l --color=auto'" >> /root/.bashrc +RUN echo "alias ll='/bin/ls -l --color=auto'" >> /root/.bashrc ENV ANTAREST_CONF /resources/application.yaml From f64cf734e776468cae7a67939d58dd3d6ac34b2c Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 15:58:20 +0100 Subject: [PATCH 11/26] fix(api-raw): correct download filename --- antarest/study/web/raw_studies_blueprint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/antarest/study/web/raw_studies_blueprint.py b/antarest/study/web/raw_studies_blueprint.py index b84a9bf107..41e214d1ad 100644 --- a/antarest/study/web/raw_studies_blueprint.py +++ b/antarest/study/web/raw_studies_blueprint.py @@ -128,7 +128,7 @@ def get_study( detail=f"Invalid plain text configuration in path '{path}': {exc}", ) from None elif content_type: - headers = {"Content-Disposition": f"attachment; filename='{resource_path.name}'"} + headers = {"Content-Disposition": f"attachment; filename={resource_path.name}"} return StreamingResponse( io.BytesIO(output), media_type=content_type, From acbc52a2ffca6789e06fd8ab82c92ca863f7b452 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Thu, 11 Jan 2024 19:23:43 +0100 Subject: [PATCH 12/26] feat(api-filesystem): add the "filesystem" blueprint to get disk usage and manage files/folders --- antarest/core/filesystem_blueprint.py | 578 ++++++++++++++++++++++++++ antarest/core/utils/web.py | 4 + antarest/main.py | 2 + 3 files changed, 584 insertions(+) create mode 100644 antarest/core/filesystem_blueprint.py diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py new file mode 100644 index 0000000000..7fde9c27a0 --- /dev/null +++ b/antarest/core/filesystem_blueprint.py @@ -0,0 +1,578 @@ +""" +Filesystem Blueprint +""" +import asyncio +import datetime +import os +import shutil +import stat +import typing as t +from pathlib import Path + +import typing_extensions +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Extra, Field +from starlette.responses import PlainTextResponse, StreamingResponse + +from antarest.core.config import Config +from antarest.core.jwt import JWTUser +from antarest.core.utils.web import APITag +from antarest.login.auth import Auth + +FilesystemName = typing_extensions.Annotated[str, Field(regex=r"^\w+$", description="Filesystem name")] +MountPointName = typing_extensions.Annotated[str, Field(regex=r"^\w+$", description="Mount point name")] + + +class FilesystemDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "name": "ws", + "mount_dirs": { + "default": "/path/to/workspaces/internal_studies", + "common": "/path/to/workspaces/common_studies", + }, + }, + }, +): + """ + Filesystem information. + + Attributes: + + - `name`: name of the filesystem. + - `mount_dirs`: mapping of the mount point names to their full path in Antares Web Server. + """ + + name: FilesystemName + mount_dirs: t.Mapping[str, Path] = Field(description="Full path of the mount points in Antares Web Server") + + +class MountPointDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "name": "default", + "path": "/path/to/workspaces/internal_studies", + "total_bytes": 1e9, + "used_bytes": 0.6e9, + "free_bytes": 1e9 - 0.6e9, + "message": f"{0.6e9 / 1e9:%} used", + }, + }, +): + """ + Disk usage information of a filesystem. + + Attributes: + + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + """ + + name: MountPointName + path: Path = Field(description="Full path of the mount point in Antares Web Server") + total_bytes: int = Field(0, description="Total size of the mount point in bytes") + used_bytes: int = Field(0, description="Used size of the mount point in bytes") + free_bytes: int = Field(0, description="Free size of the mount point in bytes") + message: str = Field("", description="A message describing the status of the mount point") + + @classmethod + async def from_path(cls, name: str, path: Path) -> "MountPointDTO": + obj = cls(name=name, path=path.resolve()) + try: + obj.total_bytes, obj.used_bytes, obj.free_bytes = shutil.disk_usage(obj.path) + except OSError as exc: + # Avoid raising an exception if the file doesn't exist + # or if the mount point is not available. + obj.message = f"N/A: {exc}" + else: + obj.message = f"{obj.used_bytes / obj.total_bytes:.1%} used" + return obj + + +class FileInfoDTO( + BaseModel, + extra=Extra.forbid, + schema_extra={ + "example": { + "path": "/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares", + "file_type": "file", + "file_count": 1, + "size_bytes": 126, + "created": "2023-12-07T17:59:43", + "modified": "2023-12-07T17:59:43", + "accessed": "2024-01-11T17:54:09", + "message": "OK", + }, + }, +): + """ + File information of a file or directory. + + Attributes: + + - `path`: full path of the file or directory in Antares Web Server. + - `file_type`: file type: "folder", "file", "symlink", "socket", "block_device", + "character_device", "fifo", or "unknown". + - `file_count`: number of files in the directory (1 for files). + - `size_bytes`: size of the file or total size of the directory in bytes. + - `created`: creation date of the file or directory (UTC). + - `modified`: last modification date of the file or directory (UTC). + - `accessed`: last access date of the file or directory (UTC). + - `message`: a message describing the status of the file. + """ + + path: Path = Field(description="Full path of the file or directory in Antares Web Server") + file_type: str = Field(description="Type of the file or directory") + file_count: int = Field(1, description="Number of files in the directory (1 for files)") + size_bytes: int = Field(0, description="Size of the file or total size of the directory in bytes") + created: datetime.datetime = Field(description="Creation date of the file or directory (UTC)") + modified: datetime.datetime = Field(description="Last modification date of the file or directory (UTC)") + accessed: datetime.datetime = Field(description="Last access date of the file or directory (UTC)") + message: str = Field("OK", description="A message describing the status of the file") + + @classmethod + async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfoDTO": + try: + file_stat = full_path.stat() + except OSError as exc: + # Avoid raising an exception if the file doesn't exist + # or if the mount point is not available. + return cls( + path=full_path, + file_type="unknown", + file_count=0, # missing + created=datetime.datetime.utcnow(), + modified=datetime.datetime.utcnow(), + accessed=datetime.datetime.utcnow(), + message=f"N/A: {exc}", + ) + + obj = cls( + path=full_path, + file_type="unknown", + file_count=1, + size_bytes=file_stat.st_size, + created=datetime.datetime.fromtimestamp(file_stat.st_ctime), + modified=datetime.datetime.fromtimestamp(file_stat.st_mtime), + accessed=datetime.datetime.fromtimestamp(file_stat.st_atime), + ) + + if stat.S_ISDIR(file_stat.st_mode): + obj.file_type = "folder" + if details: + file_count, disk_space = await _calc_details(full_path) + obj.file_count = file_count + obj.size_bytes = disk_space + elif stat.S_ISREG(file_stat.st_mode): + obj.file_type = "file" + elif stat.S_ISLNK(file_stat.st_mode): + obj.file_type = "symlink" + elif stat.S_ISSOCK(file_stat.st_mode): + obj.file_type = "socket" + elif stat.S_ISBLK(file_stat.st_mode): + obj.file_type = "block_device" + elif stat.S_ISCHR(file_stat.st_mode): + obj.file_type = "character_device" + elif stat.S_ISFIFO(file_stat.st_mode): + obj.file_type = "fifo" + else: # pragma: no cover + obj.file_type = "unknown" + + return obj + + +async def _calc_details(full_path: t.Union[str, Path]) -> t.Tuple[int, int]: + """Calculate the number of files and the total size of a directory recursively.""" + file_count = 0 + total_size = 0 + + for entry in os.scandir(full_path): + if entry.is_dir(): + sub_file_count, sub_total_size = await _calc_details(entry.path) + file_count += sub_file_count + total_size += sub_total_size + else: + file_count += 1 + total_size += entry.stat().st_size + + return file_count, total_size + + +def _is_relative_to(path: Path, base_path: Path) -> bool: + """Check if the given path is relative to the specified base path.""" + try: + path = Path(path).resolve() + base_path = Path(base_path).resolve() + return bool(path.relative_to(base_path)) + except ValueError: + return False + + +def create_file_system_blueprint(config: Config) -> APIRouter: + """ + Create the blueprint for the file system API. + + This blueprint is used to diagnose the disk space consumption of the different + workspaces (especially the "default" workspace of Antares Web), and the different + storage directories (`tmp`, `matrixstore`, `archive`, etc. defined in the configuration file). + + This blueprint is also used to list files and directories, to download a file, + and to delete a file or directory. + + Reading files is allowed for authenticated users, but deleting files is reserved + for site administrators. + + Args: + config: Application configuration. + + Returns: + The blueprint. + """ + auth = Auth(config) + bp = APIRouter( + prefix="/filesystem", + tags=[APITag.filesystem], + dependencies=[Depends(auth.get_current_user)], + include_in_schema=True, # but may be disabled in the future + ) + config_dirs = { + "res": config.resources_path, + "tmp": config.storage.tmp_dir, + "matrix": config.storage.matrixstore, + "archive": config.storage.archive_dir, + } + workspace_dirs = {name: ws_cfg.path for name, ws_cfg in config.storage.workspaces.items()} + filesystems = { + "cfg": config_dirs, + "ws": workspace_dirs, + } + + # Utility functions + # ================= + + def _get_mount_dirs(fs: str) -> t.Mapping[str, Path]: + try: + return filesystems[fs] + except KeyError: + raise HTTPException(status_code=404, detail=f"Filesystem not found: '{fs}'") from None + + def _get_mount_dir(fs: str, mount: str) -> Path: + try: + return filesystems[fs][mount] + except KeyError: + raise HTTPException(status_code=404, detail=f"Mount point not found: '{fs}/{mount}'") from None + + def _get_full_path(mount_dir: Path, path: str) -> Path: + if not path: + raise HTTPException(status_code=400, detail="Empty or missing path parameter") + full_path = (mount_dir / path).resolve() + if not _is_relative_to(full_path, mount_dir): + raise HTTPException(status_code=403, detail=f"Access denied to path: '{path}'") + if not full_path.exists(): + raise HTTPException(status_code=404, detail=f"Path not found: '{path}'") + return full_path + + # Endpoints + # ========= + + @bp.get( + "", + summary="Get filesystems information", + response_model=t.Sequence[FilesystemDTO], + ) + async def list_filesystems() -> t.Sequence[FilesystemDTO]: + """ + Get the list of filesystems and their mount points. + + Returns: + - `name`: name of the filesystem: "cfg" or "ws". + - `mount_dirs`: mapping of the mount point names to their full path in Antares Web Server. + """ + + fs = [FilesystemDTO(name=name, mount_dirs=mount_dirs) for name, mount_dirs in filesystems.items()] + return fs + + @bp.get( + "/{fs}", + summary="Get information of a filesystem", + response_model=t.Sequence[MountPointDTO], + ) + async def list_mount_points(fs: FilesystemName) -> t.Sequence[MountPointDTO]: + """ + Get the path and the disk usage of the mount points in a filesystem. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + + Returns: + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + + Possible error codes: + - 404 Not Found: If the specified filesystem doesn't exist. + """ + + mount_dirs = _get_mount_dirs(fs) + tasks = [MountPointDTO.from_path(name, path) for name, path in mount_dirs.items()] + ws = await asyncio.gather(*tasks) + return ws + + @bp.get( + "/{fs}/{mount}", + summary="Get information of a mount point", + response_model=MountPointDTO, + ) + async def get_mount_point(fs: FilesystemName, mount: MountPointName) -> MountPointDTO: + """ + Get the path and the disk usage of a mount point. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + + Returns: + - `name`: name of the mount point. + - `path`: path of the mount point. + - `total_bytes`: total size of the mount point in bytes. + - `used_bytes`: used size of the mount point in bytes. + - `free_bytes`: free size of the mount point in bytes. + - `message`: a message describing the status of the mount point. + + Possible error codes: + - 404 Not Found: If the specified filesystem or mount point doesn't exist. + """ + + mount_dir = _get_mount_dir(fs, mount) + return await MountPointDTO.from_path(mount, mount_dir) + + @bp.get( + "/{fs}/{mount}/ls", + summary="List files in a workspace", + response_model=t.Sequence[FileInfoDTO], + ) + async def list_files( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + details: bool = False, + ) -> t.Sequence[FileInfoDTO]: + """ + List files and directories in a workspace. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the directory to list. + - `details`: If true, return the number of files and the total size of each directory. + + > **⚠ Warning:** Using a glob pattern for the `path` parameter (for instance, "**/study.antares") + > or using the `details` parameter can slow down the response time of this endpoint. + + Returns: + - `path`: full path of the file or directory in Antares Web Server. + This path can contain glob patterns (e.g., `*.txt`). + - `file_type`: file type: "folder", "file", "symlink", "socket", "block_device", + "character_device", "fifo", or "unknown". + - `file_count`: number of files in the directory (1 for files). + - `size_bytes`: size of the file or total size of the directory in bytes. + - `created`: creation date of the file or directory (UTC). + - `modified`: last modification date of the file or directory (UTC). + - `accessed`: last access date of the file or directory (UTC). + - `message`: a message describing the status of the file. + + Possible error codes: + - 400 Bad Request: If the specified path is invalid (e.g., contains invalid glob patterns). + - 404 Not Found: If the specified filesystem or mount point doesn't exist. + - 403 Forbidden: If the user has no permission to access the directory. + """ + + mount_dir = _get_mount_dir(fs, mount) + + # The following code looks weird, but it's the only way to handle exceptions in generators. + tasks = [] + iterator = mount_dir.glob(path) if path else mount_dir.iterdir() + while True: + try: + file_path = next(iterator) + except StopIteration: + break + except (OSError, ValueError, IndexError) as exc: + # Unacceptable pattern: invalid glob pattern or access denied + raise HTTPException(status_code=400, detail=f"Invalid path: '{path}'. {exc}") from exc + except NotImplementedError as exc: + # Unacceptable pattern: non-relative glob pattern + raise HTTPException(status_code=403, detail=f"Access denied to path: '{path}'. {exc}") from exc + else: + file_info = FileInfoDTO.from_path(file_path, details=details) + tasks.append(file_info) + + file_info_list = await asyncio.gather(*tasks) + return file_info_list + + @bp.delete( + "/{fs}/{mount}", + summary="Remove a file or directory recursively from a workspace", + response_model=str, + ) + async def remove_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + current_user: JWTUser = Depends(auth.get_current_user), + ) -> str: + """ + Remove a file or directory recursively from a workspace. + + > **⚠ Warning:** This endpoint is reserved for site administrators. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file or directory to delete. + + Returns: + - Success message (e.g., "File deleted successfully"). + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to delete the file or directory. + - 404 Not Found: If the specified filesystem, mount point, file or directory doesn't exist. + - 417 Expectation Failed: If the specified path is not a file or directory. + """ + + if not current_user.is_site_admin(): + raise HTTPException(status_code=403, detail="User has no permission to delete files") + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + shutil.rmtree(full_path) + return f"Directory deleted successfully: '{path}'" + elif full_path.is_file(): + full_path.unlink() + return f"File deleted successfully: '{path}'" + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + + @bp.get( + "/{fs}/{mount}/cat", + summary="View a text file from a workspace", + response_class=PlainTextResponse, + response_description="File content as text", + ) + async def view_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + encoding: str = "utf-8", + ) -> str: + # noinspection SpellCheckingInspection + """ + View a text file from a workspace. + + Examples: + ```ini + [antares] + version = 820 + caption = default + created = 1701964773.965127 + lastsave = 1701964773.965131 + author = John DOE + ``` + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file to view. + - `encoding`: The encoding of the file. Defaults to "utf-8". + + Returns: + - File content as text or binary. + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to view the file. + - 404 Not Found: If the specified filesystem, mount point or file doesn't exist. + - 417 Expectation Failed: If the specified path is not a text file or if the encoding is invalid. + """ + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + raise HTTPException(status_code=417, detail=f"Path is not a file: '{path}'") + + elif full_path.is_file(): + try: + return full_path.read_text(encoding=encoding) + except LookupError as exc: + raise HTTPException(status_code=417, detail=f"Unknown encoding: '{encoding}'") from exc + except UnicodeDecodeError as exc: + raise HTTPException(status_code=417, detail=f"Failed to decode file: '{path}'") from exc + + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + + @bp.get( + "/{fs}/{mount}/download", + summary="Download a file from a workspace", + response_class=StreamingResponse, + response_description="File content as binary", + ) + async def download_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + ) -> StreamingResponse: + """ + Download a file from a workspace. + + > **Note:** Directory download is not supported yet. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file or directory to download. + + Returns: + - File content as binary. + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to download the file. + - 404 Not Found: If the specified filesystem, mount point or file doesn't exist. + - 417 Expectation Failed: If the specified path is not a regular file. + """ + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + raise HTTPException(status_code=417, detail=f"Path is not a file: '{path}'") + + elif full_path.is_file(): + + def iter_file() -> t.Iterator[bytes]: + with full_path.open(mode="rb") as file: + yield from file + + headers = {"Content-Disposition": f"attachment; filename={full_path.name}"} + return StreamingResponse(iter_file(), media_type="application/octet-stream", headers=headers) + + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + + return bp diff --git a/antarest/core/utils/web.py b/antarest/core/utils/web.py index 69a530a2b4..2990b05c41 100644 --- a/antarest/core/utils/web.py +++ b/antarest/core/utils/web.py @@ -12,6 +12,7 @@ class APITag: matrix = "Manage Matrix" tasks = "Manage tasks" misc = "Miscellaneous" + filesystem = "Filesystem Management" tags_metadata = [ @@ -54,4 +55,7 @@ class APITag: { "name": APITag.misc, }, + { + "name": APITag.filesystem, + }, ] diff --git a/antarest/main.py b/antarest/main.py index 700947a4dd..93be38ed2b 100644 --- a/antarest/main.py +++ b/antarest/main.py @@ -27,6 +27,7 @@ from antarest import __version__ from antarest.core.config import Config from antarest.core.core_blueprint import create_utils_routes +from antarest.core.filesystem_blueprint import create_file_system_blueprint from antarest.core.logging.utils import LoggingMiddleware, configure_logger from antarest.core.requests import RATE_LIMIT_CONFIG from antarest.core.swagger import customize_openapi @@ -299,6 +300,7 @@ def get_config() -> JwtSettings: allow_headers=["*"], ) application.include_router(create_utils_routes(config)) + application.include_router(create_file_system_blueprint(config)) # noinspection PyUnusedLocal @application.exception_handler(HTTPException) From 950d71ef916e307e6125012cae34e7d3fc6c5b30 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 17:14:42 +0100 Subject: [PATCH 13/26] feat(api-filesystem): add unit test to check DTO models --- antarest/core/filesystem_blueprint.py | 60 ++++---- .../filesystem_blueprint/__init__.py | 0 .../filesystem_blueprint/test_model.py | 145 ++++++++++++++++++ 3 files changed, 175 insertions(+), 30 deletions(-) create mode 100644 tests/integration/filesystem_blueprint/__init__.py create mode 100644 tests/integration/filesystem_blueprint/test_model.py diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 7fde9c27a0..37fd9ab6c1 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -119,23 +119,23 @@ class FileInfoDTO( Attributes: - `path`: full path of the file or directory in Antares Web Server. - - `file_type`: file type: "folder", "file", "symlink", "socket", "block_device", + - `file_type`: file type: "directory", "file", "symlink", "socket", "block_device", "character_device", "fifo", or "unknown". - - `file_count`: number of files in the directory (1 for files). + - `file_count`: number of files and folders in the directory (1 for files). - `size_bytes`: size of the file or total size of the directory in bytes. - - `created`: creation date of the file or directory (UTC). - - `modified`: last modification date of the file or directory (UTC). - - `accessed`: last access date of the file or directory (UTC). + - `created`: creation date of the file or directory (local time). + - `modified`: last modification date of the file or directory (local time). + - `accessed`: last access date of the file or directory (local time). - `message`: a message describing the status of the file. """ path: Path = Field(description="Full path of the file or directory in Antares Web Server") file_type: str = Field(description="Type of the file or directory") - file_count: int = Field(1, description="Number of files in the directory (1 for files)") + file_count: int = Field(1, description="Number of files and folders in the directory (1 for files)") size_bytes: int = Field(0, description="Size of the file or total size of the directory in bytes") - created: datetime.datetime = Field(description="Creation date of the file or directory (UTC)") - modified: datetime.datetime = Field(description="Last modification date of the file or directory (UTC)") - accessed: datetime.datetime = Field(description="Last access date of the file or directory (UTC)") + created: datetime.datetime = Field(description="Creation date of the file or directory (local time)") + modified: datetime.datetime = Field(description="Last modification date of the file or directory (local time)") + accessed: datetime.datetime = Field(description="Last access date of the file or directory (local time)") message: str = Field("OK", description="A message describing the status of the file") @classmethod @@ -149,9 +149,9 @@ async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfo path=full_path, file_type="unknown", file_count=0, # missing - created=datetime.datetime.utcnow(), - modified=datetime.datetime.utcnow(), - accessed=datetime.datetime.utcnow(), + created=datetime.datetime.min, + modified=datetime.datetime.min, + accessed=datetime.datetime.min, message=f"N/A: {exc}", ) @@ -166,22 +166,22 @@ async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfo ) if stat.S_ISDIR(file_stat.st_mode): - obj.file_type = "folder" + obj.file_type = "directory" if details: file_count, disk_space = await _calc_details(full_path) obj.file_count = file_count obj.size_bytes = disk_space elif stat.S_ISREG(file_stat.st_mode): obj.file_type = "file" - elif stat.S_ISLNK(file_stat.st_mode): + elif stat.S_ISLNK(file_stat.st_mode): # pragma: no cover obj.file_type = "symlink" - elif stat.S_ISSOCK(file_stat.st_mode): + elif stat.S_ISSOCK(file_stat.st_mode): # pragma: no cover obj.file_type = "socket" - elif stat.S_ISBLK(file_stat.st_mode): + elif stat.S_ISBLK(file_stat.st_mode): # pragma: no cover obj.file_type = "block_device" - elif stat.S_ISCHR(file_stat.st_mode): + elif stat.S_ISCHR(file_stat.st_mode): # pragma: no cover obj.file_type = "character_device" - elif stat.S_ISFIFO(file_stat.st_mode): + elif stat.S_ISFIFO(file_stat.st_mode): # pragma: no cover obj.file_type = "fifo" else: # pragma: no cover obj.file_type = "unknown" @@ -191,17 +191,17 @@ async def from_path(cls, full_path: Path, *, details: bool = False) -> "FileInfo async def _calc_details(full_path: t.Union[str, Path]) -> t.Tuple[int, int]: """Calculate the number of files and the total size of a directory recursively.""" - file_count = 0 - total_size = 0 - for entry in os.scandir(full_path): - if entry.is_dir(): + full_path = Path(full_path) + file_stat = full_path.stat() + file_count = 1 + total_size = file_stat.st_size + + if stat.S_ISDIR(file_stat.st_mode): + for entry in os.scandir(full_path): sub_file_count, sub_total_size = await _calc_details(entry.path) file_count += sub_file_count total_size += sub_total_size - else: - file_count += 1 - total_size += entry.stat().st_size return file_count, total_size @@ -383,13 +383,13 @@ async def list_files( Returns: - `path`: full path of the file or directory in Antares Web Server. This path can contain glob patterns (e.g., `*.txt`). - - `file_type`: file type: "folder", "file", "symlink", "socket", "block_device", + - `file_type`: file type: "directory", "file", "symlink", "socket", "block_device", "character_device", "fifo", or "unknown". - - `file_count`: number of files in the directory (1 for files). + - `file_count`: number of files of folders in the directory (1 for files). - `size_bytes`: size of the file or total size of the directory in bytes. - - `created`: creation date of the file or directory (UTC). - - `modified`: last modification date of the file or directory (UTC). - - `accessed`: last access date of the file or directory (UTC). + - `created`: creation date of the file or directory (local time). + - `modified`: last modification date of the file or directory (local time). + - `accessed`: last access date of the file or directory (local time). - `message`: a message describing the status of the file. Possible error codes: diff --git a/tests/integration/filesystem_blueprint/__init__.py b/tests/integration/filesystem_blueprint/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/filesystem_blueprint/test_model.py b/tests/integration/filesystem_blueprint/test_model.py new file mode 100644 index 0000000000..3c21340363 --- /dev/null +++ b/tests/integration/filesystem_blueprint/test_model.py @@ -0,0 +1,145 @@ +import asyncio +import datetime +import re +import shutil +from pathlib import Path + +from antarest.core.filesystem_blueprint import FileInfoDTO, FilesystemDTO, MountPointDTO + + +class TestFilesystemDTO: + def test_init(self) -> None: + example = { + "name": "ws", + "mount_dirs": { + "default": "/path/to/workspaces/internal_studies", + "common": "/path/to/workspaces/common_studies", + }, + } + dto = FilesystemDTO.parse_obj(example) + assert dto.name == example["name"] + assert dto.mount_dirs["default"] == Path(example["mount_dirs"]["default"]) + assert dto.mount_dirs["common"] == Path(example["mount_dirs"]["common"]) + + +class TestMountPointDTO: + def test_init(self) -> None: + example = { + "name": "default", + "path": "/path/to/workspaces/internal_studies", + "total_bytes": 1e9, + "used_bytes": 0.6e9, + "free_bytes": 1e9 - 0.6e9, + "message": f"{0.6e9 / 1e9:%} used", + } + dto = MountPointDTO.parse_obj(example) + assert dto.name == example["name"] + assert dto.path == Path(example["path"]) + assert dto.total_bytes == example["total_bytes"] + assert dto.used_bytes == example["used_bytes"] + assert dto.free_bytes == example["free_bytes"] + assert dto.message == example["message"] + + def test_from_path__missing_file(self) -> None: + name = "foo" + path = Path("/path/to/workspaces/internal_studies") + dto = asyncio.run(MountPointDTO.from_path(name, path)) + assert dto.name == name + assert dto.path == path + assert dto.total_bytes == 0 + assert dto.used_bytes == 0 + assert dto.free_bytes == 0 + assert dto.message.startswith("N/A:"), dto.message + + def test_from_path__file(self, tmp_path: Path) -> None: + name = "foo" + dto = asyncio.run(MountPointDTO.from_path(name, tmp_path)) + total_bytes, used_bytes, free_bytes = shutil.disk_usage(tmp_path) + assert dto.name == name + assert dto.path == tmp_path + assert dto.total_bytes == total_bytes + assert dto.used_bytes == used_bytes + assert dto.free_bytes == free_bytes + assert re.fullmatch(r"\d+(?:\.\d+)?% used", dto.message), dto.message + + +class TestFileInfoDTO: + def test_init(self) -> None: + example = { + "path": "/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares", + "file_type": "file", + "file_count": 1, + "size_bytes": 126, + "created": "2023-12-07T17:59:43", + "modified": "2023-12-07T17:59:43", + "accessed": "2024-01-11T17:54:09", + "message": "OK", + } + dto = FileInfoDTO.parse_obj(example) + assert dto.path == Path(example["path"]) + assert dto.file_type == example["file_type"] + assert dto.file_count == example["file_count"] + assert dto.size_bytes == example["size_bytes"] + assert dto.created == datetime.datetime.fromisoformat(example["created"]) + assert dto.modified == datetime.datetime.fromisoformat(example["modified"]) + assert dto.accessed == datetime.datetime.fromisoformat(example["accessed"]) + assert dto.message == example["message"] + + def test_from_path__missing_file(self) -> None: + path = Path("/path/to/workspaces/internal_studies/5a503c20-24a3-4734-9cf8-89565c9db5ec/study.antares") + dto = asyncio.run(FileInfoDTO.from_path(path)) + assert dto.path == path + assert dto.file_type == "unknown" + assert dto.file_count == 0 + assert dto.size_bytes == 0 + assert dto.created == datetime.datetime.min + assert dto.modified == datetime.datetime.min + assert dto.accessed == datetime.datetime.min + assert dto.message.startswith("N/A:"), dto.message + + def test_from_path__file(self, tmp_path: Path) -> None: + path = tmp_path / "foo.txt" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.write_bytes(b"1234567") # 7 bytes + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path)) + assert dto.path == path + assert dto.file_type == "file" + assert dto.file_count == 1 + assert dto.size_bytes == 7 + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" + + def test_from_path__directory(self, tmp_path: Path) -> None: + path = tmp_path / "foo" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.mkdir() + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path, details=False)) + assert dto.path == path + assert dto.file_type == "directory" + assert dto.file_count == 1 + assert dto.size_bytes == path.stat().st_size + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" + + def test_from_path__directory_with_files(self, tmp_path: Path) -> None: + path = tmp_path / "foo" + before = datetime.datetime.now() - datetime.timedelta(seconds=1) + path.mkdir() + (path / "bar.txt").write_bytes(b"1234567") + (path / "baz.txt").write_bytes(b"890") + after = datetime.datetime.now() + datetime.timedelta(seconds=1) + dto = asyncio.run(FileInfoDTO.from_path(path, details=True)) + assert dto.path == path + assert dto.file_type == "directory" + assert dto.file_count == 3 + assert dto.size_bytes == path.stat().st_size + 10 + assert before <= dto.created <= after + assert before <= dto.modified <= after + assert before <= dto.accessed <= after + assert dto.message == "OK" From eeef439431c4cd5fcc455336ed529f5d78ae092d Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 20:26:08 +0100 Subject: [PATCH 14/26] feat(api-filesystem): add integration tests --- .../test_filesystem_endpoints.py | 424 ++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100644 tests/integration/filesystem_blueprint/test_filesystem_endpoints.py diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py new file mode 100644 index 0000000000..ec1619d53d --- /dev/null +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -0,0 +1,424 @@ +import datetime +import operator +import re +import shutil +import typing as t +from pathlib import Path + +from starlette.testclient import TestClient + +from tests.integration.conftest import RESOURCES_DIR + + +class AnyDiskUsagePercent: + """A helper object that compares equal to any disk usage percentage.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return bool(re.fullmatch(r"\d+(?:\.\d+)?% used", other)) + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + return "" + + +class AnyIsoDateTime: + """A helper object that compares equal to any date time in ISO 8601 format.""" + + def __eq__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + try: + return bool(datetime.datetime.fromisoformat(other)) + except ValueError: + return False + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + return "" + + +class IntegerRange: + """A helper object that compares equal to any integer in a given range.""" + + def __init__(self, start: int, stop: int) -> None: + self.start = start + self.stop = stop + + def __eq__(self, other: object) -> bool: + if not isinstance(other, int): + return NotImplemented + return self.start <= other <= self.stop + + def __ne__(self, other: object) -> bool: + if not isinstance(other, str): + return NotImplemented + return not self.__eq__(other) + + def __repr__(self) -> str: + start = getattr(self, "start", None) + stop = getattr(self, "stop", None) + return f"" + + +# noinspection SpellCheckingInspection +class TestFilesystemEndpoints: + """Test the filesystem endpoints.""" + + def test_lifecycle( + self, + tmp_path: Path, + client: TestClient, + user_access_token: str, + admin_access_token: str, + caplog: t.Any, + ) -> None: + """ + Test the lifecycle of the filesystem endpoints. + + Args: + tmp_path: pytest tmp_path fixture. + caplog: pytest caplog fixture. + client: test client (tests.integration.conftest.client_fixture). + user_access_token: access token of a classic user (tests.integration.conftest.user_access_token_fixture). + admin_access_token: access token of an admin user (tests.integration.conftest.admin_access_token_fixture). + """ + # NOTE: all the following paths are based on the configuration defined in the app_fixture. + archive_dir = tmp_path / "archive_dir" + matrix_dir = tmp_path / "matrix_store" + resource_dir = RESOURCES_DIR + tmp_dir = tmp_path / "tmp" + default_workspace = tmp_path / "internal_workspace" + ext_workspace_path = tmp_path / "ext_workspace" + + expected: t.Union[t.Sequence[t.Mapping[str, t.Any]], t.Mapping[str, t.Any]] + + with caplog.at_level(level="ERROR", logger="antarest.main"): + # Count the number of errors in the caplog + err_count = 0 + + # ================================================== + # Get the list of filesystems and their mount points + # ================================================== + + # Without authentication + res = client.get("/filesystem") + assert res.status_code == 401, res.json() + assert res.json()["detail"] == "Missing cookie access_token_cookie" + # This error generates no log entry + # err_count += 1 + + # With authentication + user_headers = {"Authorization": f"Bearer {user_access_token}"} + res = client.get("/filesystem", headers=user_headers) + assert res.status_code == 200, res.json() + actual = res.json() + expected = [ + { + "name": "cfg", + "mount_dirs": { + "archive": str(archive_dir), + "matrix": str(matrix_dir), + "res": str(resource_dir), + "tmp": str(tmp_dir), + }, + }, + { + "name": "ws", + "mount_dirs": { + "default": str(default_workspace), + "ext": str(ext_workspace_path), + }, + }, + ] + assert actual == expected + + # =================================================================== + # Get the path and the disk usage of the mount points in a filesystem + # =================================================================== + + # Unknown filesystem + res = client.get("/filesystem/foo", headers=user_headers) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Filesystem not found: 'foo'" + err_count += 1 + + # Known filesystem + res = client.get("/filesystem/ws", headers=user_headers) + assert res.status_code == 200, res.json() + actual = sorted(res.json(), key=operator.itemgetter("name")) + # Both mount point are in the same filesystem, which is the `tmp_path` filesystem + total_bytes, used_bytes, free_bytes = shutil.disk_usage(tmp_path) + expected = [ + { + "name": "default", + "path": str(default_workspace), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + }, + { + "name": "ext", + "path": str(ext_workspace_path), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + }, + ] + assert actual == expected + + # ================================================ + # Get the path and the disk usage of a mount point + # ================================================ + + # Unknown mount point + res = client.get("/filesystem/ws/foo", headers=user_headers) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Mount point not found: 'ws/foo'" + err_count += 1 + + res = client.get("/filesystem/ws/default", headers=user_headers) + assert res.status_code == 200, res.json() + actual = res.json() + expected = { + "name": "default", + "path": str(default_workspace), + "total_bytes": total_bytes, + "used_bytes": used_bytes, + "free_bytes": free_bytes, + "message": AnyDiskUsagePercent(), + } + assert actual == expected + + # ========================================= + # List files and directories in a workspace + # ========================================= + + # Listing a workspace with and invalid glob pattern raises an error + res = client.get("/filesystem/ws/default/ls", headers=user_headers, params={"path": "."}) + assert res.status_code == 400, res.json() + assert res.json()["description"].startswith("Invalid path: '.'"), res.json() + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/filesystem/ws/ext/ls", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"].startswith("Access denied to path: '/foo'"), res.json() + err_count += 1 + + # Recursively search all "study.antares" files in the "default" workspace + res = client.get("/filesystem/ws/default/ls", headers=user_headers, params={"path": "**/study.antares"}) + assert res.status_code == 200, res.json() + actual = res.json() + # There is no managed study in the "default" workspace + expected = [] + assert actual == expected + + # Recursively search all "study.antares" files in the "ext" workspace + res = client.get("/filesystem/ws/ext/ls", headers=user_headers, params={"path": "**/study.antares"}) + assert res.status_code == 200, res.json() + actual = res.json() + # There is one external study in the "ext" workspace, which is "STA-mini" + expected = [ + { + "path": str(ext_workspace_path / "STA-mini" / "study.antares"), + "file_type": "file", + "file_count": 1, + "size_bytes": 112, + "created": AnyIsoDateTime(), + "accessed": AnyIsoDateTime(), + "modified": AnyIsoDateTime(), + "message": "OK", + } + ] + assert actual == expected + + # Get the details of the "STA-mini" study + res = client.get( + "/filesystem/ws/ext/ls", + headers=user_headers, + params={"path": "STA-mini", "details": True}, # type: ignore + ) + assert res.status_code == 200, res.json() + actual = res.json() + expected = [ + { + "path": str(ext_workspace_path / "STA-mini"), + "file_type": "directory", + "file_count": IntegerRange(900, 1000), # 918 + "size_bytes": IntegerRange(8_000_000, 9_000_000), # 8_597_683 + "created": AnyIsoDateTime(), + "accessed": AnyIsoDateTime(), + "modified": AnyIsoDateTime(), + "message": "OK", + } + ] + assert actual == expected + + # ======================================================= + # Remove a file or directory recursively from a workspace + # ======================================================= + + # Let's create a dummy file in the "ext" workspace + dummy_file = ext_workspace_path / "dummy.txt" + dummy_file.touch() + + # Normal users are not allowed to remove files + res = client.delete("/filesystem/ws/ext", headers=user_headers, params={"path": "dummy.txt"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "User has no permission to delete files" + err_count += 1 + + # Admin users are allowed to remove files + admin_headers = {"Authorization": f"Bearer {admin_access_token}"} + + # Providing an empty path is not allowed + res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": ""}) + assert res.status_code == 400, res.json() + assert res.json()["description"] == "Empty or missing path parameter" + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "Access denied to path: '/foo'" + err_count += 1 + + # Admin users are allowed to remove files + res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy.txt"}) + assert res.status_code == 200, res.json() + assert not dummy_file.exists() + + # Let's create a dummy directory in the "ext" workspace, and a dummy file in it + dummy_dir = ext_workspace_path / "dummy" + dummy_dir.mkdir() + dummy_file = dummy_dir / "dummy.txt" + dummy_file.touch() + + # Normal users are not allowed to remove files or directories + res = client.delete("/filesystem/ws/ext", headers=user_headers, params={"path": "dummy"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "User has no permission to delete files" + err_count += 1 + + # Admin users are allowed to remove files or directories + res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy"}) + assert res.status_code == 200, res.json() + assert not dummy_dir.exists() + + # ================================= + # View a text file from a workspace + # ================================= + + # Providing an empty path is not allowed + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": ""}) + assert res.status_code == 400, res.json() + assert res.json()["description"] == "Empty or missing path parameter" + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "Access denied to path: '/foo'" + err_count += 1 + + # Providing a directory path is not allowed + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # Let's create a dummy file in the "ext" workspace, and write some text in it + dummy_file = ext_workspace_path / "dummy.txt" + dummy_file.write_text("Hello, world!") + + # Authorized users can view text files + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + assert res.status_code == 200, res.json() + assert res.text == "Hello, world!" + + # If the file is missing, a 404 error is returned + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "missing.txt"}) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Path not found: 'missing.txt'" + err_count += 1 + + # If the file is not a text file, a 417 error is returned + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # If the user choose an unknown encoding, a 417 error is returned + res = client.get( + "/filesystem/ws/ext/cat", + headers=user_headers, + params={"path": "dummy.txt", "encoding": "unknown"}, + ) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Unknown encoding: 'unknown'" + err_count += 1 + + # If the file is not a texte file, a 417 error may be returned + dummy_file.write_bytes(b"\x81\x82\x83") # invalid utf-8 bytes + res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Failed to decode file: 'dummy.txt'" + err_count += 1 + + # ================================ + # Download a file from a workspace + # ================================ + + # Providing an empty path is not allowed + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": ""}) + assert res.status_code == 400, res.json() + assert res.json()["description"] == "Empty or missing path parameter" + err_count += 1 + + # Providing en absolute to an external workspace is not allowed + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "/foo"}) + assert res.status_code == 403, res.json() + assert res.json()["description"] == "Access denied to path: '/foo'" + err_count += 1 + + # Providing a directory path is not allowed + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # Let's create a dummy file in the "ext" workspace, and write some binary data in it + dummy_file = ext_workspace_path / "dummy.bin" + dummy_file.write_bytes(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09") + + # Authorized users can download files + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "dummy.bin"}) + assert res.status_code == 200, res.json() + assert res.content == b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" + + # If the file is missing, a 404 error is returned + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "missing.bin"}) + assert res.status_code == 404, res.json() + assert res.json()["description"] == "Path not found: 'missing.bin'" + err_count += 1 + + # Downloading a directory is not allowed + res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + assert res.status_code == 417, res.json() + assert res.json()["description"] == "Path is not a file: 'STA-mini'" + err_count += 1 + + # At the end of this unit test, the caplog should have errors + assert len(caplog.records) == err_count, caplog.records From e953652fcb36606f74bb2e937e9818738afbdfe1 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 20:26:45 +0100 Subject: [PATCH 15/26] test: correct typing in `conftest.py` --- tests/integration/conftest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a88dc4f9e8..0ac1e1763f 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,7 +6,7 @@ import jinja2 import pytest from fastapi import FastAPI -from sqlalchemy import create_engine +from sqlalchemy import create_engine # type: ignore from starlette.testclient import TestClient from antarest.dbmodel import Base @@ -22,7 +22,7 @@ @pytest.fixture(name="app") -def app_fixture(tmp_path: Path) -> FastAPI: +def app_fixture(tmp_path: Path) -> t.Generator[FastAPI, None, None]: # Currently, it is impossible to use a SQLite database in memory (with "sqlite:///:memory:") # because the database is created by the FastAPI application during each integration test, # which doesn't apply the migrations (migrations are done by Alembic). @@ -96,7 +96,7 @@ def admin_access_token_fixture(client: TestClient) -> str: ) res.raise_for_status() credentials = res.json() - return credentials["access_token"] + return t.cast(str, credentials["access_token"]) @pytest.fixture(name="user_access_token") @@ -117,7 +117,7 @@ def user_access_token_fixture( ) res.raise_for_status() credentials = res.json() - return credentials["access_token"] + return t.cast(str, credentials["access_token"]) @pytest.fixture(name="study_id") @@ -131,5 +131,5 @@ def study_id_fixture( headers={"Authorization": f"Bearer {user_access_token}"}, ) res.raise_for_status() - study_ids = res.json() + study_ids = t.cast(t.Iterable[str], res.json()) return next(iter(study_ids)) From 5243ed5d6a712f5930d6c6cfb514d8f39ce66ac3 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 21:08:30 +0100 Subject: [PATCH 16/26] feat(api-filesystem): correct SonarCloud issue cross-site scripting (XSS) attacks --- antarest/core/filesystem_blueprint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 37fd9ab6c1..402c0ef9d8 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -460,10 +460,10 @@ async def remove_file( if full_path.is_dir(): shutil.rmtree(full_path) - return f"Directory deleted successfully: '{path}'" + return "Directory deleted successfully" elif full_path.is_file(): full_path.unlink() - return f"File deleted successfully: '{path}'" + return "File deleted successfully" else: # pragma: no cover raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") From bbd250ac3d6ad38a7e5fb464cffcd12e360a2ea2 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Fri, 12 Jan 2024 22:51:32 +0100 Subject: [PATCH 17/26] feat(api-filesystem): correct issue with unit tests on Windows --- antarest/core/filesystem_blueprint.py | 2 +- .../filesystem_blueprint/test_filesystem_endpoints.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 402c0ef9d8..b2b4c3d3ee 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -85,7 +85,7 @@ class MountPointDTO( @classmethod async def from_path(cls, name: str, path: Path) -> "MountPointDTO": - obj = cls(name=name, path=path.resolve()) + obj = cls(name=name, path=path) try: obj.total_bytes, obj.used_bytes, obj.free_bytes = shutil.disk_usage(obj.path) except OSError as exc: diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py index ec1619d53d..76aedf2146 100644 --- a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -257,7 +257,7 @@ def test_lifecycle( "path": str(ext_workspace_path / "STA-mini"), "file_type": "directory", "file_count": IntegerRange(900, 1000), # 918 - "size_bytes": IntegerRange(8_000_000, 9_000_000), # 8_597_683 + "size_bytes": IntegerRange(7_000_000, 9_000_000), # nt: 7_741_619, posix: 8_597_683 "created": AnyIsoDateTime(), "accessed": AnyIsoDateTime(), "modified": AnyIsoDateTime(), From df0f1820cca0a16d07943137c96f8f699781fd2d Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sat, 13 Jan 2024 10:52:34 +0100 Subject: [PATCH 18/26] feat(api-filesystem): reorder endpoints and update Swagger documentation --- antarest/core/filesystem_blueprint.py | 104 +++++++++++++------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index b2b4c3d3ee..82b48c33a4 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -359,7 +359,7 @@ async def get_mount_point(fs: FilesystemName, mount: MountPointName) -> MountPoi @bp.get( "/{fs}/{mount}/ls", - summary="List files in a workspace", + summary="List files in a mount point", response_model=t.Sequence[FileInfoDTO], ) async def list_files( @@ -369,7 +369,7 @@ async def list_files( details: bool = False, ) -> t.Sequence[FileInfoDTO]: """ - List files and directories in a workspace. + List files and directories in a mount point. Args: - `fs`: The name of the filesystem: "cfg" or "ws". @@ -421,55 +421,9 @@ async def list_files( file_info_list = await asyncio.gather(*tasks) return file_info_list - @bp.delete( - "/{fs}/{mount}", - summary="Remove a file or directory recursively from a workspace", - response_model=str, - ) - async def remove_file( - fs: FilesystemName, - mount: MountPointName, - path: str = "", - current_user: JWTUser = Depends(auth.get_current_user), - ) -> str: - """ - Remove a file or directory recursively from a workspace. - - > **⚠ Warning:** This endpoint is reserved for site administrators. - - Args: - - `fs`: The name of the filesystem: "cfg" or "ws". - - `mount`: The name of the mount point. - - `path`: The relative path of the file or directory to delete. - - Returns: - - Success message (e.g., "File deleted successfully"). - - Possible error codes: - - 400 Bad Request: If the specified path is missing (empty). - - 403 Forbidden: If the user has no permission to delete the file or directory. - - 404 Not Found: If the specified filesystem, mount point, file or directory doesn't exist. - - 417 Expectation Failed: If the specified path is not a file or directory. - """ - - if not current_user.is_site_admin(): - raise HTTPException(status_code=403, detail="User has no permission to delete files") - - mount_dir = _get_mount_dir(fs, mount) - full_path = _get_full_path(mount_dir, path) - - if full_path.is_dir(): - shutil.rmtree(full_path) - return "Directory deleted successfully" - elif full_path.is_file(): - full_path.unlink() - return "File deleted successfully" - else: # pragma: no cover - raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") - @bp.get( "/{fs}/{mount}/cat", - summary="View a text file from a workspace", + summary="View a text file from a mount point", response_class=PlainTextResponse, response_description="File content as text", ) @@ -481,7 +435,7 @@ async def view_file( ) -> str: # noinspection SpellCheckingInspection """ - View a text file from a workspace. + View a text file from a mount point. Examples: ```ini @@ -528,7 +482,7 @@ async def view_file( @bp.get( "/{fs}/{mount}/download", - summary="Download a file from a workspace", + summary="Download a file from a mount point", response_class=StreamingResponse, response_description="File content as binary", ) @@ -538,7 +492,7 @@ async def download_file( path: str = "", ) -> StreamingResponse: """ - Download a file from a workspace. + Download a file from a mount point. > **Note:** Directory download is not supported yet. @@ -575,4 +529,50 @@ def iter_file() -> t.Iterator[bytes]: else: # pragma: no cover raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + @bp.delete( + "/{fs}/{mount}", + summary="Remove a file or directory recursively from a mount point", + response_model=str, + ) + async def remove_file( + fs: FilesystemName, + mount: MountPointName, + path: str = "", + current_user: JWTUser = Depends(auth.get_current_user), + ) -> str: + """ + Remove a file or directory recursively from a mount point. + + > **⚠ Warning:** This endpoint is reserved for site administrators. + + Args: + - `fs`: The name of the filesystem: "cfg" or "ws". + - `mount`: The name of the mount point. + - `path`: The relative path of the file or directory to delete. + + Returns: + - Success message (e.g., "File deleted successfully"). + + Possible error codes: + - 400 Bad Request: If the specified path is missing (empty). + - 403 Forbidden: If the user has no permission to delete the file or directory. + - 404 Not Found: If the specified filesystem, mount point, file or directory doesn't exist. + - 417 Expectation Failed: If the specified path is not a file or directory. + """ + + if not current_user.is_site_admin(): + raise HTTPException(status_code=403, detail="User has no permission to delete files") + + mount_dir = _get_mount_dir(fs, mount) + full_path = _get_full_path(mount_dir, path) + + if full_path.is_dir(): + shutil.rmtree(full_path) + return "Directory successfully deleted" + elif full_path.is_file(): + full_path.unlink() + return "File successfully deleted" + else: # pragma: no cover + raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") + return bp From 53923ca7f6919ed8e738d4bac6132290c3222bdc Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sat, 13 Jan 2024 14:40:48 +0100 Subject: [PATCH 19/26] test(api-filesystem): demonstrates how to compute the size of all studies --- .../test_filesystem_endpoints.py | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py index 76aedf2146..352f160133 100644 --- a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -77,10 +77,10 @@ class TestFilesystemEndpoints: def test_lifecycle( self, tmp_path: Path, + caplog: t.Any, client: TestClient, user_access_token: str, admin_access_token: str, - caplog: t.Any, ) -> None: """ Test the lifecycle of the filesystem endpoints. @@ -422,3 +422,51 @@ def test_lifecycle( # At the end of this unit test, the caplog should have errors assert len(caplog.records) == err_count, caplog.records + + def test_size_of_studies( + self, + client: TestClient, + user_access_token: str, + caplog: t.Any, + ): + """ + This test demonstrates how to compute the size of all studies. + + - First, we get the list of studies using the `/v1/studies` endpoint. + - Then, we get the size of each study using the `/filesystem/ws/{workspace}/ls` endpoint, + with the `details` parameter set to `True`. + """ + user_headers = {"Authorization": f"Bearer {user_access_token}"} + + # For this demo, we can disable the logs + with caplog.at_level(level="CRITICAL", logger="antarest.main"): + # Create a new study in the "default" workspace for this demo + res = client.post( + "/v1/studies", + headers=user_headers, + params={"name": "New Study", "version": "860"}, + ) + res.raise_for_status() + + # Get the list of studies from all workspaces + res = client.get("/v1/studies", headers=user_headers) + res.raise_for_status() + actual = res.json() + + # Get the size of each study + sizes = [] + for study in actual.values(): + res = client.get( + f"/filesystem/ws/{study['workspace']}/ls", + headers=user_headers, + params={"path": study["folder"], "details": True}, + ) + res.raise_for_status() + actual = res.json() + sizes.append(actual[0]["size_bytes"]) + + # Check the sizes + # The size of the new study should be between 200 and 300 KB. + # The suze of 'STA-mini' should be between 7 and 9 MB. + sizes.sort() + assert sizes == [IntegerRange(200_000, 300_000), IntegerRange(7_000_000, 9_000_000)] From 862c27c15ad066c3c90c5f8d12ffe82654dbe6d7 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sat, 13 Jan 2024 14:43:22 +0100 Subject: [PATCH 20/26] feat(api-filesystem): change the route prefix to use "/v1/filesystem" --- antarest/core/filesystem_blueprint.py | 2 +- .../test_filesystem_endpoints.py | 66 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 82b48c33a4..f0075e10f8 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -238,7 +238,7 @@ def create_file_system_blueprint(config: Config) -> APIRouter: """ auth = Auth(config) bp = APIRouter( - prefix="/filesystem", + prefix="/v1/filesystem", tags=[APITag.filesystem], dependencies=[Depends(auth.get_current_user)], include_in_schema=True, # but may be disabled in the future diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py index 352f160133..f1622dc4c0 100644 --- a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -111,7 +111,7 @@ def test_lifecycle( # ================================================== # Without authentication - res = client.get("/filesystem") + res = client.get("/v1/filesystem") assert res.status_code == 401, res.json() assert res.json()["detail"] == "Missing cookie access_token_cookie" # This error generates no log entry @@ -119,7 +119,7 @@ def test_lifecycle( # With authentication user_headers = {"Authorization": f"Bearer {user_access_token}"} - res = client.get("/filesystem", headers=user_headers) + res = client.get("/v1/filesystem", headers=user_headers) assert res.status_code == 200, res.json() actual = res.json() expected = [ @@ -147,13 +147,13 @@ def test_lifecycle( # =================================================================== # Unknown filesystem - res = client.get("/filesystem/foo", headers=user_headers) + res = client.get("/v1/filesystem/foo", headers=user_headers) assert res.status_code == 404, res.json() assert res.json()["description"] == "Filesystem not found: 'foo'" err_count += 1 # Known filesystem - res = client.get("/filesystem/ws", headers=user_headers) + res = client.get("/v1/filesystem/ws", headers=user_headers) assert res.status_code == 200, res.json() actual = sorted(res.json(), key=operator.itemgetter("name")) # Both mount point are in the same filesystem, which is the `tmp_path` filesystem @@ -183,12 +183,12 @@ def test_lifecycle( # ================================================ # Unknown mount point - res = client.get("/filesystem/ws/foo", headers=user_headers) + res = client.get("/v1/filesystem/ws/foo", headers=user_headers) assert res.status_code == 404, res.json() assert res.json()["description"] == "Mount point not found: 'ws/foo'" err_count += 1 - res = client.get("/filesystem/ws/default", headers=user_headers) + res = client.get("/v1/filesystem/ws/default", headers=user_headers) assert res.status_code == 200, res.json() actual = res.json() expected = { @@ -206,19 +206,19 @@ def test_lifecycle( # ========================================= # Listing a workspace with and invalid glob pattern raises an error - res = client.get("/filesystem/ws/default/ls", headers=user_headers, params={"path": "."}) + res = client.get("/v1/filesystem/ws/default/ls", headers=user_headers, params={"path": "."}) assert res.status_code == 400, res.json() assert res.json()["description"].startswith("Invalid path: '.'"), res.json() err_count += 1 # Providing en absolute to an external workspace is not allowed - res = client.get("/filesystem/ws/ext/ls", headers=user_headers, params={"path": "/foo"}) + res = client.get("/v1/filesystem/ws/ext/ls", headers=user_headers, params={"path": "/foo"}) assert res.status_code == 403, res.json() assert res.json()["description"].startswith("Access denied to path: '/foo'"), res.json() err_count += 1 # Recursively search all "study.antares" files in the "default" workspace - res = client.get("/filesystem/ws/default/ls", headers=user_headers, params={"path": "**/study.antares"}) + res = client.get("/v1/filesystem/ws/default/ls", headers=user_headers, params={"path": "**/study.antares"}) assert res.status_code == 200, res.json() actual = res.json() # There is no managed study in the "default" workspace @@ -226,7 +226,7 @@ def test_lifecycle( assert actual == expected # Recursively search all "study.antares" files in the "ext" workspace - res = client.get("/filesystem/ws/ext/ls", headers=user_headers, params={"path": "**/study.antares"}) + res = client.get("/v1/filesystem/ws/ext/ls", headers=user_headers, params={"path": "**/study.antares"}) assert res.status_code == 200, res.json() actual = res.json() # There is one external study in the "ext" workspace, which is "STA-mini" @@ -246,7 +246,7 @@ def test_lifecycle( # Get the details of the "STA-mini" study res = client.get( - "/filesystem/ws/ext/ls", + "/v1/filesystem/ws/ext/ls", headers=user_headers, params={"path": "STA-mini", "details": True}, # type: ignore ) @@ -275,7 +275,7 @@ def test_lifecycle( dummy_file.touch() # Normal users are not allowed to remove files - res = client.delete("/filesystem/ws/ext", headers=user_headers, params={"path": "dummy.txt"}) + res = client.delete("/v1/filesystem/ws/ext", headers=user_headers, params={"path": "dummy.txt"}) assert res.status_code == 403, res.json() assert res.json()["description"] == "User has no permission to delete files" err_count += 1 @@ -284,19 +284,19 @@ def test_lifecycle( admin_headers = {"Authorization": f"Bearer {admin_access_token}"} # Providing an empty path is not allowed - res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": ""}) + res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": ""}) assert res.status_code == 400, res.json() assert res.json()["description"] == "Empty or missing path parameter" err_count += 1 # Providing en absolute to an external workspace is not allowed - res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "/foo"}) + res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "/foo"}) assert res.status_code == 403, res.json() assert res.json()["description"] == "Access denied to path: '/foo'" err_count += 1 # Admin users are allowed to remove files - res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy.txt"}) + res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy.txt"}) assert res.status_code == 200, res.json() assert not dummy_file.exists() @@ -307,13 +307,13 @@ def test_lifecycle( dummy_file.touch() # Normal users are not allowed to remove files or directories - res = client.delete("/filesystem/ws/ext", headers=user_headers, params={"path": "dummy"}) + res = client.delete("/v1/filesystem/ws/ext", headers=user_headers, params={"path": "dummy"}) assert res.status_code == 403, res.json() assert res.json()["description"] == "User has no permission to delete files" err_count += 1 # Admin users are allowed to remove files or directories - res = client.delete("/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy"}) + res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy"}) assert res.status_code == 200, res.json() assert not dummy_dir.exists() @@ -322,19 +322,19 @@ def test_lifecycle( # ================================= # Providing an empty path is not allowed - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": ""}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": ""}) assert res.status_code == 400, res.json() assert res.json()["description"] == "Empty or missing path parameter" err_count += 1 # Providing en absolute to an external workspace is not allowed - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "/foo"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "/foo"}) assert res.status_code == 403, res.json() assert res.json()["description"] == "Access denied to path: '/foo'" err_count += 1 # Providing a directory path is not allowed - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) assert res.status_code == 417, res.json() assert res.json()["description"] == "Path is not a file: 'STA-mini'" err_count += 1 @@ -344,25 +344,25 @@ def test_lifecycle( dummy_file.write_text("Hello, world!") # Authorized users can view text files - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) assert res.status_code == 200, res.json() assert res.text == "Hello, world!" # If the file is missing, a 404 error is returned - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "missing.txt"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "missing.txt"}) assert res.status_code == 404, res.json() assert res.json()["description"] == "Path not found: 'missing.txt'" err_count += 1 # If the file is not a text file, a 417 error is returned - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "STA-mini"}) assert res.status_code == 417, res.json() assert res.json()["description"] == "Path is not a file: 'STA-mini'" err_count += 1 # If the user choose an unknown encoding, a 417 error is returned res = client.get( - "/filesystem/ws/ext/cat", + "/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt", "encoding": "unknown"}, ) @@ -372,7 +372,7 @@ def test_lifecycle( # If the file is not a texte file, a 417 error may be returned dummy_file.write_bytes(b"\x81\x82\x83") # invalid utf-8 bytes - res = client.get("/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) + res = client.get("/v1/filesystem/ws/ext/cat", headers=user_headers, params={"path": "dummy.txt"}) assert res.status_code == 417, res.json() assert res.json()["description"] == "Failed to decode file: 'dummy.txt'" err_count += 1 @@ -382,19 +382,19 @@ def test_lifecycle( # ================================ # Providing an empty path is not allowed - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": ""}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": ""}) assert res.status_code == 400, res.json() assert res.json()["description"] == "Empty or missing path parameter" err_count += 1 # Providing en absolute to an external workspace is not allowed - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "/foo"}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "/foo"}) assert res.status_code == 403, res.json() assert res.json()["description"] == "Access denied to path: '/foo'" err_count += 1 # Providing a directory path is not allowed - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) assert res.status_code == 417, res.json() assert res.json()["description"] == "Path is not a file: 'STA-mini'" err_count += 1 @@ -404,18 +404,18 @@ def test_lifecycle( dummy_file.write_bytes(b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09") # Authorized users can download files - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "dummy.bin"}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "dummy.bin"}) assert res.status_code == 200, res.json() assert res.content == b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09" # If the file is missing, a 404 error is returned - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "missing.bin"}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "missing.bin"}) assert res.status_code == 404, res.json() assert res.json()["description"] == "Path not found: 'missing.bin'" err_count += 1 # Downloading a directory is not allowed - res = client.get("/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) + res = client.get("/v1/filesystem/ws/ext/download", headers=user_headers, params={"path": "STA-mini"}) assert res.status_code == 417, res.json() assert res.json()["description"] == "Path is not a file: 'STA-mini'" err_count += 1 @@ -433,7 +433,7 @@ def test_size_of_studies( This test demonstrates how to compute the size of all studies. - First, we get the list of studies using the `/v1/studies` endpoint. - - Then, we get the size of each study using the `/filesystem/ws/{workspace}/ls` endpoint, + - Then, we get the size of each study using the `/v1/filesystem/ws/{workspace}/ls` endpoint, with the `details` parameter set to `True`. """ user_headers = {"Authorization": f"Bearer {user_access_token}"} @@ -457,7 +457,7 @@ def test_size_of_studies( sizes = [] for study in actual.values(): res = client.get( - f"/filesystem/ws/{study['workspace']}/ls", + f"/v1/filesystem/ws/{study['workspace']}/ls", headers=user_headers, params={"path": study["folder"], "details": True}, ) From 7d0a5f1345b6dc2db2d34eb444cd05914a71f5cd Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Sat, 13 Jan 2024 14:58:36 +0100 Subject: [PATCH 21/26] style(api-filesystem): use `te` alias to import `typing_extensions` --- antarest/core/filesystem_blueprint.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index f0075e10f8..48366a4070 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -9,7 +9,7 @@ import typing as t from pathlib import Path -import typing_extensions +import typing_extensions as te from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Extra, Field from starlette.responses import PlainTextResponse, StreamingResponse @@ -19,8 +19,8 @@ from antarest.core.utils.web import APITag from antarest.login.auth import Auth -FilesystemName = typing_extensions.Annotated[str, Field(regex=r"^\w+$", description="Filesystem name")] -MountPointName = typing_extensions.Annotated[str, Field(regex=r"^\w+$", description="Mount point name")] +FilesystemName = te.Annotated[str, Field(regex=r"^\w+$", description="Filesystem name")] +MountPointName = te.Annotated[str, Field(regex=r"^\w+$", description="Mount point name")] class FilesystemDTO( From 87ab4c81dd9ea32a095ed70fd850036a16155a40 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 15 Jan 2024 10:58:42 +0100 Subject: [PATCH 22/26] feat(api-filesystem): correct issue with unit tests on Windows --- .../filesystem_blueprint/test_filesystem_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py index f1622dc4c0..c56cb8ad4f 100644 --- a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -466,7 +466,7 @@ def test_size_of_studies( sizes.append(actual[0]["size_bytes"]) # Check the sizes - # The size of the new study should be between 200 and 300 KB. + # The size of the new study should be between 140 and 300 KB. # The suze of 'STA-mini' should be between 7 and 9 MB. sizes.sort() - assert sizes == [IntegerRange(200_000, 300_000), IntegerRange(7_000_000, 9_000_000)] + assert sizes == [IntegerRange(140_000, 300_000), IntegerRange(7_000_000, 9_000_000)] From ceb39a6c6a31293e858dc1040aa7d33e6d6a4695 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 15 Jan 2024 17:09:59 +0100 Subject: [PATCH 23/26] feat(api-filesystem): abandon the `delete` functionality --- antarest/core/filesystem_blueprint.py | 49 +----------------- .../test_filesystem_endpoints.py | 51 ------------------- 2 files changed, 1 insertion(+), 99 deletions(-) diff --git a/antarest/core/filesystem_blueprint.py b/antarest/core/filesystem_blueprint.py index 48366a4070..3b15ebd03a 100644 --- a/antarest/core/filesystem_blueprint.py +++ b/antarest/core/filesystem_blueprint.py @@ -224,8 +224,7 @@ def create_file_system_blueprint(config: Config) -> APIRouter: workspaces (especially the "default" workspace of Antares Web), and the different storage directories (`tmp`, `matrixstore`, `archive`, etc. defined in the configuration file). - This blueprint is also used to list files and directories, to download a file, - and to delete a file or directory. + This blueprint is also used to list files and directories, and to view or download a file. Reading files is allowed for authenticated users, but deleting files is reserved for site administrators. @@ -529,50 +528,4 @@ def iter_file() -> t.Iterator[bytes]: else: # pragma: no cover raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") - @bp.delete( - "/{fs}/{mount}", - summary="Remove a file or directory recursively from a mount point", - response_model=str, - ) - async def remove_file( - fs: FilesystemName, - mount: MountPointName, - path: str = "", - current_user: JWTUser = Depends(auth.get_current_user), - ) -> str: - """ - Remove a file or directory recursively from a mount point. - - > **⚠ Warning:** This endpoint is reserved for site administrators. - - Args: - - `fs`: The name of the filesystem: "cfg" or "ws". - - `mount`: The name of the mount point. - - `path`: The relative path of the file or directory to delete. - - Returns: - - Success message (e.g., "File deleted successfully"). - - Possible error codes: - - 400 Bad Request: If the specified path is missing (empty). - - 403 Forbidden: If the user has no permission to delete the file or directory. - - 404 Not Found: If the specified filesystem, mount point, file or directory doesn't exist. - - 417 Expectation Failed: If the specified path is not a file or directory. - """ - - if not current_user.is_site_admin(): - raise HTTPException(status_code=403, detail="User has no permission to delete files") - - mount_dir = _get_mount_dir(fs, mount) - full_path = _get_full_path(mount_dir, path) - - if full_path.is_dir(): - shutil.rmtree(full_path) - return "Directory successfully deleted" - elif full_path.is_file(): - full_path.unlink() - return "File successfully deleted" - else: # pragma: no cover - raise HTTPException(status_code=417, detail=f"Unknown file type: '{path}'") - return bp diff --git a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py index c56cb8ad4f..a547313896 100644 --- a/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py +++ b/tests/integration/filesystem_blueprint/test_filesystem_endpoints.py @@ -266,57 +266,6 @@ def test_lifecycle( ] assert actual == expected - # ======================================================= - # Remove a file or directory recursively from a workspace - # ======================================================= - - # Let's create a dummy file in the "ext" workspace - dummy_file = ext_workspace_path / "dummy.txt" - dummy_file.touch() - - # Normal users are not allowed to remove files - res = client.delete("/v1/filesystem/ws/ext", headers=user_headers, params={"path": "dummy.txt"}) - assert res.status_code == 403, res.json() - assert res.json()["description"] == "User has no permission to delete files" - err_count += 1 - - # Admin users are allowed to remove files - admin_headers = {"Authorization": f"Bearer {admin_access_token}"} - - # Providing an empty path is not allowed - res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": ""}) - assert res.status_code == 400, res.json() - assert res.json()["description"] == "Empty or missing path parameter" - err_count += 1 - - # Providing en absolute to an external workspace is not allowed - res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "/foo"}) - assert res.status_code == 403, res.json() - assert res.json()["description"] == "Access denied to path: '/foo'" - err_count += 1 - - # Admin users are allowed to remove files - res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy.txt"}) - assert res.status_code == 200, res.json() - assert not dummy_file.exists() - - # Let's create a dummy directory in the "ext" workspace, and a dummy file in it - dummy_dir = ext_workspace_path / "dummy" - dummy_dir.mkdir() - dummy_file = dummy_dir / "dummy.txt" - dummy_file.touch() - - # Normal users are not allowed to remove files or directories - res = client.delete("/v1/filesystem/ws/ext", headers=user_headers, params={"path": "dummy"}) - assert res.status_code == 403, res.json() - assert res.json()["description"] == "User has no permission to delete files" - err_count += 1 - - # Admin users are allowed to remove files or directories - res = client.delete("/v1/filesystem/ws/ext", headers=admin_headers, params={"path": "dummy"}) - assert res.status_code == 200, res.json() - assert not dummy_dir.exists() - # ================================= # View a text file from a workspace # ================================= From 57e7259dfdae41038d6aaddfaef8043643f5c169 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:38:50 +0100 Subject: [PATCH 24/26] feat(ui-config): enhance Scenario Playlist loading --- .../dialogs/ScenarioPlaylistDialog/index.tsx | 107 +++++++++--------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx index b90ba5771a..832f7cac08 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/General/dialogs/ScenarioPlaylistDialog/index.tsx @@ -9,7 +9,6 @@ import { StudyMetadata } from "../../../../../../../../common/types"; import usePromise from "../../../../../../../../hooks/usePromise"; import BasicDialog from "../../../../../../../common/dialogs/BasicDialog"; import TableForm from "../../../../../../../common/TableForm"; -import BackdropLoading from "../../../../../../../common/loaders/BackdropLoading"; import UsePromiseCond from "../../../../../../../common/utils/UsePromiseCond"; import { DEFAULT_WEIGHT, @@ -78,58 +77,60 @@ function ScenarioPlaylistDialog(props: Props) { //////////////////////////////////////////////////////////////// return ( - } - ifResolved={(defaultValues) => ( - {t("button.close")}} - // TODO: add `maxHeight` and `fullHeight` in BasicDialog` - PaperProps={{ sx: { height: 500 } }} - maxWidth="sm" - > - - - - - - - - - `MC Year ${row.id}`, - tableRef, - stretchH: "all", - className: "htCenter", - cells: handleCellsRender, - }} - /> - - )} - /> + {t("button.close")}} + // TODO: add `maxHeight` and `fullHeight` in BasicDialog` + PaperProps={{ sx: { height: 500 } }} + maxWidth="sm" + fullWidth + > + ( + <> + + + + + + + + + `MC Year ${row.id}`, + tableRef, + stretchH: "all", + className: "htCenter", + cells: handleCellsRender, + }} + /> + + )} + /> + ); } From f88db1a46b5c8e6a2fd71ca88629d4ff5497e603 Mon Sep 17 00:00:00 2001 From: Samir Kamal <1954121+skamril@users.noreply.github.com> Date: Mon, 15 Jan 2024 17:43:16 +0100 Subject: [PATCH 25/26] fix(ui-config): remove unused item menu --- .../Configuration/RegionalDistricts/index.tsx | 7 -- .../explore/Configuration/index.tsx | 29 ++++---- .../common/page/UnderConstruction.tsx | 66 ------------------- 3 files changed, 12 insertions(+), 90 deletions(-) delete mode 100644 webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx delete mode 100644 webapp/src/components/common/page/UnderConstruction.tsx diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx deleted file mode 100644 index 025121ed72..0000000000 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/RegionalDistricts/index.tsx +++ /dev/null @@ -1,7 +0,0 @@ -import UnderConstruction from "../../../../../common/page/UnderConstruction"; - -function RegionalDistricts() { - return ; -} - -export default RegionalDistricts; diff --git a/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx b/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx index bd56ed29fc..2645b4aba4 100644 --- a/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx +++ b/webapp/src/components/App/Singlestudy/explore/Configuration/index.tsx @@ -5,7 +5,6 @@ import { useMemo, useState } from "react"; import { useOutletContext } from "react-router"; import { useTranslation } from "react-i18next"; import { StudyMetadata } from "../../../../../common/types"; -import UnderConstruction from "../../../../common/page/UnderConstruction"; import PropertiesView from "../../../../common/PropertiesView"; import SplitLayoutView from "../../../../common/SplitLayoutView"; import ListElement from "../common/ListElement"; @@ -13,7 +12,6 @@ import AdequacyPatch from "./AdequacyPatch"; import AdvancedParameters from "./AdvancedParameters"; import General from "./General"; import Optimization from "./Optimization"; -import RegionalDistricts from "./RegionalDistricts"; import TimeSeriesManagement from "./TimeSeriesManagement"; import TableMode from "../../../../common/TableMode"; @@ -28,13 +26,12 @@ function Configuration() { [ { id: 0, name: "General" }, { id: 1, name: "Time-series management" }, - { id: 2, name: "Regional districts" }, - { id: 3, name: "Optimization preferences" }, - Number(study.version) >= 830 && { id: 4, name: "Adequacy Patch" }, - { id: 5, name: "Advanced parameters" }, - { id: 6, name: t("study.configuration.economicOpt") }, - { id: 7, name: t("study.configuration.geographicTrimmingAreas") }, - { id: 8, name: t("study.configuration.geographicTrimmingLinks") }, + { id: 2, name: "Optimization preferences" }, + Number(study.version) >= 830 && { id: 3, name: "Adequacy Patch" }, + { id: 4, name: "Advanced parameters" }, + { id: 5, name: t("study.configuration.economicOpt") }, + { id: 6, name: t("study.configuration.geographicTrimmingAreas") }, + { id: 7, name: t("study.configuration.geographicTrimmingLinks") }, ].filter(Boolean), [study.version, t], ); @@ -59,13 +56,11 @@ function Configuration() { {R.cond([ [R.equals(0), () => ], [R.equals(1), () => ], - [R.equals(1), () => ], - [R.equals(2), () => ], - [R.equals(3), () => ], - [R.equals(4), () => ], - [R.equals(5), () => ], + [R.equals(2), () => ], + [R.equals(3), () => ], + [R.equals(4), () => ], [ - R.equals(6), + R.equals(5), () => ( ( ( - - - {t("common.underConstruction")} - - {previewImage ? ( - - - - PREVIEW - - - preview - - - - ) : undefined} - - ); -} - -UnderConstruction.defaultProps = { - previewImage: undefined, -}; - -export default UnderConstruction; From 17feb4b7e7d4dfd3325b8bd44607ebe1f37a8041 Mon Sep 17 00:00:00 2001 From: Laurent LAPORTE Date: Mon, 15 Jan 2024 18:12:59 +0100 Subject: [PATCH 26/26] build: new bug fix release v2.16.3 (2024-01-17) --- antarest/__init__.py | 4 ++-- docs/CHANGELOG.md | 27 +++++++++++++++++++++++++++ setup.py | 2 +- sonar-project.properties | 2 +- webapp/package-lock.json | 4 ++-- webapp/package.json | 2 +- 6 files changed, 34 insertions(+), 7 deletions(-) diff --git a/antarest/__init__.py b/antarest/__init__.py index 6a92596e63..136deb4f14 100644 --- a/antarest/__init__.py +++ b/antarest/__init__.py @@ -7,9 +7,9 @@ # Standard project metadata -__version__ = "2.16.2" +__version__ = "2.16.3" __author__ = "RTE, Antares Web Team" -__date__ = "2024-01-10" +__date__ = "2024-01-17" # noinspection SpellCheckingInspection __credits__ = "(c) Réseau de Transport de l’Électricité (RTE)" diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 15fa0dedad..02a27f5a98 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -1,6 +1,33 @@ Antares Web Changelog ===================== +v2.16.3 (2024-01-17) +-------------------- + +### Features + +* **api-filesystem:** add new API endpoints to manage filesystem and get disk usage [`#1895`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1895) +* **ui-district:** enhance Scenario Playlist loading and remove Regional District menu [`#1897`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1897) + + +### Bug Fixes + +* **config:** use "CONG. PROB" for thematic trimming (fix typo) [`#1893`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1893) +* **ui-debug:** correct debug view for JSON configuration [`#1898`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1898) +* **ui-debug:** correct debug view for textual matrices [`#1896`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1896) +* **ui-hydro:** add areas encoding to hydro tabs path [`#1894`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1894) + + +### Refactoring + +* **ui-debug:** code review [`#1892`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1892) + + +### Continuous Integration + +* **docker:** add the `ll` alias in `.bashrc` to simplify debugging [`#1880`](https://github.com/AntaresSimulatorTeam/AntaREST/pull/1880) + + v2.16.2 (2024-01-10) -------------------- diff --git a/setup.py b/setup.py index db663998a7..d3007244b0 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="AntaREST", - version="2.16.2", + version="2.16.3", description="Antares Server", long_description=Path("README.md").read_text(encoding="utf-8"), long_description_content_type="text/markdown", diff --git a/sonar-project.properties b/sonar-project.properties index 770a971f0c..384c692e84 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -6,5 +6,5 @@ sonar.exclusions=antarest/gui.py,antarest/main.py sonar.python.coverage.reportPaths=coverage.xml sonar.python.version=3.8 sonar.javascript.lcov.reportPaths=webapp/coverage/lcov.info -sonar.projectVersion=2.16.2 +sonar.projectVersion=2.16.3 sonar.coverage.exclusions=antarest/gui.py,antarest/main.py,antarest/singleton_services.py,antarest/worker/archive_worker_service.py,webapp/**/* \ No newline at end of file diff --git a/webapp/package-lock.json b/webapp/package-lock.json index fdeb700d83..0258965201 100644 --- a/webapp/package-lock.json +++ b/webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "dependencies": { "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", diff --git a/webapp/package.json b/webapp/package.json index 1fd7eea24d..76c1838ef0 100644 --- a/webapp/package.json +++ b/webapp/package.json @@ -1,6 +1,6 @@ { "name": "antares-web", - "version": "2.16.2", + "version": "2.16.3", "private": true, "engines": { "node": "18.16.1"