From 5f0e15af7c046eb99910972d3c3c82d2ed23d101 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Tue, 25 Jul 2023 11:45:48 +0200 Subject: [PATCH 1/3] feat(watcher): scanning a managed workspace only logs a warning msg --- antarest/study/storage/rawstudy/watcher.py | 5 +++- tests/integration/test_integration_watcher.py | 30 +++++++++++++------ 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/antarest/study/storage/rawstudy/watcher.py b/antarest/study/storage/rawstudy/watcher.py index 0d6e85045b..05ff203d88 100644 --- a/antarest/study/storage/rawstudy/watcher.py +++ b/antarest/study/storage/rawstudy/watcher.py @@ -179,7 +179,10 @@ def oneshot_scan( """ def scan_task(notifier: TaskUpdateNotifier) -> TaskResult: - self.scan(workspace, path) + try: + self.scan(workspace, path) + except CannotScanInternalWorkspace as e: + logger.warning(e) return TaskResult(success=True, message="Scan completed") return self.task_service.add_task( diff --git a/tests/integration/test_integration_watcher.py b/tests/integration/test_integration_watcher.py index b5da0a2d18..ac115755ce 100644 --- a/tests/integration/test_integration_watcher.py +++ b/tests/integration/test_integration_watcher.py @@ -1,20 +1,32 @@ from fastapi import FastAPI from starlette.testclient import TestClient +from antarest.core.tasks.model import TaskStatus -def test_integration_xpansion(app: FastAPI, tmp_path: str): - client = TestClient(app, raise_server_exceptions=False) + +def test_integration_watcher(app: FastAPI, tmp_path: str): + client = TestClient(app) res = client.post( "/v1/login", json={"username": "admin", "password": "admin"} ) admin_credentials = res.json() headers = {"Authorization": f'Bearer {admin_credentials["access_token"]}'} - client.post( - f"/v1/watcher/_scan", - headers=headers, - ) - client.post( - f"/v1/watcher/_scan?path=/tmp", + res = client.post("/v1/watcher/_scan", headers=headers) + assert res.status_code == 422 + assert res.json() == { + "description": "field required", + "exception": "RequestValidationError", + "body": None, + } + + task_id = client.post( + "/v1/watcher/_scan?path=/default", headers=headers, - ) + ).json() + + # asserts that scanning default workspace doesn't fail + res = client.get( + f"v1/tasks/{task_id}?wait_for_completion=true", headers=headers + ).json() + assert res["status"] == TaskStatus.COMPLETED.value From d2c02a80e4ab19e0431a301346a4af234076d132 Mon Sep 17 00:00:00 2001 From: belthlemar Date: Thu, 31 Aug 2023 14:47:02 +0200 Subject: [PATCH 2/3] fix(watcher): add tests and watcher task fails if scanning managed workspace --- antarest/core/exceptions.py | 8 -- antarest/study/storage/rawstudy/watcher.py | 18 +++- tests/integration/test_integration_watcher.py | 32 ------- tests/integration/test_watcher.py | 95 +++++++++++++++++++ tests/storage/business/test_watcher.py | 6 +- 5 files changed, 112 insertions(+), 47 deletions(-) delete mode 100644 tests/integration/test_integration_watcher.py create mode 100644 tests/integration/test_watcher.py diff --git a/antarest/core/exceptions.py b/antarest/core/exceptions.py index f49d3edd19..efee03c2b1 100644 --- a/antarest/core/exceptions.py +++ b/antarest/core/exceptions.py @@ -269,11 +269,3 @@ def __init__(self, *district_ids: str): class BadEditInstructionException(HTTPException): def __init__(self, message: str) -> None: super().__init__(HTTPStatus.UNPROCESSABLE_ENTITY, message) - - -class CannotScanInternalWorkspace(HTTPException): - def __init__(self) -> None: - super().__init__( - HTTPStatus.BAD_REQUEST, - "You cannot scan the default internal workspace", - ) diff --git a/antarest/study/storage/rawstudy/watcher.py b/antarest/study/storage/rawstudy/watcher.py index 05ff203d88..e2d4eebe06 100644 --- a/antarest/study/storage/rawstudy/watcher.py +++ b/antarest/study/storage/rawstudy/watcher.py @@ -1,7 +1,6 @@ import logging import re import tempfile -import threading from html import escape from http import HTTPStatus from http.client import HTTPException @@ -12,7 +11,6 @@ from filelock import FileLock from antarest.core.config import Config -from antarest.core.exceptions import CannotScanInternalWorkspace from antarest.core.interfaces.service import IService from antarest.core.requests import RequestParameters from antarest.core.tasks.model import TaskResult, TaskType @@ -31,6 +29,11 @@ def __init__(self, message: str) -> None: super().__init__(HTTPStatus.BAD_REQUEST, message) +class CannotScanInternalWorkspace(Exception): + def __init__(self, message: str) -> None: + super().__init__(message) + + class Watcher(IService): """ Files Watcher to listen raw studies changes and trigger a database update. @@ -128,7 +131,7 @@ def _rec_scan( return [StudyFolder(path, workspace, groups)] else: - folders: List[StudyFolder] = list() + folders: List[StudyFolder] = [] if path.is_dir(): for child in path.iterdir(): try: @@ -183,6 +186,9 @@ def scan_task(notifier: TaskUpdateNotifier) -> TaskResult: self.scan(workspace, path) except CannotScanInternalWorkspace as e: logger.warning(e) + return TaskResult( + success=False, message=f"Scan failed: {str(e)}" + ) return TaskResult(success=True, message="Scan completed") return self.task_service.add_task( @@ -205,11 +211,13 @@ def scan( """ stopwatch = StopWatch() - studies: List[StudyFolder] = list() + studies: List[StudyFolder] = [] directory_path: Optional[Path] = None if workspace_directory_path is not None and workspace_name: if workspace_name == DEFAULT_WORKSPACE_NAME: - raise CannotScanInternalWorkspace + raise CannotScanInternalWorkspace( + "You cannot scan the default internal workspace" + ) try: workspace = self.config.storage.workspaces[workspace_name] except KeyError: diff --git a/tests/integration/test_integration_watcher.py b/tests/integration/test_integration_watcher.py deleted file mode 100644 index ac115755ce..0000000000 --- a/tests/integration/test_integration_watcher.py +++ /dev/null @@ -1,32 +0,0 @@ -from fastapi import FastAPI -from starlette.testclient import TestClient - -from antarest.core.tasks.model import TaskStatus - - -def test_integration_watcher(app: FastAPI, tmp_path: str): - client = TestClient(app) - res = client.post( - "/v1/login", json={"username": "admin", "password": "admin"} - ) - admin_credentials = res.json() - headers = {"Authorization": f'Bearer {admin_credentials["access_token"]}'} - - res = client.post("/v1/watcher/_scan", headers=headers) - assert res.status_code == 422 - assert res.json() == { - "description": "field required", - "exception": "RequestValidationError", - "body": None, - } - - task_id = client.post( - "/v1/watcher/_scan?path=/default", - headers=headers, - ).json() - - # asserts that scanning default workspace doesn't fail - res = client.get( - f"v1/tasks/{task_id}?wait_for_completion=true", headers=headers - ).json() - assert res["status"] == TaskStatus.COMPLETED.value diff --git a/tests/integration/test_watcher.py b/tests/integration/test_watcher.py new file mode 100644 index 0000000000..0db6c07a3e --- /dev/null +++ b/tests/integration/test_watcher.py @@ -0,0 +1,95 @@ +from fastapi import FastAPI +from starlette.testclient import TestClient + +from antarest.core.tasks.model import TaskResult + + +def test_scan_dir__no_path(app: FastAPI, admin_access_token: str) -> None: + client = TestClient(app) + headers = {"Authorization": f"Bearer {admin_access_token}"} + + res = client.post("/v1/watcher/_scan", headers=headers) + assert res.status_code == 422 + assert res.json() == { + "description": "field required", + "exception": "RequestValidationError", + "body": None, + } + + +def test_scan_dir__default_workspace( + app: FastAPI, admin_access_token: str +) -> None: + client = TestClient(app) + headers = {"Authorization": f"Bearer {admin_access_token}"} + + task_id = client.post( + "/v1/watcher/_scan?path=/default", + headers=headers, + ) + # asserts that the POST request did not raise an Exception + assert task_id.status_code == 200 + + # asserts that the task failed + res = client.get( + f"v1/tasks/{task_id.json()}?wait_for_completion=true", headers=headers + ) + task_result = TaskResult.parse_obj(res.json()["result"]) + assert not task_result.success + assert ( + task_result.message + == "Scan failed: You cannot scan the default internal workspace" + ) + assert task_result.return_value is None + + +def test_scan_dir__unknown_folder( + app: FastAPI, admin_access_token: str +) -> None: + client = TestClient(app) + headers = {"Authorization": f"Bearer {admin_access_token}"} + + fake_workspace_name = "fake_workspace" + task_id = client.post( + f"/v1/watcher/_scan?path={fake_workspace_name}", + headers=headers, + ) + + # asserts that the POST request did not raise an Exception + assert task_id.status_code == 200 + + # asserts that the task failed + res = client.get( + f"v1/tasks/{task_id.json()}?wait_for_completion=true", headers=headers + ) + task_result = TaskResult.parse_obj(res.json()["result"]) + assert not task_result.success + assert ( + task_result.message + == f"Task {task_id.json()} failed: Unhandled exception " + f"(, 'Workspace {fake_workspace_name} not found')" + f"\nSee the logs for detailed information and the error traceback." + ) + assert task_result.return_value is None + + +def test_scan_dir__nominal_case(app: FastAPI, admin_access_token: str) -> None: + client = TestClient(app) + headers = {"Authorization": f"Bearer {admin_access_token}"} + + task_id = client.post( + "/v1/watcher/_scan?path=ext", + headers=headers, + ) + + # asserts that the POST request succeeded + assert task_id.status_code == 200 + + # asserts that the task succeeded + res = client.get( + f"v1/tasks/{task_id.json()}?wait_for_completion=true", headers=headers + ) + task_result = TaskResult.parse_obj(res.json()["result"]) + assert task_result.success + assert task_result.message == "Scan completed" + assert task_result.return_value is None diff --git a/tests/storage/business/test_watcher.py b/tests/storage/business/test_watcher.py index 4123c1255d..5999ae2eba 100644 --- a/tests/storage/business/test_watcher.py +++ b/tests/storage/business/test_watcher.py @@ -7,12 +7,14 @@ from sqlalchemy import create_engine from antarest.core.config import Config, StorageConfig, WorkspaceConfig -from antarest.core.exceptions import CannotScanInternalWorkspace from antarest.core.persistence import Base from antarest.core.utils.fastapi_sqlalchemy import DBSessionMiddleware from antarest.login.model import Group from antarest.study.model import StudyFolder, DEFAULT_WORKSPACE_NAME -from antarest.study.storage.rawstudy.watcher import Watcher +from antarest.study.storage.rawstudy.watcher import ( + Watcher, + CannotScanInternalWorkspace, +) from tests.storage.conftest import SimpleSyncTaskService From c25490d168fa19ed38980652ab3c13a9848aea7b Mon Sep 17 00:00:00 2001 From: belthlemar Date: Thu, 31 Aug 2023 14:49:56 +0200 Subject: [PATCH 3/3] feat(tests): reorganise tests in test_watcher.py --- tests/integration/test_watcher.py | 50 +++++++++++++++---------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/tests/integration/test_watcher.py b/tests/integration/test_watcher.py index 0db6c07a3e..cdfeda44ad 100644 --- a/tests/integration/test_watcher.py +++ b/tests/integration/test_watcher.py @@ -4,17 +4,26 @@ from antarest.core.tasks.model import TaskResult -def test_scan_dir__no_path(app: FastAPI, admin_access_token: str) -> None: +def test_scan_dir__nominal_case(app: FastAPI, admin_access_token: str) -> None: client = TestClient(app) headers = {"Authorization": f"Bearer {admin_access_token}"} - res = client.post("/v1/watcher/_scan", headers=headers) - assert res.status_code == 422 - assert res.json() == { - "description": "field required", - "exception": "RequestValidationError", - "body": None, - } + task_id = client.post( + "/v1/watcher/_scan?path=ext", + headers=headers, + ) + + # asserts that the POST request succeeded + assert task_id.status_code == 200 + + # asserts that the task succeeded + res = client.get( + f"v1/tasks/{task_id.json()}?wait_for_completion=true", headers=headers + ) + task_result = TaskResult.parse_obj(res.json()["result"]) + assert task_result.success + assert task_result.message == "Scan completed" + assert task_result.return_value is None def test_scan_dir__default_workspace( @@ -73,23 +82,14 @@ def test_scan_dir__unknown_folder( assert task_result.return_value is None -def test_scan_dir__nominal_case(app: FastAPI, admin_access_token: str) -> None: +def test_scan_dir__no_path(app: FastAPI, admin_access_token: str) -> None: client = TestClient(app) headers = {"Authorization": f"Bearer {admin_access_token}"} - task_id = client.post( - "/v1/watcher/_scan?path=ext", - headers=headers, - ) - - # asserts that the POST request succeeded - assert task_id.status_code == 200 - - # asserts that the task succeeded - res = client.get( - f"v1/tasks/{task_id.json()}?wait_for_completion=true", headers=headers - ) - task_result = TaskResult.parse_obj(res.json()["result"]) - assert task_result.success - assert task_result.message == "Scan completed" - assert task_result.return_value is None + res = client.post("/v1/watcher/_scan", headers=headers) + assert res.status_code == 422 + assert res.json() == { + "description": "field required", + "exception": "RequestValidationError", + "body": None, + }