diff --git a/client/lomas_client/scripts/run_notebook.py b/client/lomas_client/scripts/run_notebook.py index 9632b70fc..484aa5d61 100644 --- a/client/lomas_client/scripts/run_notebook.py +++ b/client/lomas_client/scripts/run_notebook.py @@ -9,6 +9,7 @@ from lomas_server.administration.scripts.lomas_demo_setup import ( lomas_demo_setup, ) + from lomas_server.models.config import Config as ServerConfig except ImportError: pass @@ -46,6 +47,10 @@ def run_notebook( if importlib.util.find_spec("lomas_server") is None: raise ImportError("lomas_server library not found, cannot run lomas_demo_setup.") + config = ServerConfig() + config.database.wipe() + config.database.set_bootstrap(config.bootstrap) + lomas_demo_setup() nb = nbformat.read(notebook_file, as_version=4) diff --git a/client/lomas_client/tests/test_integrations.py b/client/lomas_client/tests/test_integrations.py index b87718ea0..e0cedb467 100644 --- a/client/lomas_client/tests/test_integrations.py +++ b/client/lomas_client/tests/test_integrations.py @@ -27,15 +27,22 @@ del_all_dex_users, ) from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup -from lomas_server.models.config import AdminConfig +from lomas_server.models.config import AdminConfig, Config enable_features("contrib") @pytest.fixture def demo_setup(): + config = Config() + config.database.set_bootstrap(config.bootstrap) + lomas_demo_setup() + yield + + config.database.wipe() + @dataclass(frozen=True) class Aria: diff --git a/devenv.nix b/devenv.nix index fe2bf02a3..11621fb71 100644 --- a/devenv.nix +++ b/devenv.nix @@ -186,13 +186,13 @@ in coverage.module = { processes.worker = { cwd = lib.mkForce "${config.git.root}"; - exec = lib.mkForce "exec coverage run --data-file=.coverage.worker -m lomas_server.worker"; + exec = lib.mkForce "mkdir -p ./logs/ && exec coverage run --data-file=.coverage.worker -m lomas_server.worker &> ./logs/worker.log"; }; # override the UT script to generate coverage scripts.ut = wrapScript { exec = '' - exec pytest --cov-append --cov-report term-missing --cov --no-cov-on-fail --cov-config=${config.env.COVERAGE_RCFILE} + mkdir -p ./logs/ && exec pytest --cov-append --cov-report term-missing --cov --no-cov-on-fail --cov-config=${config.env.COVERAGE_RCFILE} &> ./logs/pytest.log ''; }; @@ -257,7 +257,6 @@ in # Lomas demo setup LOMAS_ADMIN_server_url = "http://localhost:${toString config.lomas.port}"; # public lomas service url from dashboard LOMAS_ADMIN_server_service = "http://localhost:${toString config.lomas.port}"; - LOMAS_ADMIN_database_url = "/tmp/admin.db"; LOMAS_ADMIN_USER_YAML = user_yaml_path; LOMAS_ADMIN_DATASET_YAML = dataset_yaml_path; LOMAS_ADMIN_DEX_CONFIG__URL = "grpc://${config.lomas.dex.adminAddress}:${toString config.lomas.dex.adminPort}"; diff --git a/devenv/lomas.nix b/devenv/lomas.nix index 7e5fba871..30214dcf7 100644 --- a/devenv/lomas.nix +++ b/devenv/lomas.nix @@ -159,7 +159,9 @@ in { processes.lomas-server = { - exec = "exec python uvicorn_serve.py"; + exec = '' + rm -f ${config.env.LOMAS_SERVICE_admin_database_url} && exec python uvicorn_serve.py + ''; cwd = "${config.git.root}/server/lomas_server"; ready = { http.get = { diff --git a/requirements.txt b/requirements.txt index a5ffdaacd..3fbaed957 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,10 +22,10 @@ bcrypt==5.0.0 # via lomas-server beautifulsoup4==4.14.3 # via nbconvert bleach==6.3.0 # via nbconvert blinker==1.9.0 # via streamlit -boto3==1.43.11 # via lomas-server -botocore==1.43.11 # via boto3, s3transfer +boto3==1.43.12 # via lomas-server +botocore==1.43.12 # via boto3, s3transfer cachetools==7.1.3 # via streamlit -certifi==2026.4.22 # via httpcore, httpx, requests +certifi==2026.5.20 # via httpcore, httpx, requests cffi==2.0.0 # via argon2-cffi-bindings, cryptography charset-normalizer==3.4.7 # via requests chex==0.1.91 # via mbi @@ -48,13 +48,14 @@ executing==2.2.1 # via stack-data fastapi==0.136.1 # via lomas-core fastapi-cli==0.0.24 # via fastapi fastjsonschema==2.21.2 # via nbformat +filelock==3.29.0 # via lomas-server fonttools==4.63.0 # via matplotlib fqdn==1.5.1 # via jsonschema gitdb==4.0.12 # via gitpython gitpython==3.1.50 # via streamlit googleapis-common-protos==1.75.0 # via opentelemetry-exporter-otlp-proto-grpc, opentelemetry-exporter-otlp-proto-http graphviz==0.21 # via smartnoise-sql -greenlet==3.5.0 # via sqlalchemy +greenlet==3.5.1 # via sqlalchemy grpcio==1.80.0 # via opentelemetry-exporter-otlp-proto-grpc h11==0.16.0 # via httpcore, uvicorn html5rdf==1.2.1 # via rdflib @@ -70,8 +71,8 @@ ipython-pygments-lexers==1.1.1 # via ipython ipywidgets==8.1.8 # via jupyter isoduration==20.11.0 # via jsonschema itsdangerous==2.2.0 # via streamlit -jax==0.10.0 # via chex, mbi, optax -jaxlib==0.10.0 # via chex, jax, mbi, optax +jax==0.10.1 # via chex, mbi, optax +jaxlib==0.10.1 # via chex, jax, mbi, optax jedi==0.20.0 # via ipython jinja2==3.1.6 # via altair, fastapi, jupyter-server, jupyterlab, jupyterlab-server, myst-parser, nbconvert, nbsphinx, pydeck, sphinx jmespath==1.1.0 # via boto3, botocore @@ -192,10 +193,10 @@ rfc3339-validator==0.1.4 # via jsonschema, jupyter-events rfc3986-validator==0.1.1 # via jsonschema, jupyter-events rfc3987-syntax==1.1.0 # via jsonschema rich==15.0.0 # via rich-toolkit, typer -rich-toolkit==0.19.9 # via fastapi-cli +rich-toolkit==0.19.10 # via fastapi-cli roman-numerals==4.1.0 # via sphinx rpds-py==0.30.0 # via jsonschema, referencing -ruff==0.15.13 # via lomas (pyproject.toml) +ruff==0.15.14 # via lomas (pyproject.toml) s3transfer==0.17.0 # via boto3 scikit-learn==1.8.0 # via diffprivlib-logger, opendp scipy==1.17.1 # via diffprivlib-logger, jax, jaxlib, mbi, scikit-learn @@ -247,5 +248,5 @@ webencodings==0.5.1 # via bleach, tinycss2 websocket-client==1.9.0 # via jupyter-server websockets==16.0 # via streamlit, uvicorn widgetsnbextension==4.0.15 # via ipywidgets -wrapt==2.1.2 # via deprecated, opentelemetry-instrumentation, opentelemetry-instrumentation-aio-pika +wrapt==2.2.0 # via deprecated, opentelemetry-instrumentation, opentelemetry-instrumentation-aio-pika yarl==1.24.2 # via aio-pika, aiormq diff --git a/server/lomas_server/admin_database/admin_database.py b/server/lomas_server/admin_database/admin_database.py index 9477e9a76..79fe1e0d8 100644 --- a/server/lomas_server/admin_database/admin_database.py +++ b/server/lomas_server/admin_database/admin_database.py @@ -433,3 +433,33 @@ def save_query(self, user_name: str, query: LomasRequestModel, response: QueryRe @abstractmethod def wipe(self) -> None: """Wipe the entire Database.""" + + @abstractmethod + def set_bootstrap(self, bootstrap: str) -> None: + """Sets the bootstrap value. + + Also sets the bootstrap disabled value to False. + + Args: + bootstrap (str): Bootstrap creds to set. + """ + + @abstractmethod + def get_bootstrap(self) -> str | None: + """Returns the bootstrap credential value or None if it has not been set. + + Returns: + str | None: The bootstrap credential value or None if it has not been set. + """ + + @abstractmethod + def set_bootstrap_disabled(self, bootstrap_disabled: bool = True) -> None: + """Sets the bootstrap disabled value.""" + + @abstractmethod + def get_bootstrap_disabled(self) -> bool: + """Get the bootstrap disabled value. + + Returns: + bool: The bootstrap disabled value. False by default if not set in the DB. + """ diff --git a/server/lomas_server/admin_database/constants.py b/server/lomas_server/admin_database/constants.py index 9f97aa3de..5a1356ad9 100644 --- a/server/lomas_server/admin_database/constants.py +++ b/server/lomas_server/admin_database/constants.py @@ -8,6 +8,14 @@ class TopDBKey(StrEnum): USERS = "users" DATASETS = "datasets" METADATA = "metadata" + MISC_KEYS = "misc" + + +class MiscDBKeys(StrEnum): + """Key for selecting sub elements in misc collection.""" + + BOOTSTRAP_DISABLED = "bootstrap_disabled" + BOOTSTRAP = "bootstrap" class BudgetDBKey(StrEnum): diff --git a/server/lomas_server/admin_database/local_database.py b/server/lomas_server/admin_database/local_database.py index 2ef8f334f..04cfc1cf4 100644 --- a/server/lomas_server/admin_database/local_database.py +++ b/server/lomas_server/admin_database/local_database.py @@ -10,6 +10,7 @@ import yaml from csvw_eo.metadata_structure import TableMetadata from fastapi import UploadFile +from filelock import SoftFileLock from pydantic import HttpUrl from lomas_core.error_handler import InternalServerException @@ -32,7 +33,7 @@ user_must_exist, user_must_have_access_to_dataset, ) -from lomas_server.admin_database.constants import BudgetDBKey, TopDBKey as TK +from lomas_server.admin_database.constants import BudgetDBKey, MiscDBKeys, TopDBKey as TK if sys.version_info >= (3, 12): from typing import override @@ -42,6 +43,8 @@ logger = get_lomas_logger(__name__) +lock: SoftFileLock = SoftFileLock("/tmp/admindb.lock", is_singleton=True, timeout=10) + class LocalAdminDatabase(AdminDatabase): """Local Admin database in a single file.""" @@ -49,23 +52,30 @@ class LocalAdminDatabase(AdminDatabase): path: Path """Database accepts existing path or new (creatable) path.""" + @lock def model_post_init(self, _: Any, /) -> None: # create the file if it doesn't exists yet (makes open with flag='r' safe) shelve.open(self.path).close() @override + @lock def wipe(self) -> None: if (p := Path(self.path)).exists(): p.unlink() + # Recreate file to make open with flag="r" safe + shelve.open(self.path).close() + @lock def load_users_collection(self, users: list[User]) -> None: with shelve.open(self.path, writeback=True) as db: db[TK.USERS] = {user.id.name: user.model_dump() for user in users} + @lock def users(self) -> list[User]: with shelve.open(self.path, flag="r") as db: return list(map(User.model_validate, db.get(TK.USERS, {}).values())) + @lock def load_dataset_collection(self, datasets: list[DSInfo], path_prefix: Path) -> None: with shelve.open(self.path, writeback=True) as db: # Step 1: add datasets @@ -127,10 +137,12 @@ def load_dataset_collection(self, datasets: list[DSInfo], path_prefix: Path) -> db[TK.METADATA][dataset_name] = TableMetadata.from_dict(metadata_dict).model_dump() logger.info(f"Added metadata of {dataset_name} dataset.") + @lock def datasets(self) -> list[DSInfo]: with shelve.open(self.path, flag="r") as db: return list(map(DSInfo.model_validate, db.get(TK.DATASETS, []))) + @lock def add_datasets_via_yaml( self, yaml_file: Path | BinaryIO | SpooledTemporaryFile, @@ -162,6 +174,7 @@ def add_datasets_via_yaml( yaml_dict = yaml.safe_load(yaml_file) self.load_dataset_collection(DatasetsCollection(**yaml_dict).datasets, path_prefix) + @lock def add_dataset( self, dataset_name: str, @@ -272,12 +285,14 @@ def add_dataset( db[TK.METADATA] = db.get(TK.METADATA, {}) | {dataset_name: validated_metadata} db.sync() + @lock def del_dataset(self, dataset_name: str) -> None: with shelve.open(self.path, writeback=True) as db: for ds in db[TK.DATASETS]: if ds["dataset_name"] == dataset_name: db[TK.DATASETS].remove(ds) + @lock def add_dataset_to_user(self, username: str, dataset_name: str, epsilon: float, delta: float) -> None: with shelve.open(self.path, writeback=True) as db: user = User.model_validate(db[TK.USERS][username]) @@ -289,6 +304,7 @@ def add_dataset_to_user(self, username: str, dataset_name: str, epsilon: float, ) db[TK.USERS][username] = user_updated.model_dump() + @lock def del_dataset_to_user(self, username: str, dataset_name: str) -> None: with shelve.open(self.path, writeback=True) as db: user = User.model_validate(db[TK.USERS][username]) @@ -299,6 +315,7 @@ def del_dataset_to_user(self, username: str, dataset_name: str) -> None: ) db[TK.USERS][username] = user_updated.model_dump() + @lock def add_users_via_yaml(self, yaml_file: Path | BinaryIO | SpooledTemporaryFile, clean: bool) -> None: """Add all users from yaml file to the user collection. @@ -322,6 +339,7 @@ def add_users_via_yaml(self, yaml_file: Path | BinaryIO | SpooledTemporaryFile, yaml_dict = yaml.safe_load(yaml_file) self.load_users_collection(UserCollection(**yaml_dict).users) + @lock def add_user( self, username: str, @@ -359,20 +377,24 @@ def add_user( db[TK.USERS][username] = validated_user @user_must_exist + @lock def del_user(self, username: str) -> None: with shelve.open(self.path, writeback=True) as db: del db[TK.USERS][username] @override + @lock def does_user_exist(self, user_name: str) -> bool: return user_name in map(lambda user: user.id.name, self.users()) @override + @lock def does_dataset_exist(self, dataset_name: str) -> bool: return dataset_name in map(lambda ds: ds.dataset_name, self.datasets()) @override @dataset_must_exist + @lock def get_dataset(self, dataset_name: str) -> DSInfo: with shelve.open(self.path, flag="r") as db: dataset = next(filter(lambda ds: ds["dataset_name"] == dataset_name, db[TK.DATASETS])) @@ -386,6 +408,7 @@ def get_dataset_metadata(self, dataset_name: str) -> TableMetadata: return TableMetadata.model_validate(metadata) @dataset_must_exist + @lock def set_dataset_metadata(self, dataset_name: str, json_file: UploadFile) -> None: json_file.seek(0) content = json_file.read() @@ -398,12 +421,14 @@ def set_dataset_metadata(self, dataset_name: str, json_file: UploadFile) -> None @override @user_must_exist + @lock def is_user_admin(self, user_name: str) -> bool: with shelve.open(self.path, flag="r") as db: return db[TK.USERS][user_name]["admin"] @override @user_must_exist + @lock def get_and_set_may_user_query(self, user_name: str, may_query: bool) -> bool: with shelve.open(self.path, writeback=True) as db: previous_may_query = db[TK.USERS][user_name]["may_query"] @@ -412,6 +437,7 @@ def get_and_set_may_user_query(self, user_name: str, may_query: bool) -> bool: @override @user_must_exist + @lock def has_user_access_to_dataset(self, user_name: str, dataset_name: str) -> bool: @dataset_must_exist def has_access_to_dataset(self: Self, dataset_name: str) -> bool: @@ -427,6 +453,7 @@ def has_access_to_dataset(self: Self, dataset_name: str) -> bool: return has_access_to_dataset(self, dataset_name) @override + @lock def get_epsilon_or_delta(self, user_name: str, dataset_name: str, parameter: BudgetDBKey) -> float: with shelve.open(self.path, flag="r") as db: return sum( @@ -440,6 +467,7 @@ def get_epsilon_or_delta(self, user_name: str, dataset_name: str, parameter: Bud ) @override + @lock def update_epsilon_or_delta( self, user_name: str, @@ -453,6 +481,7 @@ def update_epsilon_or_delta( if ds["dataset_name"] == dataset_name: ds[parameter] += spent_value + @lock def set_epsilon_or_delta( self, user_name: str, @@ -468,6 +497,7 @@ def set_epsilon_or_delta( @override @user_must_have_access_to_dataset + @lock def get_user_previous_queries( self, user_name: str, @@ -480,6 +510,7 @@ def match(archive: dict[str, str]) -> bool: return list(filter(match, db.get(TK.ARCHIVE, []))) @override + @lock def save_query(self, user_name: str, query: LomasRequestModel, response: QueryResponse) -> None: with shelve.open(self.path, writeback=True) as db: to_archive = self.prepare_save_query(user_name, query, response) @@ -492,7 +523,36 @@ def get_archives_of_user(self, username: str) -> list[dict]: with shelve.open(self.path, flag="r") as db: return [archive for archive in db.get(TK.ARCHIVE, []) if archive["user_name"] == username] + @lock def drop_collection(self, collection: str) -> None: with shelve.open(self.path, writeback=True) as db: if collection in db: del db[collection] + + @override + @lock + def set_bootstrap(self, bootstrap: str) -> None: + with shelve.open(self.path, writeback=True) as db: + db[TK.MISC_KEYS] = {MiscDBKeys.BOOTSTRAP_DISABLED: False, MiscDBKeys.BOOTSTRAP: bootstrap} + + @override + @lock + def get_bootstrap(self) -> str | None: + with shelve.open(self.path, flag="r") as db: + if TK.MISC_KEYS in db: + return db[TK.MISC_KEYS].get(MiscDBKeys.BOOTSTRAP, None) + return None + + @override + @lock + def set_bootstrap_disabled(self, bootstrap_disabled: bool = True) -> None: + with shelve.open(self.path, writeback=True) as db: + db[TK.MISC_KEYS][MiscDBKeys.BOOTSTRAP_DISABLED] = bootstrap_disabled + + @override + @lock + def get_bootstrap_disabled(self) -> bool: + with shelve.open(self.path, flag="r") as db: + if TK.MISC_KEYS in db: + return db[TK.MISC_KEYS].get(MiscDBKeys.BOOTSTRAP_DISABLED, False) + return False diff --git a/server/lomas_server/administration/dashboard/about.py b/server/lomas_server/administration/dashboard/about.py index c729398e1..5cc6f833d 100644 --- a/server/lomas_server/administration/dashboard/about.py +++ b/server/lomas_server/administration/dashboard/about.py @@ -4,10 +4,10 @@ from returns.io import IOFailure, IOSuccess from returns.maybe import Maybe from returns.pipeline import flow -from returns.pointfree import bind_result, map_ +from returns.pointfree import alt, bind_result, lash, map_ from returns.result import Failure, Success -from lomas_server.administration.dashboard.utils import get_config, query_lomas_auth +from lomas_server.administration.dashboard.utils import get_config, query_lomas_auth, recover_if_410 def main() -> None: @@ -101,6 +101,22 @@ def about() -> None: ) ) + flow( + query_lomas_auth("/bootstrap", httpx.get), + lash(lambda e: recover_if_410(e, default=False)), + alt(lambda e: st.write(f":red-badge[unavailable]: {e}")), + map_(lambda e: True if e is None else False), # Define bootstrap_exists + map_( + lambda bootstrap_exists: ( + st.write( + ":red-badge[Bootstrap permissions enabled!] Lomas admin api endpoints are authorized with bootstrap credentials. Disable bootstrap permissions!" + ) + if bootstrap_exists + else st.write(":green-badge[Bootstrap permissions disabled]") + ) + ), + ) + if __name__ == "__main__": main() diff --git a/server/lomas_server/administration/dashboard/database_administration.py b/server/lomas_server/administration/dashboard/database_administration.py index 56cd83eb1..909e517b3 100644 --- a/server/lomas_server/administration/dashboard/database_administration.py +++ b/server/lomas_server/administration/dashboard/database_administration.py @@ -1,4 +1,3 @@ -from collections.abc import Callable from functools import partial from pathlib import Path @@ -11,7 +10,7 @@ from returns.iterables import Fold from returns.maybe import Maybe, Some from returns.pipeline import flow -from returns.pointfree import alt, bind, bind_result, map_ +from returns.pointfree import alt, bind, bind_result, lash, map_ from returns.result import Failure, ResultE, Success from lomas_core.models.collections import User, UserCollection, UserId @@ -20,6 +19,7 @@ from lomas_server.admin_database.constants import TopDBKey as TK from lomas_server.administration.dashboard.utils import ( call_if_dex, + confirm_delete, ensure_user_has_datasets, find_user, get_config, @@ -28,6 +28,7 @@ get_users, list_users, query_lomas_auth, + recover_if_410, ) from lomas_server.administration.dex.dex_admin import ( add_dex_user, @@ -43,6 +44,35 @@ DELTA_STEP = 0.00001 +st.title("Bootstrap permissions") + +bootstrap_exists = flow( + query_lomas_auth("/bootstrap", httpx.get), + lash(lambda e: recover_if_410(e, default=False)), + alt(lambda e: st.error(f"Error while fetching bootstrap state: {e}")), + map_(lambda e: True if e is None else False), # Define bootstrap_exists +) + +delete_bootstrap = False +match bootstrap_exists: + case IOFailure(): + pass # case handled above + case IOSuccess(Success(True)): + with st.container(horizontal_alignment="center"): + delete_bootstrap = st.button( + "Delete bootstrap permissions", type="primary", key="del_bootstrap_button" + ) + case IOSuccess(Success(False)): + st.success("Bootstrap permissions already removed") + +if delete_bootstrap: + confirm_delete( + "Delete bootstrap permissions permanently?", + lambda: query_lomas_auth("/bootstrap", httpx.delete), + "Bootstrap has been deleted", + ) + +st.divider() st.title("Users") row_selector = flow( @@ -198,29 +228,6 @@ def drop_lomas_collection(collection_name: str) -> IOResultE[httpx.Response]: return query_lomas_auth(f"/collections/{collection_name}", httpx.delete) -@st.dialog("Confirm deletion") -def confirm_delete( - message: str, on_confirm: Callable[[], IOResultE[httpx.Response]], success_message: str -) -> None: - st.warning(message) - - col1, col2 = st.columns(2) - - with col1: - if st.button("Yes", type="primary"): - match on_confirm(): - case IOFailure(fail): - st.write(f"Operation failed: {fail}") - case _: - st.write(success_message) - - st.rerun() - - with col2: - if st.button("No"): - st.rerun() - - ############################# # GUI and user interactions # ############################# diff --git a/server/lomas_server/administration/dashboard/utils.py b/server/lomas_server/administration/dashboard/utils.py index 48927f365..0de70282e 100644 --- a/server/lomas_server/administration/dashboard/utils.py +++ b/server/lomas_server/administration/dashboard/utils.py @@ -63,11 +63,44 @@ def unwrap_Failure(res: IOResultE[Maybe[IOResultE]]) -> IOResultE[Maybe[IOResult return dex_config_res +@st.dialog("Confirm deletion") +def confirm_delete( + message: str, on_confirm: Callable[[], IOResultE[httpx.Response]], success_message: str +) -> None: + st.warning(message) + + col1, col2 = st.columns(2) + + with col1: + if st.button("Yes", type="primary"): + match on_confirm(): + case IOFailure(fail): + st.write(f"Operation failed: {fail}") + case _: + st.write(success_message) + + st.rerun() + + with col2: + if st.button("No"): + st.rerun() + + @impure_safe def parse_if_ok(response: httpx.Response) -> str: return response.raise_for_status().json() +def recover_if_410(e: Exception, default: Any = None) -> IOResultE: + match e: + case httpx.HTTPStatusError(): + if e.response.status_code == 410: + return IOSuccess(default) + case _: + pass + return IOFailure(e) + + def query_lomas( endpoint: str, verb: Callable[..., httpx.Response], **kwargs: dict[str, Any] ) -> IOResultE[httpx.Response]: diff --git a/server/lomas_server/administration/tests/test_streamlit_app.py b/server/lomas_server/administration/tests/test_streamlit_app.py index 46668ab8d..bbbc7e1c4 100644 --- a/server/lomas_server/administration/tests/test_streamlit_app.py +++ b/server/lomas_server/administration/tests/test_streamlit_app.py @@ -14,6 +14,7 @@ from lomas_server.administration.dashboard.utils import query_lomas from lomas_server.administration.scripts.lomas_demo_setup import lomas_demo_setup from lomas_server.app import app +from lomas_server.models.config import Config from lomas_server.tests.utils import free_pass_env test_data_folder = (Path(__file__).parent / "../../tests/test_data").resolve() @@ -21,8 +22,15 @@ @pytest.fixture def demo_setup(): + config = Config() + config.database.set_bootstrap(config.bootstrap) + lomas_demo_setup() + yield + + config.database.wipe() + @pytest.fixture def switch_data_dir(): @@ -61,12 +69,13 @@ def test_about_page(dashbord_dir: Path) -> None: assert "The Lomas Administration Dashboard" in at.markdown[0].value - assert "**Documentation**: [server documentation]" in at.markdown[-4].value - assert "**Support**: If you encounter any issues " in at.markdown[-3].value + assert "**Documentation**: [server documentation]" in at.markdown[-5].value + assert "**Support**: If you encounter any issues " in at.markdown[-4].value assert "Server Status" in at.header[3].value - assert "localhost:" in at.markdown[-2].value - assert "Dex is only supported for demo purposes" in at.markdown[-1].value + assert "localhost:" in at.markdown[-3].value + assert "Dex is only supported for demo purposes" in at.markdown[-2].value + assert "User is not logged" in at.markdown[-1].value def test_admin_page(dashbord_dir: Path, client: TestClient, demo_setup) -> None: diff --git a/server/lomas_server/app.py b/server/lomas_server/app.py index 009f7d761..5c1e5fc70 100644 --- a/server/lomas_server/app.py +++ b/server/lomas_server/app.py @@ -12,7 +12,6 @@ ) from lomas_core.instrumentation import init_telemetry from lomas_core.models.constants import get_lomas_logger, init_logging -from lomas_server.admin_database.local_database import LocalAdminDatabase from lomas_server.dp_queries.dp_libraries.opendp import ( set_opendp_features_config, ) @@ -47,9 +46,15 @@ async def lifespan(lomas_app: FastAPI) -> AsyncGenerator[None]: # Load admin database try: logger.info("Loading admin database") - lomas_app.state.admin_database = LocalAdminDatabase(path=config.admin_database_url) + lomas_app.state.admin_database = config.database logger.info("Loading authenticator") lomas_app.state.authenticator = config.authenticator + + if not config.database.get_bootstrap_disabled(): + logger.info("Setting bootstrap credentials.") + config.database.set_bootstrap(config.bootstrap) + else: + logger.warning("Not setting bootstrap credentials because already disabled in the admin database") lomas_app.state.bootstrap = config.bootstrap lomas_app.state.private_db_credentials = config.private_db_credentials except InternalServerException as e: diff --git a/server/lomas_server/routes/routes_admin.py b/server/lomas_server/routes/routes_admin.py index d4a44b10b..f07ab4a75 100644 --- a/server/lomas_server/routes/routes_admin.py +++ b/server/lomas_server/routes/routes_admin.py @@ -623,6 +623,19 @@ def set_dataset_metadata_admin( db.set_dataset_metadata(dataset_name, file.file) +@router.get("/bootstrap") +def get_bootstrap( + request: Request, + _: Annotated[UserId, Security(get_user_id_from_authenticator, scopes=[Scopes.ADMIN])], + response: Response, +) -> None: + # Just returns ok if bootstrap still set. + if request.app.state.admin_database.get_bootstrap_disabled(): + response.status_code = status.HTTP_410_GONE + else: + response.status_code = status.HTTP_200_OK + + @router.delete("/bootstrap") def delete_bootstrap( request: Request, @@ -630,7 +643,7 @@ def delete_bootstrap( response: Response, ) -> None: # Bootstrap never set or already removed -> gone forever - if request.app.state.bootstrap is None: + if request.app.state.admin_database.get_bootstrap_disabled(): response.status_code = status.HTTP_410_GONE else: - request.app.state.bootstrap = None + request.app.state.admin_database.set_bootstrap_disabled(True) diff --git a/server/lomas_server/routes/utils.py b/server/lomas_server/routes/utils.py index f114b1e7c..1f45fd142 100644 --- a/server/lomas_server/routes/utils.py +++ b/server/lomas_server/routes/utils.py @@ -167,7 +167,8 @@ def get_user_id_from_authenticator( UserId: A UserId instance extracted from the token. """ # Bootstrap initialization - if (bootstrap_cred := request.app.state.bootstrap) is not None: + if not request.app.state.admin_database.get_bootstrap_disabled(): + bootstrap_cred = request.app.state.admin_database.get_bootstrap() match auth_creds: case HTTPAuthorizationCredentials(scheme="Bearer") if auth_creds.credentials == bootstrap_cred: logger.warning("Bootstrap User Bypass") diff --git a/server/lomas_server/tests/test_api_root.py b/server/lomas_server/tests/test_api_root.py index 653ae3f90..ba8f44084 100644 --- a/server/lomas_server/tests/test_api_root.py +++ b/server/lomas_server/tests/test_api_root.py @@ -34,6 +34,9 @@ def setUp(self) -> None: # Fill up database if needed path_prefix = Path(__file__).parent / "test_data" + self.config.database.wipe() + self.config.database.set_bootstrap(self.config.bootstrap) + self.config.database.add_users_via_yaml( yaml_file=(path_prefix / "test_user_collection.yaml"), clean=True, diff --git a/server/lomas_server/tests/test_auth.py b/server/lomas_server/tests/test_auth.py index 31d16f0a2..575d0a546 100644 --- a/server/lomas_server/tests/test_auth.py +++ b/server/lomas_server/tests/test_auth.py @@ -1,4 +1,5 @@ import os +import posix as Status import httpx import pytest @@ -18,19 +19,52 @@ @pytest.fixture def demo_setup(): - lomas_demo_setup() + # Make sure bootstrap is enabled + config = Config() + config.database.set_bootstrap(config.bootstrap) + + assert lomas_demo_setup() == Status.EX_OK yield - config = Config() admin_config = AdminConfig() dex_config = admin_config.dex_config assert dex_config is not None - del_all_dex_users(dex_config) - query_lomas("/collections/users", httpx.delete, headers={"Authorization": f"Bearer {config.bootstrap}"}) query_lomas( - "/collections/datasets", httpx.delete, headers={"Authorization": f"Bearer {config.bootstrap}"} + "/collections/users", httpx.delete, headers=get_auth_header("lomas_admin@example.com", "lomas_admin") + ) + query_lomas( + "/collections/datasets", + httpx.delete, + headers=get_auth_header("lomas_admin@example.com", "lomas_admin"), ) + del_all_dex_users(dex_config) + + +def test_bootstrap(demo_setup: None) -> None: + config = Config() + + # Test bootstrap creds + with TestClient(app, headers={"Authorization": f"Bearer {config.bootstrap}"}) as client: + response = client.get("/dataset/PENGUIN") + assert response.status_code == status.HTTP_200_OK + + response = client.get("/bootstrap") + assert response.status_code == status.HTTP_200_OK + + response = client.delete("/bootstrap") + assert response.status_code == status.HTTP_200_OK + + response = client.delete("/bootstrap") + assert response.status_code == status.HTTP_403_FORBIDDEN + + # Check response codes with proper admin headers once bootstrap removed + with TestClient(app, headers=get_auth_header("lomas_admin@example.com", "lomas_admin")) as client: + response = client.delete("/bootstrap") + assert response.status_code == status.HTTP_410_GONE + + response = client.get("/bootstrap") + assert response.status_code == status.HTTP_410_GONE def get_auth_header(user_name: str, user_password: str) -> dict[str, str]: diff --git a/server/pyproject.toml b/server/pyproject.toml index 2cd65576f..164506697 100644 --- a/server/pyproject.toml +++ b/server/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "cryptography>=44", "sqlglot>=27.8.0", "smartnoise-sql @ git+https://github.com/lancelotmarti/smartnoise-sdk.git@main#subdirectory=sql", + "filelock>=3.29.0", ] [project.optional-dependencies] diff --git a/uv.lock b/uv.lock index a6ce9ec73..3c4592002 100644 --- a/uv.lock +++ b/uv.lock @@ -889,6 +889,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/a8/20d0723294217e47de6d9e2e40fd4a9d2f7c4b6ef974babd482a59743694/fastjsonschema-2.21.2-py3-none-any.whl", hash = "sha256:1c797122d0a86c5cace2e54bf4e819c36223b552017172f32c5c024a6b77e463", size = 24024, upload-time = "2025-08-14T18:49:34.776Z" }, ] +[[package]] +name = "filelock" +version = "3.29.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, +] + [[package]] name = "fonttools" version = "4.62.1" @@ -1869,6 +1878,7 @@ dependencies = [ { name = "aio-pika" }, { name = "boto3" }, { name = "cryptography" }, + { name = "filelock" }, { name = "lomas-core" }, { name = "opentelemetry-instrumentation-aio-pika" }, { name = "opentelemetry-instrumentation-fastapi" }, @@ -1890,6 +1900,7 @@ requires-dist = [ { name = "bcrypt", marker = "extra == 'all'", specifier = ">=5.0.0" }, { name = "boto3", specifier = ">=1.34" }, { name = "cryptography", specifier = ">=44" }, + { name = "filelock", specifier = ">=3.29.0" }, { name = "lomas-core", editable = "core" }, { name = "opentelemetry-instrumentation-aio-pika", specifier = ">=0.50b0" }, { name = "opentelemetry-instrumentation-fastapi", specifier = ">=0.50b0" },