Merge pull request #9 from NickSebClark/feature/mongo-db

Mongo DB Support

.github/workflows/tests.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        python -m pip install flake8 pytest mypy coverage
+        python -m pip install pytest mypy coverage pymongo mongomock
     - name: Lint with ruff
       uses: chartboost/ruff-action@v1
     - name: Test with pytest
@@ -38,6 +38,7 @@ jobs:
     - name: mypy
       run: |
         mypy -p pyprexor
+        mypy -p pyprexor_datastore
     - name: Upload coverage
       if: github.ref == 'refs/heads/main'
       uses: codecov/codecov-action@v3

.gitignore

@@ -160,3 +160,5 @@ cython_debug/
 #.idea/
 
 .vscode/
+
+credentials.toml

README.md

@@ -33,4 +33,16 @@ poetry install
 
 ## Contribution Guide
 
-The project is linted with [ruff](https://github.com/astral-sh/ruff), styled with [black](https://github.com/psf/black) and type checked with [mypy](https://github.com/python/mypy).
+The project is linted with [ruff](https://github.com/astral-sh/ruff), styled with [black](https://github.com/psf/black) and type checked with [mypy](https://github.com/python/mypy).
+
+## Datastores
+
+Pyprexor ships with an *InMemoryDatastore*. This allows parameter sets to be loaded into memory on initialisation. Process data is added to a list in memory and can be read out on command. See the [example.py](/example_app/example.py) for usage.
+
+### MongoDB
+
+A basic wrapper around the MongoDB API is provided in [pyprexor_datastore.mongo](pyrprexor_datastore/mongo.py). It requires that pymongo is installed separately (i.e. ```pip install mongo```). If running the package from source, you an use :
+
+```
+poetry install --with mongo
+```

example_app/example.py

@@ -1,5 +1,5 @@
 import pyprexor as ex
-from pyprexor.datastore import InMemoryDataStore
+from pyprexor_datastore.datastore import InMemoryDataStore
 
 
 # Annotate functions we want execute with pyprexor

pyprexor/core.py

@@ -1,9 +1,10 @@
 import inspect
-import pyprexor.datastore as ds
+import pyprexor_datastore.datastore as ds
 from datetime import datetime
 import getpass
 import time
 import warnings
+from typing import Any
 
 datastore: ds.Datastore
 sw_version = "0.0.0"
@@ -16,10 +17,24 @@ class TypeWarning(UserWarning):
 
 
 class PyProcess:
+    """Class style decorator. Provides core Pyprexor functionality."""
+
     def __init__(self, func=None):
         self.func = func
 
-    def __call__(self, set_id):
+    def __call__(self, set_id: str) -> Any:
+        """Reads parameter set, calls the wrapped function with the parameters from the parameter set then populates a
+        process data object and writes it to the datastore.
+
+        Args:
+            set_id (str): Parameter set id to pull parameters from.
+
+        Raises:
+            KeyError: Required parameter not in the parameter set.
+
+        Returns:
+            Any: Normal function return
+        """
         print(f"Executing {self.func.__name__} with parameter set {set_id}...")
 
         global datastore
@@ -56,6 +71,7 @@ def __call__(self, set_id):
 
         process_data = {}
         process_data["name"] = self.func.__name__
+        process_data["parameter_set_id"] = set_id
         process_data["sw_version"] = sw_version
         process_data["user"] = getpass.getuser()
         process_data["datetime"] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
@@ -70,6 +86,12 @@ def __call__(self, set_id):
 
 
 def initialise(store: ds.Datastore, version: str = ""):
+    """Initialises globals used by pyprexor.
+
+    Args:
+        store (ds.Datastore): Datastore object to use.
+        version (str, optional): Version string to store in process data. Defaults to "".
+    """
     global datastore
     datastore = store
 

pyprexor_datastore/datastore.py

@@ -0,0 +1,58 @@
+class Datastore:
+    def get_parameter_set(self, id: str) -> dict:
+        raise NotImplementedError
+
+    def write_process_data(self, data: dict) -> str:
+        raise NotImplementedError
+
+
+class InMemoryDataStore(Datastore):
+    """Datastore which stores parameter sets and process data in memory. Parameter sets are stored as a dict of dicts.
+    The top-level dict key is the id of the parameter set to allow easy access.
+    The process data is simply a list of dicts.
+    """
+
+    def __init__(self, parameter_sets: list[dict], key: str = "id", re_index: bool = False):
+        """Initialises datastore with a list of parametersets.
+
+        Args:
+            parameter_sets (list[dict]): Parameter sets to initialise the data store with
+            key (str, optional): key string. Defaults to "id".
+            re_index (bool, optional): Reinitialises the key in the param sets to numbers from 0. Defaults to False.
+        """
+        if re_index:
+            self.parameter_sets = {str(index): parameter_set for (index, parameter_set) in enumerate(parameter_sets)}
+            for id in self.parameter_sets:
+                self.parameter_sets[id]["id"] = id
+        else:
+            self.parameter_sets = {parameter_set[key]: parameter_set for parameter_set in parameter_sets}
+
+        self.process_data: list[dict] = []
+        self.key = key
+
+    def get_parameter_set(self, id: str) -> dict:
+        """Get the parameter set from the dictionary of parameter sets
+
+        Args:
+            id (str): Id of parameter set to return.
+
+        Returns:
+            dict: Returned parameter set.
+        """
+        return self.parameter_sets[id]
+
+    def write_process_data(self, process_data: dict):
+        """Adds a process data dict to the store.
+
+        Args:
+            process_data (dict): Process data to add.
+        """
+        self.process_data.append(process_data)
+
+    def read_all_process_data(self) -> list[dict]:
+        """Gets all stored process data dicts.
+
+        Returns:
+            list[dict]: The process data in the datastore.
+        """
+        return self.process_data

pyprexor_datastore/mongo.py

@@ -0,0 +1,63 @@
+from pyprexor_datastore.datastore import Datastore
+import pymongo
+
+
+class MongoDataStore(Datastore):
+    """Basic wrapper around pymongo calls to MongoDB for pyprexor."""
+
+    def __init__(self, uri: str, database: str = "pyprexor"):
+        client: pymongo.MongoClient = pymongo.MongoClient(uri)
+
+        # check we have a good connection
+        with pymongo.timeout(0.5):
+            client.admin.command("ping")
+
+        self.db = client[database]
+
+    def get_parameter_set(self, id: str) -> dict:
+        """Get a parameter set from the parameter_sets collection
+
+        Args:
+            id (str): Id of parameter set to get.
+
+        Raises:
+            KeyError: pymongo find returns none when no document is found. We want a KeyError in this case.
+
+        Returns:
+            dict: Parameter set
+        """
+        ps = self.db.parameter_sets.find_one({"_id": id})
+        if ps:
+            return ps
+        else:
+            raise KeyError(f"Parameter set id {id} not present")
+
+    def write_process_data(self, data: dict) -> str:
+        """Write process data object to process_data collection.
+
+        Args:
+            data (dict): Data to write
+
+        Returns:
+            str: Inserted Id
+        """
+        return self.db.process_data.insert_one(data).inserted_id
+
+    def write_parameter_set(self, parameter_set: dict) -> str:
+        """Write parameter set to parameter_sets collection.
+
+        Args:
+            parameter_set (dict): Data to write
+
+        Returns:
+            str: Inserted id
+        """
+        return self.db.parameter_sets.insert_one(parameter_set).inserted_id
+
+    def read_all_process_data(self) -> list[dict]:
+        """Get all the documents present in the process_data collection.
+
+        Returns:
+            list[dict]: List of documents
+        """
+        return [data for data in self.db.process_data.find()]

pyproject.toml

@@ -4,7 +4,7 @@ version = "0.0.0"
 description = ""
 authors = ["_MircoWave_ <[email protected]>"]
 readme = "README.md"
-packages = [{include = "pyprexor"}]
+packages = [{include = "pyprexor"}, {include = "pyprexor_datastore"}]
 repository = "https://github.com/NickSebClark/pyprexor"
 
 [tool.poetry.dependencies]
@@ -16,6 +16,11 @@ pytest = "^7.4.0"
 ruff = "^0.0.285"
 mypy = "^1.5.1"
 coverage = "^7.3.0"
+mongomock = "^4.1.2"
+
+
+[tool.poetry.group.mongo.dependencies]
+pymongo = "^4.5.0"
 
 [tool.poetry-dynamic-versioning]
 enable = true

pyprexor/tests/test_core.py → tests/test_core.py

@@ -1,5 +1,5 @@
 import pyprexor.core as core
-from pyprexor.datastore import InMemoryDataStore
+from pyprexor_datastore.datastore import InMemoryDataStore
 from unittest.mock import patch
 from datetime import datetime
 import pytest
@@ -13,7 +13,7 @@ def addition_process(a: int, b: int = 2) -> int:
 
 def test_all_parameters_present():
     """Execute a process with all the parameters present and check we have the required outputs are present."""
-    data_store = InMemoryDataStore([{"id": 1, "a": 1, "b": 3}])
+    data_store = InMemoryDataStore([{"id": "1", "a": 1, "b": 3}])
     version = "1.0.0"
     user = "testuser"
 
@@ -24,11 +24,12 @@ def test_all_parameters_present():
     with patch("pyprexor.core.datetime") as mock_datetime, patch("pyprexor.core.getpass") as mock_getpass:
         mock_datetime.now.return_value = datetime(2023, 8, 21, 22, 6, 42)
         mock_getpass.getuser.return_value = user
-        process(1)
+        process("1")
 
     process_data = data_store.read_all_process_data()[0]
 
     assert process_data["datetime"] == "2023-08-21 22:06:42"
+    assert process_data["parameter_set_id"] == "1"
     assert process_data["user"] == user
     assert process_data["data"] == 4
     assert process_data["name"] == "addition_process"

pyprexor/tests/test_datastore.py → tests/test_datastore.py

@@ -1,4 +1,4 @@
-import pyprexor.datastore as datastore
+import pyprexor_datastore.datastore as datastore
 import pytest
 
 

tests/test_in_memory_datastore.py

@@ -0,0 +1,36 @@
+import pyprexor_datastore.datastore as datastore
+import pytest
+
+
+@pytest.fixture
+def default_data():
+    return [{"id": 2, "param": 8}, {"id": 3, "param": 9}, {"id": 4, "param": 10}]
+
+
+def test_init_re_index(default_data):
+    """Test a new InMemoryDatastore with the re_index option. Check the id of the first element is set to 0."""
+    store = datastore.InMemoryDataStore(default_data, re_index=True)
+
+    default_data[0]["id"] = "0"
+
+    assert default_data[0] == store.get_parameter_set("0")
+
+
+def test_init(default_data):
+    """Test a new InMemoryDatastore with default parameters"""
+    store = datastore.InMemoryDataStore(default_data)
+
+    assert default_data[0] == store.get_parameter_set(2)
+
+
+def test_read_write_process_data(default_data):
+    """Round trip of process data."""
+
+    store = datastore.InMemoryDataStore({})
+
+    store.write_process_data(default_data[0])
+    store.write_process_data(default_data[1])
+
+    returned_process_data = store.read_all_process_data()
+
+    assert returned_process_data == [default_data[0], default_data[1]]