diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 89a6f09..2fdfc61 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -18,9 +18,6 @@ jobs: python --version python -m pip install --upgrade pip pip install --upgrade setuptools wheel twine - # - name: Run Unit Tests - # run: | - # python -m unittest discover -v -f ./tests - name: Build and Package run: | python setup.py sdist bdist_wheel diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000..a8ca1e9 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,62 @@ +name: Run Unit Tests + +on: [push, pull_request] + +jobs: + unit-tests: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements.txt + - name: Run tests + run: | + python -m unittest discover -v -f ./tests/offline + + integration-tests: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + + services: + localstack: + image: localstack/localstack + ports: + - 4566:4566 + env: + SERVICES: ssm + DEFAULT_REGION: eu-west-1 + + env: + FIDELIUS_AWS_KEY_ARN: arn:aws:kms:eu-west-1:123456789012:alias/fidelius-key + FIDELIUS_AWS_REGION_NAME: eu-west-1 + FIDELIUS_AWS_ENDPOINT_URL: http://localhost:4566 + FIDELIUS_AWS_ACCESS_KEY_ID: somemadeupstuff + FIDELIUS_AWS_SECRET_ACCESS_KEY: notarealkey + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade -r requirements.txt + - name: Run tests + run: | + python -m unittest discover -v -f ./tests/localstack diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e88c2..7472871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,27 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.0.0-beta.1] - 2024-04-11 + +### Added + +- An interface for Fidelius gateway repos and admins to fulfil +- A mock implementation of a Fidelius gateway repo and admin that use a + simple singleton dict to store (and share) data during runtime +- Unittests for the mock implementation +- Unittests for the Parameter Store implementation using LocalStack +- A Factory class to get different implementation classes +- Methods to delete parameters +- Config params for the `AwsParamStoreRepo` to use a custom AWS endpoint in + order to hook up to stuff like LocalStack for testing and such + +### Changed + +- The API a little bit so we're no longer backwards compatible (hence the + major version bump to 1.0.0) +- All config params can now be explicitly given to the `AwsParamStoreRepo` + in addition to being picked up from environment variables if not supplied + ## [0.6.0] - 2024-04-05 diff --git a/README.md b/README.md index d85887c..ca82bc1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,10 @@ package in mind but should work for other cases as well. **IMPORTANT:** This has been migrated more-or-less _"as-is"_ from CCP Tool's internal repo and hasn't yet been given the love it needs to be properly open-sourced and user friendly for other people _(unless you read though the -code and find it perfectly fits your use case)_. +code and find it perfectly fits your use case)_. + +**ALSO IMPORTANT:** This README hasn't been updated to reflect changes in +version 1.0.0 yet. Sowwie! :-/ ## What should be stored with Fidelius diff --git a/fidelius/__init__.py b/fidelius/__init__.py index e16bff7..420f4a1 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.6.0' +__version__ = '1.0.0-beta.1' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/fideliusapi/__init__.py b/fidelius/fideliusapi/__init__.py index dc58555..569645b 100644 --- a/fidelius/fideliusapi/__init__.py +++ b/fidelius/fideliusapi/__init__.py @@ -1,3 +1,3 @@ -from fidelius.structs import * -from fidelius.gateway.paramstore import * -from fidelius.utils import * +from fidelius.structs.api import * +from fidelius.gateway.interface import * +from fidelius.gateway import FideliusFactory diff --git a/fidelius/gateway/__init__.py b/fidelius/gateway/__init__.py index e69de29..529fb93 100644 --- a/fidelius/gateway/__init__.py +++ b/fidelius/gateway/__init__.py @@ -0,0 +1,20 @@ +__all__ = [ + 'FideliusFactory', +] + +from fidelius.structs import * +from .interface import * +from ccptools.tpu import strimp + +import logging +log = logging.getLogger(__name__) + + +class FideliusFactory: + @staticmethod + def get_class(impl: str = 'paramstore') -> Type[IFideliusRepo]: + return strimp.get_class(f'fidelius.gateway.{impl}._std.FideliusRepo', logger=log, reraise=True) + + @staticmethod + def get_admin_class(impl: str = 'paramstore') -> Type[IFideliusAdminRepo]: + return strimp.get_class(f'fidelius.gateway.{impl}._std.FideliusAdmin', logger=log, reraise=True) diff --git a/fidelius/gateway/_abstract.py b/fidelius/gateway/_abstract.py new file mode 100644 index 0000000..4a62576 --- /dev/null +++ b/fidelius/gateway/_abstract.py @@ -0,0 +1,183 @@ +__all__ = [ + '_BaseFideliusRepo', + '_BaseFideliusAdminRepo', +] +from .interface import * + +from fidelius.structs import * +import logging +log = logging.getLogger(__file__) + + +class _BaseFideliusRepo(IFideliusRepo, abc.ABC): + """Covers a lot of basic functionality common across most storage back-ends. + """ + + _APP_PATH_FORMAT = '/fidelius/{group}/{env}/apps/{app}/{name}' + _SHARED_PATH_FORMAT = '/fidelius/{group}/{env}/shared/{folder}/{name}' + + _EXPRESSION_APP_FORMAT = '${{__FID__:{name}}}' + _EXPRESSION_SHARED_FORMAT = '${{__FID__:{folder}:{name}}}' + + _EXPRESSION_PATTERN = re.compile(r'\${__FID__:(?:(?P\w+):)?(?P[\w/-]+)}') + + def __init__(self, app_props: FideliusAppProps, **kwargs): + log.debug('_BaseFideliusRepo.__init__') + self._app_props = app_props + + # Any kwargs should have been handled by implementation specific stuff, + # unless there's a derpy config so let's warn just in case! + if kwargs: + for k, v in kwargs.items(): + log.warning(f'AbstractFideliusRepo for some unhandled kwargs: {k}={v}') + + @property + def app_props(self) -> FideliusAppProps: + """The current application properties. + """ + return self._app_props + + def make_app_path(self, env: Optional[str] = None) -> str: + """The full path to application specific parameters/secrets. + """ + return self._APP_PATH_FORMAT.format(group=self.app_props.group, + env=env or self.app_props.env, + app=self.app_props.app, + name='{name}') + + def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: + """The full path to group shared parameters/secrets. + """ + return self._SHARED_PATH_FORMAT.format(group=self.app_props.group, + env=env or self.app_props.env, + folder=folder, + name='{name}') + + def get_expression_string(self, name: str, folder: Optional[str] = None) -> str: + """Return a Fidelius expression string (e.g. to use in configuration + files) which the `replace` method can parse and replace with + parameters/secrets fetched from the storage backend. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + :param name: The name of the parameter/secret to create an expression for. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific expression. + :return: The Fidelius expression string. + """ + if folder: + return self._EXPRESSION_SHARED_FORMAT.format(name=name, folder=folder) + else: + return self._EXPRESSION_APP_FORMAT.format(name=name) + + def get_full_path(self, name: str, folder: Optional[str] = None, env: Optional[str] = None) -> str: + """Gets the full path to a parameter/secret. + + Parameter/secret paths can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to use a shared or app path based on whether or + not a folder name was given. + + :param name: The name of the parameter/secret to build a path to. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret path. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The full path to a parameter/secret. + """ + if folder: + return self.make_shared_path(folder, env=env).format(name=name) + else: + return self.make_app_path(env=env).format(name=name) + + def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + """Gets the given parameter/secret from the storage this repo uses. + + Parameters/secrets can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to get a shared or app parameter based on whether or + not a folder name was given. + + Unless disabled by the `no_default` parameter, this method will attempt + to fetch a parameter using the `env=default` if one was not found for + the env in the current app properties. + + :param name: The name of the parameter/secret to get. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret. + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested parameter/secret or None if it was not found. + """ + log.debug('_BaseFideliusRepo.get(name=%s, folder=%s, no_default=%s))', name, folder, no_default) + if folder: + val = self.get_shared_param(name=name, folder=folder) + log.debug('_BaseFideliusRepo.get->get_shared_param val=%s', val) + if val is not None: + return val + + if no_default: + log.debug('_BaseFideliusRepo.get->(shared) no_default STOP!') + return None + + log.debug('_BaseFideliusRepo.get->(shared) Lets try the default!!!') + return self.get_shared_param(name=name, folder=folder, env='default') + else: + val = self.get_app_param(name=name) + log.debug('_BaseFideliusRepo.get->get_app_param val=%s', val) + if val is not None: + return val + + if no_default: + log.debug('_BaseFideliusRepo.get->(app) no_default STOP!') + return None + + log.debug('_BaseFideliusRepo.get->(app) Lets try the default!!!') + return self.get_app_param(name=name, env='default') + + def replace(self, string: str, no_default: bool = False) -> str: + """Take in a containing a Fidelius parameter/secret configuration + expression (e.g. read from a config file), parses it, fetches the + relevant parameter/secret and returns. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + An empty string is returned if the parameter was not found and if the + string does not match the expression format, it will be returned + unchanged. + + :param string: The expression to replace with an actual parameter/secret + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested value, an empty string or the original string + """ + m = self._EXPRESSION_PATTERN.match(string) + if m: + return self.get(m.group('name'), m.group('folder'), no_default=no_default) or '' + return string + + +class _BaseFideliusAdminRepo(_BaseFideliusRepo, IFideliusAdminRepo, abc.ABC): + """Covers a lot of admin basic functionality common across most storage back-ends. + """ + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + log.debug('_BaseFideliusAdminRepo.__init__ (this should set tags?!?)') + super().__init__(app_props, **kwargs) + self._tags = tags + + @property + def tags(self) -> Optional[FideliusTags]: + return self._tags + + def set_env(self, env: str): + self.app_props.env = env diff --git a/fidelius/gateway/interface.py b/fidelius/gateway/interface.py new file mode 100644 index 0000000..5be615a --- /dev/null +++ b/fidelius/gateway/interface.py @@ -0,0 +1,359 @@ +__all__ = [ + 'IFideliusRepo', + 'IFideliusAdminRepo', +] +import abc +from fidelius.structs import * + + +class IFideliusRepo(abc.ABC): + @abc.abstractmethod + def __init__(self, app_props: FideliusAppProps, **kwargs): + """Initialise a new Fidelius Repository. + + :param app_props: The application properties to use. + """ + pass + + @property + @abc.abstractmethod + def app_props(self) -> FideliusAppProps: + """The current application properties. + """ + pass + + @abc.abstractmethod + def make_app_path(self, env: Optional[str] = None) -> str: + """The full path to application specific parameters/secrets. + """ + pass + + @abc.abstractmethod + def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: + """The full path to group shared parameters/secrets. + """ + pass + + @abc.abstractmethod + def get_expression_string(self, name: str, folder: Optional[str] = None) -> str: + """Return a Fidelius expression string (e.g. to use in configuration + files) which the `replace` method can parse and replace with + parameters/secrets fetched from the storage backend. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + :param name: The name of the parameter/secret to create an expression for. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific expression. + :return: The Fidelius expression string. + """ + pass + + @abc.abstractmethod + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + """Gets an application specific parameter/secret. + + :param name: The name of the parameter/secret to get. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The requested parameter/secret or None if it was not found. + """ + pass + + @abc.abstractmethod + def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + """Gets the given parameter/secret from the storage this repo uses. + + Parameters/secrets can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to get a shared or app parameter based on whether or + not a folder name was given. + + Unless disabled by the `no_default` parameter, this method will attempt + to fetch a parameter using the `env=default` if one was not found for + the env in the current app properties. + + :param name: The name of the parameter/secret to get. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret. + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested parameter/secret or None if it was not found. + """ + pass + + @abc.abstractmethod + def get_full_path(self, name: str, folder: Optional[str] = None, env: Optional[str] = None) -> str: + """Gets the full path to a parameter/secret. + + Parameter/secret paths can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to use a shared or app path based on whether or + not a folder name was given. + + :param name: The name of the parameter/secret to build a path to. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret path. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The full path to a parameter/secret. + """ + pass + + @abc.abstractmethod + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + """Gets a group shared parameter/secret. + + :param name: The name of the parameter/secret to get. + :param folder: The folder where the group shared parameter/secret is + located. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The requested parameter/secret or None if it was not found. + """ + pass + + @abc.abstractmethod + def replace(self, string: str, no_default: bool = False) -> str: + """Take in a containing a Fidelius parameter/secret configuration + expression (e.g. read from a config file), parses it, fetches the + relevant parameter/secret and returns. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + An empty string is returned if the parameter was not found and if the + string does not match the expression format, it will be returned + unchanged. + + :param string: The expression to replace with an actual parameter/secret + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested value, an empty string or the original string + """ + pass + + +class IFideliusAdminRepo(IFideliusRepo): + @abc.abstractmethod + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + """Initialise a new Fidelius Admin Repository. + + :param app_props: The application properties to use. + :param tags: An optional set of meta-data tags to use when creating new + parameters (if supported by the underlying + parameter/secret storage). Note that updating a parameter + does not update/change tags, they are only applied when + creating new parameters! + """ + pass + + @property + @abc.abstractmethod + def tags(self) -> Optional[FideliusTags]: + """The tags to use when creating new parameters. + """ + pass + + @abc.abstractmethod + def set_env(self, env: str): + """Sets the current app properties env to something else (for admin purposes) + """ + pass + + @abc.abstractmethod + def create_param(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new unencrypted application parameter. + + :param name: The parameter name. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def update_param(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing unencrypted application parameter. + + :param name: The parameter name. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_param(self, name: str, env: Optional[str] = None): + """Deletes an existing unencrypted application parameter. + + :param name: The parameter name. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ + pass + + @abc.abstractmethod + def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new unencrypted group-shared parameter under the given + shared folder. + + :param name: The parameter name. + :param folder: The shared folder to place the parameter under. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing unencrypted group-shared parameter under the + given shared folder. + + :param name: The parameter name. + :param folder: The shared folder to place the parameter under. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + """Deletes an existing unencrypted group-shared parameter under the + given shared folder. + + :param name: The parameter name. + :param folder: The shared folder parameter is placed under. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ + pass + + @abc.abstractmethod + def create_secret(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new encrypted application secret. + + :param name: The secret name. + :param value: The secret value. + :param description: An optional description of the secret (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def update_secret(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing encrypted application secret. + + :param name: The secret name. + :param value: The secret value. + :param description: An optional description of the secret (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_secret(self, name: str, env: Optional[str] = None): + """Deletes an existing encrypted application secret. + + :param name: The parameter name. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ + pass + + @abc.abstractmethod + def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new encrypted group-shared secret under the given + shared folder. + + :param name: The secret name. + :param folder: The shared folder to place the secret under. + :param value: The secret value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing encrypted group-shared secret under the + given shared folder. + + :param name: The secret name. + :param folder: The shared folder to place the secret under. + :param value: The secret value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + """Deletes an existing encrypted group-shared secret under the + given shared folder. + + :param name: The secret name. + :param folder: The shared folder secret is placed under. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ + pass diff --git a/fidelius/gateway/mock/__init__.py b/fidelius/gateway/mock/__init__.py new file mode 100644 index 0000000..50bf551 --- /dev/null +++ b/fidelius/gateway/mock/__init__.py @@ -0,0 +1,2 @@ +from ._mockrepo import * +from ._mockadmin import * diff --git a/fidelius/gateway/mock/_inmemcache.py b/fidelius/gateway/mock/_inmemcache.py new file mode 100644 index 0000000..8befe1a --- /dev/null +++ b/fidelius/gateway/mock/_inmemcache.py @@ -0,0 +1,19 @@ +__all__ = [ + '_SingletonDict', +] + +from ccptools.structs import * + + +class _SingletonDict(dict, metaclass=Singleton): + """Simple Singleton dict :) + + It's point is simply to act as a shared centralized store for the mock + stuff, mimicking how multiple instances of Fidelius Repos and/or Admin + Repos would nevertheless fetch data from the same source. + + This is just to "mock" the shared parameter/secret stuff. + + ...sneaky, right? + """ + pass diff --git a/fidelius/gateway/mock/_mockadmin.py b/fidelius/gateway/mock/_mockadmin.py new file mode 100644 index 0000000..5684006 --- /dev/null +++ b/fidelius/gateway/mock/_mockadmin.py @@ -0,0 +1,95 @@ +__all__ = [ + 'MockFideliusAdmin', +] + +from fidelius.gateway._abstract import * +from fidelius.structs import * +from ._mockrepo import * + +import logging +log = logging.getLogger(__name__) + + +class MockFideliusAdmin(_BaseFideliusAdminRepo, MockFideliusRepo): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + """This mock version of the Fidelius Admin stores created and updated + params in memory only (although the cache is a singleton so multiple + instances of both admin and repo will be useing the same dict/data. + + Note that it does NOT extend the functionality of its non-Admin sibling, + the MockFideliusRepo and thus does not return a base64 encoded version + of every requested param/secret key name, but instead only uses its own + internal in-memory cache, and thus, `get` will not return anything that + hasn't been created first during that particular runtime. + + This is mainly intended for unit testing other packages and apps that + use Fidelius. + """ + log.debug('MockFideliusAdmin.__init__') + super().__init__(app_props, tags, **kwargs) + + def _create(self, name: str, value: str, env: Optional[str] = None, folder: Optional[str] = None) -> (str, str): + key = self.get_full_path(name, folder=folder, env=env) + if key in self._cache: + raise FideliusParameterAlreadyExists(f'parameter already exists: {key}') + self._cache[key] = value + return key, self.get_expression_string(name, folder=folder) + + def _update(self, name: str, value: str, env: Optional[str] = None, folder: Optional[str] = None) -> (str, str): + key = self.get_full_path(name, folder=folder, env=env) + if key not in self._cache: + raise FideliusParameterNotFound(f'parameter not found: {key}') + self._cache[key] = value + return key, self.get_expression_string(name, folder=folder) + + def _delete(self, name: str, env: Optional[str] = None, folder: Optional[str] = None): + key = self.get_full_path(name, folder=folder, env=env) + if key not in self._cache: + raise FideliusParameterNotFound(f'parameter not found: {key}') + del self._cache[key] + + def create_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env) + + def update_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env) + + def delete_param(self, name: str, env: Optional[str] = None): + self._delete(name, env=env) + + def create_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, + env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env, folder=folder) + + def update_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, + env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env, folder=folder) + + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + self._delete(name, env=env, folder=folder) + + def create_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env) + + def update_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env) + + def delete_secret(self, name: str, env: Optional[str] = None): + self._delete(name, env=env) + + def create_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env, folder=folder) + + def update_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env, folder=folder) + + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + self._delete(name, env=env, folder=folder) diff --git a/fidelius/gateway/mock/_mockrepo.py b/fidelius/gateway/mock/_mockrepo.py new file mode 100644 index 0000000..7af693f --- /dev/null +++ b/fidelius/gateway/mock/_mockrepo.py @@ -0,0 +1,31 @@ +__all__ = [ + 'MockFideliusRepo', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * +from ._inmemcache import _SingletonDict + +import base64 + +import logging +log = logging.getLogger(__name__) + + +class MockFideliusRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, **kwargs): + """This mock variation of the FideliusRepo simply returns a base64 + encoded version of the full path of the requested parameter/secret. + + This is mainly intended for unit testing other packages and apps that + use Fidelius. + """ + log.debug('MockFideliusRepo.__init__') + super().__init__(app_props, **kwargs) + self._cache: _SingletonDict[str, str] = _SingletonDict() + + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + return self._cache.get(self.get_full_path(name, env=env), None) + + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + return self._cache.get(self.get_full_path(name, folder, env=env), None) diff --git a/fidelius/gateway/mock/_std.py b/fidelius/gateway/mock/_std.py new file mode 100644 index 0000000..3dc55cd --- /dev/null +++ b/fidelius/gateway/mock/_std.py @@ -0,0 +1,4 @@ +"""Here we import this modules implementation classes with generic names for the +Factory to use""" +from ._mockrepo import MockFideliusRepo as FideliusRepo +from ._mockadmin import MockFideliusAdmin as FideliusAdmin diff --git a/fidelius/gateway/paramadmin.py b/fidelius/gateway/paramadmin.py deleted file mode 100644 index 55c71df..0000000 --- a/fidelius/gateway/paramadmin.py +++ /dev/null @@ -1,99 +0,0 @@ -__all__ = [ - 'ParameterStoreAdmin', -] -from .paramstore import * -from fidelius.structs import * - -import logging -log = logging.getLogger(__name__) - - -class ParameterStoreAdmin(ParameterStore): - def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): - super().__init__(app, group, env) - self._tags = Tags(application=self._app, owner=owner, tier=env, finance=finance, **extra_tags) - - def set_env(self, env: str): - self._env = env - self._tags.tier = env - - def _set_parameter(self, - full_name: str, - value: str, - encrypted: bool = False, - overwrite: bool = False, - description: Optional[str] = None) -> Dict: - kwargs = dict(Name=full_name, - Description=description or full_name, - Value=value, - Type='SecureString' if encrypted else 'String', - Overwrite=overwrite, - Tags=self._tags.to_aws_format(), - Tier='Standard') - if encrypted: - kwargs['KeyId'] = self._KEY_ID - - response = self._ssm.put_parameter(**kwargs) - return response - - def create_param(self, name: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=False, - encrypted=False) - - def update_param(self, name: str, value: str, description: Optional[str] = None): - if self.get(name=name, no_default=True): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=True, - encrypted=False) - else: - raise ValueError('that parameter does not exists yet, use create_param') - - def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), - value=value, - description=description, - overwrite=False, - encrypted=False) - - def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), - value=value, - description=description, - overwrite=True, - encrypted=False) - - def create_secret(self, name: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=False, - encrypted=True) - - def update_secret(self, name: str, value: str, description: Optional[str] = None): - if self.get(name=name, no_default=True): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=True, - encrypted=True) - else: - raise ValueError('that secret does not exists yet, use create_secret') - - def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), - value=value, - description=description, - overwrite=False, - encrypted=True) - - def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), - value=value, - description=description, - overwrite=True, - encrypted=True) diff --git a/fidelius/gateway/paramstore.py b/fidelius/gateway/paramstore.py deleted file mode 100644 index 428feb7..0000000 --- a/fidelius/gateway/paramstore.py +++ /dev/null @@ -1,91 +0,0 @@ -__all__ = [ - 'ParameterStore', -] - -from fidelius.structs import * - -import boto3 -import os - -import logging -log = logging.getLogger(__name__) - - -class ParameterStore(object): - _KEY_ID = os.environ.get('FIDELIUS_AWS_KEY_ARN', '') - _DEFAULT_REGION_NAME = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') - _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' - _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' - - def __init__(self, app: str, group: str, env: str): - if not self._KEY_ID: - raise RuntimeError('Fidelius requires the ARN for the KMS key to use to be in the FIDELIUS_AWS_KEY_ARN environment variable') - - self._app = app - self._group = group - self._env = env - self._force_log_secrecy() - self._ssm = boto3.client('ssm', - region_name=os.environ.get('FIDELIUS_AWS_REGION_NAME', self._DEFAULT_REGION_NAME), - aws_access_key_id=os.environ.get('FIDELIUS_AWS_ACCESS_KEY_ID', None), - aws_secret_access_key=os.environ.get('FIDELIUS_AWS_SECRET_ACCESS_KEY', None)) - - self._cache: Dict[str, str] = {} - self._loaded: bool = False - self._loaded_folders: Set[str] = set() - self._default_store: Optional[ParameterStore] = None - if self._env != 'default': - self._default_store = ParameterStore(self._app, self._group, 'default') - - def _full_path(self, name: str, folder: Optional[str] = None) -> str: - if folder: - return self._SHARED_FULL_NAME.format(group=self._group, folder=folder, env=self._env, name=name) - else: - return self._APP_FULL_NAME.format(group=self._group, app=self._app, env=self._env, name=name) - - def _nameless_path(self, folder: Optional[str] = None) -> str: - if folder: - return self._full_path(name='', folder=folder)[:-1] - else: - return self._full_path(name='', folder=folder)[:-1] - - def _force_log_secrecy(self): - # We don't allow debug or less logging of botocore's HTTP requests cause - # those logs have unencrypted passwords in them! - botolog = logging.getLogger('botocore') - if botolog.level < logging.INFO: - botolog.setLevel(logging.INFO) - - def _load_all(self, folder: Optional[str] = None): - self._force_log_secrecy() - if folder: - if folder in self._loaded_folders: - return - else: - if self._loaded: - return - - response = self._ssm.get_parameters_by_path( - Path=self._nameless_path(folder), - Recursive=True, - WithDecryption=True - ) - for p in response.get('Parameters', []): - key = p.get('Name') - if key: - self._cache[key] = p.get('Value') - - if folder: - self._loaded_folders.add(folder) - else: - self._loaded = True - - def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: - self._load_all(folder) - return self._cache.get(self._full_path(name, folder), - None if no_default else self._get_default(name, folder)) - - def _get_default(self, name: str, folder: Optional[str] = None) -> Optional[str]: - if self._default_store: - return self._default_store.get(name, folder) - return None diff --git a/fidelius/gateway/paramstore/__init__.py b/fidelius/gateway/paramstore/__init__.py new file mode 100644 index 0000000..cf4483a --- /dev/null +++ b/fidelius/gateway/paramstore/__init__.py @@ -0,0 +1,2 @@ +from ._paramstorerepo import * +from ._paramstoreadmin import * diff --git a/fidelius/gateway/paramstore/_paramstoreadmin.py b/fidelius/gateway/paramstore/_paramstoreadmin.py new file mode 100644 index 0000000..79d168d --- /dev/null +++ b/fidelius/gateway/paramstore/_paramstoreadmin.py @@ -0,0 +1,161 @@ +__all__ = [ + 'AwsParameterStoreAdmin', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * +from ._paramstorerepo import * + +import logging +log = logging.getLogger(__name__) + + +class AwsParameterStoreAdmin(_BaseFideliusAdminRepo, AwsParamStoreRepo): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + log.debug('AwsParameterStoreAdmin.__init__') + super().__init__(app_props, tags, **kwargs) + + def create_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + res = self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=False, + encrypted=False) + return path, self.get_expression_string(name) + + def update_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + if self.get_app_param(name, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') + + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=True, + encrypted=False) + return path, self.get_expression_string(name) + + def delete_param(self, name: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, env=env)) + + def create_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=False, + encrypted=False) + return path, self.get_expression_string(name, folder=folder) + + def update_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + if self.get_shared_param(name, folder=folder, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') + + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=True, + encrypted=False) + return path, self.get_expression_string(name, folder=folder) + + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, folder=folder, env=env)) + + def create_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=False, + encrypted=True) + return path, self.get_expression_string(name) + + def update_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + if self.get_app_param(name, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') + + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=True, + encrypted=True) + return path, self.get_expression_string(name) + + def delete_secret(self, name: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, env=env)) + + def create_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=False, + encrypted=True) + return path, self.get_expression_string(name, folder=folder) + + def update_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + if self.get_shared_param(name, folder=folder, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') + + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=True, + encrypted=True) + return path, self.get_expression_string(name, folder=folder) + + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, folder=folder, env=env)) + + def _tags_to_aws_format(self) -> Optional[List[Dict[str, str]]]: + if self.tags: + return [{'Key': k, 'Value': v} for k, v in self.tags.to_dict().items()] + return None + + def _set_parameter(self, + full_name: str, + value: str, + encrypted: bool = False, + overwrite: bool = False, + description: Optional[str] = None) -> Dict: + kwargs = dict(Name=full_name, + Description=description or full_name, + Value=value, + Type='SecureString' if encrypted else 'String', + Overwrite=overwrite, + Tier='Standard') + if not overwrite: + tags = self._tags_to_aws_format() + if tags: + kwargs['Tags'] = tags + + if encrypted: + kwargs['KeyId'] = self._aws_key_arn + + try: + response = self._ssm.put_parameter(**kwargs) + return response + except self._ssm.exceptions.ParameterAlreadyExists: + raise FideliusParameterAlreadyExists(f'parameter already exists: {full_name}') + + except self._ssm.exceptions.ParameterNotFound: + raise FideliusParameterNotFound(f'parameter not found: {full_name}') + + def _delete_parameter(self, full_name: str) -> Dict: + try: + response = self._ssm.delete_parameter(Name=full_name) + return response + except self._ssm.exceptions.ParameterNotFound: + raise FideliusParameterNotFound(f'parameter not found: {full_name}') diff --git a/fidelius/gateway/paramstore/_paramstorerepo.py b/fidelius/gateway/paramstore/_paramstorerepo.py new file mode 100644 index 0000000..55556e1 --- /dev/null +++ b/fidelius/gateway/paramstore/_paramstorerepo.py @@ -0,0 +1,117 @@ +__all__ = [ + 'AwsParamStoreRepo', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * + + +import boto3 +import os + +import logging +log = logging.getLogger(__name__) + + +class AwsParamStoreRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, + aws_access_key_id: str = None, + aws_secret_access_key: str = None, + aws_key_arn: str = None, + aws_region_name: str = None, + aws_endpoint_url: str = None, + flush_cache_every_time: bool = False, + **kwargs): + """Fidelius Admin Repo that uses AWS' Simple Systems Manager's Parameter Store as a back end. + + :param app_props: The current application properties. + :param aws_access_key_id: Optional AWS_ACCESS_KEY_ID which is otherwise + pulled from the FIDELIUS_AWS_ACCESS_KEY_ID or + AWS_ACCESS_KEY_ID environment variables. + :param aws_secret_access_key: Optional AWS_SECRET_ACCESS_KEY which is otherwise + pulled from the FIDELIUS_AWS_SECRET_ACCESS_KEY or + AWS_SECRET_ACCESS_KEY environment variables. + :param aws_key_arn: Optional ARN to an AWS KMS encryption key that'll be + used to encrypt secret parameters. If not provided + it'll be extracted from the FIDELIUS_AWS_KEY_ARN + environment variable and if that is missing as well, + an EnvironmentError is raised. + :param aws_region_name: Optional AWS region name, which is otherwise + extracted from the FIDELIUS_AWS_REGION_NAME or + AWS_DEFAULT_REGION environment variables or just + set to `eu-west-1` by default is completely + missing. + :param aws_endpoint_url: Optional custom AWS endpoint URL intended for + testing and development, e.g. by spinning up a + LocalStack container and pointing to that + instead of a live AWS environment. + :param flush_cache_every_time: Optional flat that'll flush the entire + cache before every operation if set to + True and is just intended for testing + purposes. + """ + super().__init__(app_props, **kwargs) + self._flush_cache_every_time = flush_cache_every_time + + self._aws_key_arn = aws_key_arn or os.environ.get('FIDELIUS_AWS_KEY_ARN', '') + if not self._aws_key_arn: + raise EnvironmentError('Fidelius AwsParamStoreRepo requires the ARN for the KMS key argument when initialising or in the FIDELIUS_AWS_KEY_ARN environment variable') + + self._region_name = aws_region_name or os.environ.get('FIDELIUS_AWS_REGION_NAME', None) or os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') + + self._aws_endpoint_url = aws_endpoint_url or os.environ.get('FIDELIUS_AWS_ENDPOINT_URL', '') + + self._force_log_secrecy() + self._ssm = boto3.client('ssm', + region_name=self._region_name, + endpoint_url=self._aws_endpoint_url or None, + aws_access_key_id=aws_access_key_id or os.environ.get('FIDELIUS_AWS_ACCESS_KEY_ID', None) or os.environ.get('AWS_ACCESS_KEY_ID', None), + aws_secret_access_key=aws_secret_access_key or os.environ.get('FIDELIUS_AWS_SECRET_ACCESS_KEY', None) or os.environ.get('AWS_SECRET_ACCESS_KEY', None)) + + self._cache: Dict[str, str] = {} + self._loaded_paths: Set[str] = set() + + def _nameless_path(self, folder: Optional[str] = None, env: Optional[str] = None) -> str: + return self.get_full_path(name='', folder=folder, env=env)[:-1] + + @staticmethod + def _force_log_secrecy(): + # We don't allow debug or less logging of botocore's HTTP requests cause + # those logs have unencrypted passwords in them! + botolog = logging.getLogger('botocore') + if botolog.level < logging.INFO: + botolog.setLevel(logging.INFO) + + def _load_path(self, folder: Optional[str] = None, env: Optional[str] = None): + log.debug('AwsParamStoreRepo._load_path(folder=%s, env=%s)', folder, env) + self._force_log_secrecy() + + # This is stuff for unit-testing only! + if self._flush_cache_every_time: + self._loaded_paths = set() + self._cache = {} + + path = self._nameless_path(folder, env) + if path in self._loaded_paths: + return + + response = self._ssm.get_parameters_by_path( + Path=path, + Recursive=True, + WithDecryption=True + ) + + for p in response.get('Parameters', []): + key = p.get('Name') + if key: + self._cache[key] = p.get('Value') + + self._loaded_paths.add(path) + + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + self._load_path(env=env) + return self._cache.get(self.get_full_path(name, env=env), None) + + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + self._load_path(folder=folder, env=env) + return self._cache.get(self.get_full_path(name, folder, env=env), None) diff --git a/fidelius/gateway/paramstore/_std.py b/fidelius/gateway/paramstore/_std.py new file mode 100644 index 0000000..87552f2 --- /dev/null +++ b/fidelius/gateway/paramstore/_std.py @@ -0,0 +1,2 @@ +from ._paramstorerepo import AwsParamStoreRepo as FideliusRepo +from ._paramstoreadmin import AwsParameterStoreAdmin as FideliusAdmin diff --git a/fidelius/structs/__init__.py b/fidelius/structs/__init__.py index 0b83f41..8c7ec00 100644 --- a/fidelius/structs/__init__.py +++ b/fidelius/structs/__init__.py @@ -1,2 +1,2 @@ from ._base import * -from ._tags import * +from .api import * diff --git a/fidelius/structs/_appprops.py b/fidelius/structs/_appprops.py new file mode 100644 index 0000000..d8fb0af --- /dev/null +++ b/fidelius/structs/_appprops.py @@ -0,0 +1,22 @@ +__all__ = [ + 'FideliusAppProps', +] +import dataclasses + + +@dataclasses.dataclass +class FideliusAppProps: + """Application properties, used to get or set specific variations of + parameters/secrets. + + :ivar app: The name of the application itself (module name or slug) + :ivar group: The app's "group" (think "business domain" or "namespace"). + The purpose of the "group" attribute is to categorize shared + parameters/secrets such that all apps in the same group, can + access all the shared (non-app specific) parameters/secrets of + that group. + :ivar env: Deployment environment (e.g., "prod", "dev", "test"). + """ + app: str + group: str + env: str = 'default' diff --git a/fidelius/structs/_base.py b/fidelius/structs/_base.py index 36f969b..b0a7460 100644 --- a/fidelius/structs/_base.py +++ b/fidelius/structs/_base.py @@ -1,3 +1 @@ -from typing import * - -import datetime +from ccptools.structs import * diff --git a/fidelius/structs/_errors.py b/fidelius/structs/_errors.py new file mode 100644 index 0000000..5fc9218 --- /dev/null +++ b/fidelius/structs/_errors.py @@ -0,0 +1,22 @@ +__all__ = [ + 'FideliusError', + 'FideliusAdminError', + 'FideliusParameterNotFound', + 'FideliusParameterAlreadyExists', +] + + +class FideliusError(Exception): + pass + + +class FideliusAdminError(FideliusError): + pass + + +class FideliusParameterNotFound(FideliusAdminError, KeyError): + pass + + +class FideliusParameterAlreadyExists(FideliusAdminError, ValueError): + pass diff --git a/fidelius/structs/_tags.py b/fidelius/structs/_tags.py index fff0478..848f471 100644 --- a/fidelius/structs/_tags.py +++ b/fidelius/structs/_tags.py @@ -1,10 +1,10 @@ __all__ = [ - 'Tags', + 'FideliusTags', ] from ._base import * -class Tags: +class FideliusTags: __slots__ = ('application', 'owner', 'tier', 'finance', '_other') def __init__(self, application: str, owner: str, tier: str = 'default', finance: str = 'COST', **kwargs): @@ -32,6 +32,10 @@ def __setattr__(self, name: str, value: Optional[str]): def __delattr__(self, name: str): self.__setattr__(name, None) + def __repr__(self) -> str: + tags = ', '.join([f"{k}='{v}'" for k, v in self.to_dict().items()]) + return f'{self.__class__.__name__}({tags})' + def to_dict(self) -> Dict[str, str]: d = { 'application': self.application, @@ -42,6 +46,3 @@ def to_dict(self) -> Dict[str, str]: if self._other: d.update(self._other) return d - - def to_aws_format(self) -> List[Dict[str, str]]: - return [{'Key': k, 'Value': v} for k, v in self.to_dict().items()] diff --git a/fidelius/structs/api.py b/fidelius/structs/api.py new file mode 100644 index 0000000..63287ce --- /dev/null +++ b/fidelius/structs/api.py @@ -0,0 +1,3 @@ +from ._tags import * +from ._appprops import * +from ._errors import * diff --git a/fidelius/utils/__init__.py b/fidelius/utils/__init__.py deleted file mode 100644 index acc496b..0000000 --- a/fidelius/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .replacer import * diff --git a/fidelius/utils/replacer.py b/fidelius/utils/replacer.py deleted file mode 100644 index c9267bc..0000000 --- a/fidelius/utils/replacer.py +++ /dev/null @@ -1,15 +0,0 @@ -__all__ = [ - 'fidelius_replace', -] -import re -from fidelius.gateway.paramstore import ParameterStore - -_VAR_PATTERN = re.compile(r'\${__FID__:(?:(?P\w+):)?(?P[\w/-]+)}') - - -def fidelius_replace(value: str, pmstore: ParameterStore) -> str: - m = _VAR_PATTERN.match(value) - if m: - return pmstore.get(m.group('name'), m.group('shared_group')) - return value - diff --git a/pyproject.toml b/pyproject.toml index ad14603..ac234f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ classifiers = [ "Topic :: Utilities" ] dependencies = [ - "boto3 >=1.20, <2" + "ccptools >=1.1, <2", + "boto3 >=1.20, <2" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 83e2cbe..1269059 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -boto3 >=1.20, <2 \ No newline at end of file +ccptools >=1.1, <2 +boto3 >=1.20, <2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/localstack/__init__.py b/tests/localstack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/localstack/test_paramstore.py b/tests/localstack/test_paramstore.py new file mode 100644 index 0000000..99b67f3 --- /dev/null +++ b/tests/localstack/test_paramstore.py @@ -0,0 +1,260 @@ +import unittest + +from fidelius.fideliusapi import * +from typing import * + +import os + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + +_APP_PROPS = FideliusAppProps(app='mock-app', group='tempunittestgroup', env='mock') +_APP_PROPS_TWO = FideliusAppProps(app='other-app', group='tempunittestgroup', env='mock') +_APP_PROPS_THREE = FideliusAppProps(app='mock-app', group='tempunittestgroup', env='test') + + +def _extract_param_names(response: Dict) -> List[str]: + return [p.get('Name') for p in response.get('Parameters', [])] + + +def _get_param_names_by_path(ssm, path: str) -> List[str]: + return _extract_param_names(ssm.get_parameters_by_path(Path=path, + Recursive=True, + WithDecryption=False)) + + +def _delete_all_params(): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + ssm = fia._ssm # noqa + paths = [ + '/fidelius/tempunittestgroup/', + ] + param_set = set() + for p in paths: + param_set.update(_get_param_names_by_path(ssm, p)) + + if param_set: + res = ssm.delete_parameters(Names=list(param_set)) + log.debug('_delete_all_params -> %s', res) + + +class TestParamstore(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Check if certain environment variables are set + env_var = os.getenv('FIDELIUS_AWS_ENDPOINT_URL') + if not env_var: + raise unittest.SkipTest("Environment variable 'FIDELIUS_AWS_ENDPOINT_URL' not set. " + "Skipping all tests in TestParamstore.") + # Just in case...! + _delete_all_params() + + def test_paramstore(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('MY_DB_NAME', 'brain') + fia.create_secret('MY_DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('MY_DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('MY_DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('MY_DB_NAME')) + self.assertEqual('myBADpassword', fid.get('MY_DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('MY_DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('MY_DB_USERNAME', 'dbstuff')) + _delete_all_params() + + def test_paramstore_admin_params(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_TWO')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_param('DB_PASSWORD_TWO') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_param('DB_PASSWORD_TWO', 'myBADpassword') + + key, expression = fia.create_param('DB_PASSWORD_TWO', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_TWO', key) + self.assertEqual('${__FID__:DB_PASSWORD_TWO}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_TWO')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_param('DB_PASSWORD_TWO', 'myWORSEpassword') + + key, expression = fia.update_param('DB_PASSWORD_TWO', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_TWO', key) + self.assertEqual('${__FID__:DB_PASSWORD_TWO}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_TWO')) + + fia.delete_param('DB_PASSWORD_TWO') + + self.assertIsNone(fia.get('DB_PASSWORD_TWO')) + _delete_all_params() + + def test_paramstore_admin_secrets(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_THREE')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_secret('DB_PASSWORD_THREE') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_secret('DB_PASSWORD_THREE', 'myBADpassword') + + key, expression = fia.create_secret('DB_PASSWORD_THREE', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_THREE', key) + self.assertEqual('${__FID__:DB_PASSWORD_THREE}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_THREE')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_secret('DB_PASSWORD_THREE', 'myWORSEpassword') + + key, expression = fia.update_secret('DB_PASSWORD_THREE', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_THREE', key) + self.assertEqual('${__FID__:DB_PASSWORD_THREE}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_THREE')) + + fia.delete_secret('DB_PASSWORD_THREE') + + self.assertIsNone(fia.get('DB_PASSWORD_THREE')) + _delete_all_params() + + def test_paramstore_admin_shared_params(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia2 = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS_TWO, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FOUR', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FOUR}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FOUR', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FOUR}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + + fia.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + _delete_all_params() + + def test_paramstore_admin_shared_secrets(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia2 = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS_TWO, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FIVE', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FIVE}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FIVE', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FIVE}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + + fia.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + _delete_all_params() + + def test_paramstore_defaults(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('DB_NAME', 'brain', env='default') + fia.create_secret('DB_PASSWORD', 'defaultPassword', env='default') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.localhost', env='default') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock', env='default') + + fia.create_secret('DB_PASSWORD', 'mockPassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + + fid2 = FideliusFactory.get_class('paramstore')(_APP_PROPS_THREE, flush_cache_every_time=True) + self.assertIsNone(fid2.get('mock-value')) + self.assertEqual('brain', fid2.get('DB_NAME')) + self.assertEqual('defaultPassword', fid2.get('DB_PASSWORD')) + self.assertEqual('braindb.localhost', fid2.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid2.get('DB_USERNAME', 'dbstuff')) + + self.assertIsNone(fid.get('DB_NAME', no_default=True)) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD', no_default=True)) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff', no_default=True)) + self.assertIsNone(fid.get('DB_USERNAME', 'dbstuff', no_default=True)) + _delete_all_params() + + def test_paramstore_replacer(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('DB2_NAME', 'brain') + fia.create_secret('DB2_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB2_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB2_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertEqual('brain', fid.replace('${__FID__:DB2_NAME}')) + self.assertEqual('myBADpassword', fid.replace('${__FID__:DB2_PASSWORD}')) + self.assertEqual('braindb.mock.cc', fid.replace('${__FID__:dbstuff:DB2_HOST}')) + self.assertEqual('svc-mock', fid.replace('${__FID__:dbstuff:DB2_USERNAME}')) + self.assertEqual('', fid.replace('${__FID__:I_DONT_EXIST}')) + self.assertEqual('I am incorrectly formatted!', fid.replace('I am incorrectly formatted!')) + _delete_all_params() diff --git a/tests/offline/__init__.py b/tests/offline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/offline/test_mock.py b/tests/offline/test_mock.py new file mode 100644 index 0000000..80791cf --- /dev/null +++ b/tests/offline/test_mock.py @@ -0,0 +1,235 @@ +import unittest + +from fidelius.fideliusapi import * + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + +_APP_PROPS = FideliusAppProps(app='mock-app', group='somegroup', env='mock') +_APP_PROPS_TWO = FideliusAppProps(app='other-app', group='somegroup', env='mock') +_APP_PROPS_THREE = FideliusAppProps(app='mock-app', group='somegroup', env='test') + +from fidelius.gateway.mock._inmemcache import _SingletonDict # noqa + + +def _clear_cache(): + _SingletonDict().clear() + + +class TestMock(unittest.TestCase): + def test_mock(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain') + fia.create_secret('DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('myBADpassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + _clear_cache() + + def test_mock_admin_params(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + self.assertIsNone(fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_param('DB_PASSWORD') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_param('DB_PASSWORD', 'myBADpassword') + + key, expression = fia.create_param('DB_PASSWORD', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_param('DB_PASSWORD', 'myWORSEpassword') + + key, expression = fia.update_param('DB_PASSWORD', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD')) + + fia.delete_param('DB_PASSWORD') + + self.assertIsNone(fia.get('DB_PASSWORD')) + _clear_cache() + + def test_mock_admin_secrets(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + self.assertIsNone(fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_secret('DB_PASSWORD') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_secret('DB_PASSWORD', 'myBADpassword') + + key, expression = fia.create_secret('DB_PASSWORD', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_secret('DB_PASSWORD', 'myWORSEpassword') + + key, expression = fia.update_secret('DB_PASSWORD', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD')) + + fia.delete_secret('DB_PASSWORD') + + self.assertIsNone(fia.get('DB_PASSWORD')) + _clear_cache() + + def test_mock_admin_shared_params(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia2 = FideliusFactory.get_admin_class('mock')(_APP_PROPS_TWO) + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_param('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_param('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD', 'dbfolder')) + + fia.delete_shared_param('DB_PASSWORD', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + _clear_cache() + + def test_mock_admin_shared_secrets(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia2 = FideliusFactory.get_admin_class('mock')(_APP_PROPS_TWO) + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD', 'dbfolder')) + + fia.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + _clear_cache() + + def test_mock_defaults(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain', env='default') + fia.create_secret('DB_PASSWORD', 'defaultPassword', env='default') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.localhost', env='default') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock', env='default') + + fia.create_secret('DB_PASSWORD', 'mockPassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + + fid2 = FideliusFactory.get_class('mock')(_APP_PROPS_THREE) + self.assertIsNone(fid2.get('mock-value')) + self.assertEqual('brain', fid2.get('DB_NAME')) + self.assertEqual('defaultPassword', fid2.get('DB_PASSWORD')) + self.assertEqual('braindb.localhost', fid2.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid2.get('DB_USERNAME', 'dbstuff')) + + self.assertIsNone(fid.get('DB_NAME', no_default=True)) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD', no_default=True)) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff', no_default=True)) + self.assertIsNone(fid.get('DB_USERNAME', 'dbstuff', no_default=True)) + _clear_cache() + + def test_mock_replacer(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain') + fia.create_secret('DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertEqual('brain', fid.replace('${__FID__:DB_NAME}')) + self.assertEqual('myBADpassword', fid.replace('${__FID__:DB_PASSWORD}')) + self.assertEqual('braindb.mock.cc', fid.replace('${__FID__:dbstuff:DB_HOST}')) + self.assertEqual('svc-mock', fid.replace('${__FID__:dbstuff:DB_USERNAME}')) + self.assertEqual('', fid.replace('${__FID__:I_DONT_EXIST}')) + self.assertEqual('I am incorrectly formatted!', fid.replace('I am incorrectly formatted!')) + _clear_cache() diff --git a/tests/offline/test_singleton_dict.py b/tests/offline/test_singleton_dict.py new file mode 100644 index 0000000..9042540 --- /dev/null +++ b/tests/offline/test_singleton_dict.py @@ -0,0 +1,80 @@ +import unittest + +from fidelius.gateway.mock._inmemcache import _SingletonDict # noqa + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + + +class TestSingletonDict(unittest.TestCase): + def test_singleton_dict(self): + d = _SingletonDict() + d.clear() # Just in case! + + d2 = _SingletonDict() + + self.assertFalse(bool(d)) + self.assertFalse(bool(d2)) + + self.assertFalse('foo' in d) + self.assertFalse('foo' in d2) + + self.assertIsNone(d.get('foo')) + self.assertIsNone(d2.get('foo')) + + with self.assertRaises(KeyError): + _ = d['foo'] + + with self.assertRaises(KeyError): + _ = d2['foo'] + + d['foo'] = 'bar' + + self.assertTrue(bool(d)) + self.assertTrue(bool(d2)) + + self.assertTrue('foo' in d) + self.assertTrue('foo' in d2) + + self.assertEqual('bar', d.get('foo')) + self.assertEqual('bar', d2.get('foo')) + + self.assertEqual('bar', d['foo']) + self.assertEqual('bar', d2['foo']) + + d3 = _SingletonDict() + + self.assertTrue(bool(d3)) + self.assertTrue('foo' in d3) + self.assertEqual('bar', d3.get('foo')) + self.assertEqual('bar', d3['foo']) + + d3['foo'] = 'not bar' + + self.assertEqual('not bar', d['foo']) + self.assertEqual('not bar', d2['foo']) + self.assertEqual('not bar', d3['foo']) + + del d2['foo'] + + self.assertFalse(bool(d)) + self.assertFalse(bool(d2)) + self.assertFalse(bool(d3)) + + self.assertFalse('foo' in d) + self.assertFalse('foo' in d2) + self.assertFalse('foo' in d3) + + self.assertIsNone(d.get('foo')) + self.assertIsNone(d2.get('foo')) + self.assertIsNone(d3.get('foo')) + + with self.assertRaises(KeyError): + _ = d['foo'] + + with self.assertRaises(KeyError): + _ = d2['foo'] + + with self.assertRaises(KeyError): + _ = d3['foo'] diff --git a/tests/res/mock_params.json b/tests/res/mock_params.json new file mode 100644 index 0000000..adebc65 --- /dev/null +++ b/tests/res/mock_params.json @@ -0,0 +1,15 @@ +{ + "/fidelius/mygroup/default/apps/myapp/DB_PASSWORD": "default-pass", + "/fidelius/mygroup/local/apps/myapp/DB_PASSWORD": "local-pass", + "/fidelius/mygroup/test/apps/myapp/DB_PASSWORD": "test-pass", + "/fidelius/mygroup/prod/apps/myapp/DB_PASSWORD": "prod-pass", + + "/fidelius/mygroup/default/apps/myapp/DB_USERNAME": "svc_mock_user", + "/fidelius/mygroup/prod/apps/myapp/DB_USERNAME": "svc_mock_prod_user", + + "/fidelius/mygroup/default/apps/otherapp/DB_USERNAME": "svc_mock_other_user", + "/fidelius/mygroup/test/apps/otherapp/DB_USERNAME": "svc_mock_other_test_user", + + "/fidelius/mygroup/default/shared/database/DB_HOST": "dev-host.mock.cc", + "/fidelius/mygroup/prod/shared/database/DB_HOST": "real-host.mock.cc" +} \ No newline at end of file