From fd6bc66ceb7e1f197c328bc2e65340be9e7bf7cb Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Tue, 9 Apr 2024 08:09:16 +0000 Subject: [PATCH 1/8] Version 0.7.0-alpha.1 - WIP Checkin - Switching computers --- CHANGELOG.md | 9 +++ fidelius/__init__.py | 2 +- fidelius/gateway/interface.py | 72 +++++++++++++++++++ fidelius/gateway/mock/__init__.py | 2 + fidelius/gateway/mock/_mockadmin.py | 0 fidelius/gateway/mock/_mockrepo.py | 33 +++++++++ fidelius/gateway/paramstore/__init__.py | 2 + .../_paramstoreadmin.py} | 6 +- .../_paramstorerepo.py} | 12 ++-- fidelius/structs/_appcfg.py | 23 ++++++ pyproject.toml | 3 +- requirements.txt | 3 +- tests/__init__.py | 0 tests/test_some_basic_stuff.py | 7 ++ 14 files changed, 162 insertions(+), 12 deletions(-) create mode 100644 fidelius/gateway/interface.py create mode 100644 fidelius/gateway/mock/__init__.py create mode 100644 fidelius/gateway/mock/_mockadmin.py create mode 100644 fidelius/gateway/mock/_mockrepo.py create mode 100644 fidelius/gateway/paramstore/__init__.py rename fidelius/gateway/{paramadmin.py => paramstore/_paramstoreadmin.py} (96%) rename fidelius/gateway/{paramstore.py => paramstore/_paramstorerepo.py} (90%) create mode 100644 fidelius/structs/_appcfg.py create mode 100644 tests/__init__.py create mode 100644 tests/test_some_basic_stuff.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9e88c2..528896e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,15 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] - ? + +### Added + +- An interface for Fidelius gateway repos and admins to fulfil +- A mock implementation of a Fidelius gateway repo and admin that can read + in mock values from JSON files and mess with them in-memory, for use in + unit-tests of other code that uses Fidelius + ## [0.6.0] - 2024-04-05 diff --git a/fidelius/__init__.py b/fidelius/__init__.py index e16bff7..63f8679 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.6.0' +__version__ = '0.7.0-alpha.1' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/gateway/interface.py b/fidelius/gateway/interface.py new file mode 100644 index 0000000..36b6cde --- /dev/null +++ b/fidelius/gateway/interface.py @@ -0,0 +1,72 @@ +__all__ = [ + 'IFideliusRepo', + 'IFideliusAdminRepo', +] +import abc +from typing import * + + +class IFideliusRepo(abc.ABC): + _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' + _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' + + @abc.abstractmethod + def __init__(self, app: str, group: str, env: str, **kwargs): + self._app = app + self._group = group + self._env = env + + @abc.abstractmethod + def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + pass + + def full_path(self, name: str, folder: Optional[str] = None) -> str: + if folder: + return self._SHARED_FULL_NAME.format(group=self._group, folder=folder, env=self._env, name=name) + else: + return self._APP_FULL_NAME.format(group=self._group, app=self._app, env=self._env, name=name) + + def nameless_path(self, folder: Optional[str] = None) -> str: + return self.full_path(name='', folder=folder)[:-1] + + +class IFideliusAdminRepo(IFideliusRepo): + @abc.abstractmethod + def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): + pass + + @abc.abstractmethod + def set_env(self, env: str): + pass + + @abc.abstractmethod + def create_param(self, name: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def update_param(self, name: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def create_secret(self, name: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def update_secret(self, name: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): + pass + + @abc.abstractmethod + def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): + pass diff --git a/fidelius/gateway/mock/__init__.py b/fidelius/gateway/mock/__init__.py new file mode 100644 index 0000000..50bf551 --- /dev/null +++ b/fidelius/gateway/mock/__init__.py @@ -0,0 +1,2 @@ +from ._mockrepo import * +from ._mockadmin import * diff --git a/fidelius/gateway/mock/_mockadmin.py b/fidelius/gateway/mock/_mockadmin.py new file mode 100644 index 0000000..e69de29 diff --git a/fidelius/gateway/mock/_mockrepo.py b/fidelius/gateway/mock/_mockrepo.py new file mode 100644 index 0000000..f765dca --- /dev/null +++ b/fidelius/gateway/mock/_mockrepo.py @@ -0,0 +1,33 @@ +__all__ = [ + 'MockFideliusRepo', +] + +from fidelius.structs import * +from fidelius.gateway.interface import * + +import json + +import logging +log = logging.getLogger(__name__) + + +class MockFideliusRepo(IFideliusRepo): + _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' + _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' + + def __init__(self, app: str, group: str, env: str, pre_seeded_cache: Optional[Union[dict, str]] = None, **kwargs): + super().__init__(app, group, env) + self._cache: Dict[str, str] = {} + self._loaded: bool = False + + self._pre_seeded_cache = pre_seeded_cache + + def _load_all(self, folder: Optional[str] = None): + if isinstance(self._pre_seeded_cache, dict): + self._cache = self._pre_seeded_cache + elif isinstance(self._pre_seeded_cache, str) and self._pre_seeded_cache.lower().endswith('.json'): + with open(self._pre_seeded_cache, 'r') as fin: + self._cache = json.load(fin) + + def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + pass diff --git a/fidelius/gateway/paramstore/__init__.py b/fidelius/gateway/paramstore/__init__.py new file mode 100644 index 0000000..cf4483a --- /dev/null +++ b/fidelius/gateway/paramstore/__init__.py @@ -0,0 +1,2 @@ +from ._paramstorerepo import * +from ._paramstoreadmin import * diff --git a/fidelius/gateway/paramadmin.py b/fidelius/gateway/paramstore/_paramstoreadmin.py similarity index 96% rename from fidelius/gateway/paramadmin.py rename to fidelius/gateway/paramstore/_paramstoreadmin.py index 55c71df..363b505 100644 --- a/fidelius/gateway/paramadmin.py +++ b/fidelius/gateway/paramstore/_paramstoreadmin.py @@ -1,14 +1,16 @@ __all__ = [ 'ParameterStoreAdmin', ] -from .paramstore import * + from fidelius.structs import * +from fidelius.gateway.interface import * +from ._paramstorerepo import * import logging log = logging.getLogger(__name__) -class ParameterStoreAdmin(ParameterStore): +class ParameterStoreAdmin(IFideliusAdminRepo, ParameterStore): def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): super().__init__(app, group, env) self._tags = Tags(application=self._app, owner=owner, tier=env, finance=finance, **extra_tags) diff --git a/fidelius/gateway/paramstore.py b/fidelius/gateway/paramstore/_paramstorerepo.py similarity index 90% rename from fidelius/gateway/paramstore.py rename to fidelius/gateway/paramstore/_paramstorerepo.py index 428feb7..c5cedd8 100644 --- a/fidelius/gateway/paramstore.py +++ b/fidelius/gateway/paramstore/_paramstorerepo.py @@ -3,6 +3,8 @@ ] from fidelius.structs import * +from fidelius.gateway.interface import * + import boto3 import os @@ -11,13 +13,12 @@ log = logging.getLogger(__name__) -class ParameterStore(object): +class ParameterStore(IFideliusRepo): _KEY_ID = os.environ.get('FIDELIUS_AWS_KEY_ARN', '') _DEFAULT_REGION_NAME = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') - _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' - _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' def __init__(self, app: str, group: str, env: str): + super().__init__(app, group, env) if not self._KEY_ID: raise RuntimeError('Fidelius requires the ARN for the KMS key to use to be in the FIDELIUS_AWS_KEY_ARN environment variable') @@ -44,10 +45,7 @@ def _full_path(self, name: str, folder: Optional[str] = None) -> str: return self._APP_FULL_NAME.format(group=self._group, app=self._app, env=self._env, name=name) def _nameless_path(self, folder: Optional[str] = None) -> str: - if folder: - return self._full_path(name='', folder=folder)[:-1] - else: - return self._full_path(name='', folder=folder)[:-1] + return self._full_path(name='', folder=folder)[:-1] def _force_log_secrecy(self): # We don't allow debug or less logging of botocore's HTTP requests cause diff --git a/fidelius/structs/_appcfg.py b/fidelius/structs/_appcfg.py new file mode 100644 index 0000000..dde9d3e --- /dev/null +++ b/fidelius/structs/_appcfg.py @@ -0,0 +1,23 @@ +__all__ = [ + 'AppProps', +] +from ccptools.structs import * + + +@dataclasses.dataclass +class AppProps: + """Application properties, used to get or set specific variations of + parameters/secrets.s + + :ivar app: The name of the application itself (module name or slug) + :ivar group: The app's "group" (think "business domain" or "namespace"). + The purpose of the "group" attribute is to categorize shared + parameters/secrets such that all apps in the same group, can + access all the shared (non-app specific) parameters/secrets of + that group. + :ivar env: Deployment environment (e.g., "prod", "dev", "test"). + """ + app: str + group: str + env: str = 'default' +æ \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index ad14603..ac234f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,8 @@ classifiers = [ "Topic :: Utilities" ] dependencies = [ - "boto3 >=1.20, <2" + "ccptools >=1.1, <2", + "boto3 >=1.20, <2" ] [project.urls] diff --git a/requirements.txt b/requirements.txt index 83e2cbe..1269059 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -boto3 >=1.20, <2 \ No newline at end of file +ccptools >=1.1, <2 +boto3 >=1.20, <2 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_some_basic_stuff.py b/tests/test_some_basic_stuff.py new file mode 100644 index 0000000..374e0e4 --- /dev/null +++ b/tests/test_some_basic_stuff.py @@ -0,0 +1,7 @@ +import unittest + +from fidelius.gateway.mock import * + + +class TestSomeBasicStuff(unittest.TestCase): + pass \ No newline at end of file From 6a0c748a341ac448db31f37f057290100de65124 Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Wed, 10 Apr 2024 08:51:39 +0000 Subject: [PATCH 2/8] Version 0.7.0-dev.2 - WIP Checkin - Switching computers --- fidelius/__init__.py | 2 +- fidelius/fideliusapi/__init__.py | 5 +- fidelius/gateway/_abstract.py | 157 +++++++++ fidelius/gateway/interface.py | 327 ++++++++++++++++-- .../gateway/paramstore/_paramstoreadmin.py | 2 +- .../gateway/paramstore/_paramstorerepo.py | 45 ++- fidelius/structs/__init__.py | 2 +- fidelius/structs/{_appcfg.py => _appprops.py} | 7 +- fidelius/structs/_base.py | 4 +- fidelius/structs/_tags.py | 5 +- fidelius/structs/api.py | 2 + fidelius/utils/__init__.py | 1 - fidelius/utils/replacer.py | 15 - tests/res/mock_params.json | 15 + 14 files changed, 518 insertions(+), 71 deletions(-) create mode 100644 fidelius/gateway/_abstract.py rename fidelius/structs/{_appcfg.py => _appprops.py} (90%) create mode 100644 fidelius/structs/api.py delete mode 100644 fidelius/utils/__init__.py delete mode 100644 fidelius/utils/replacer.py create mode 100644 tests/res/mock_params.json diff --git a/fidelius/__init__.py b/fidelius/__init__.py index 63f8679..4eaf7c9 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.7.0-alpha.1' +__version__ = '0.7.0-dev.2' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/fideliusapi/__init__.py b/fidelius/fideliusapi/__init__.py index dc58555..5dee26c 100644 --- a/fidelius/fideliusapi/__init__.py +++ b/fidelius/fideliusapi/__init__.py @@ -1,3 +1,4 @@ -from fidelius.structs import * -from fidelius.gateway.paramstore import * +from fidelius.structs.api import * from fidelius.utils import * + +from fidelius.gateway.paramstore import * diff --git a/fidelius/gateway/_abstract.py b/fidelius/gateway/_abstract.py new file mode 100644 index 0000000..b31810a --- /dev/null +++ b/fidelius/gateway/_abstract.py @@ -0,0 +1,157 @@ +__all__ = [ + '_BaseFideliusRepo', +] +from .interface import * + +from fidelius.structs import * +import logging +log = logging.getLogger(__file__) + + +class _BaseFideliusRepo(IFideliusRepo, abc.ABC): + """Covers a lot of basic functionality common across most storage back-ends. + """ + + _APP_PATH_FORMAT = '/fidelius/{group}/{env}/apps/{app}/{name}' + _SHARED_PATH_FORMAT = '/fidelius/{group}/{env}/shared/{folder}/{name}' + + _EXPRESSION_APP_FORMAT = '${{__FID__:{name}}}' + _EXPRESSION_SHARED_FORMAT = '${{__FID__:{folder}:{name}}}' + + _EXPRESSION_PATTERN = re.compile(r'\${__FID__:(?:(?P\w+):)?(?P[\w/-]+)}') + + def __init__(self, app_props: FideliusAppProps, **kwargs): + self._app_props = app_props + + # Any kwargs should have been handled by implementation specific stuff, + # unless there's a derpy config so let's warn just in case! + if kwargs: + for k, v in kwargs.items(): + log.warning(f'AbstractFideliusRepo for some unhandled kwargs: {k}={v}') + + @property + def app_props(self) -> FideliusAppProps: + """The current application properties. + """ + return self._app_props + + def make_app_path(self, env: Optional[str] = None) -> str: + """The full path to application specific parameters/secrets. + """ + return self._SHARED_PATH_FORMAT.format(group=self.app_props.group, + env=env or self.app_props.env, + app=self.app_props.app, + name='{name}') + + def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: + """The full path to group shared parameters/secrets. + """ + return self._APP_PATH_FORMAT.format(group=self.app_props.group, + env=env or self.app_props.env, + folder=folder, + name='{name}') + + def get_expression_string(self, name: str, folder: Optional[str] = None) -> str: + """Return a Fidelius expression string (e.g. to use in configuration + files) which the `replace` method can parse and replace with + parameters/secrets fetched from the storage backend. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + :param name: The name of the parameter/secret to create an expression for. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific expression. + :return: The Fidelius expression string. + """ + if folder: + return self._EXPRESSION_SHARED_FORMAT.format(name=name, folder=folder) + else: + return self._EXPRESSION_APP_FORMAT.format(name=name) + + def get_full_path(self, name: str, folder: Optional[str] = None, env: Optional[str] = None) -> str: + """Gets the full path to a parameter/secret. + + Parameter/secret paths can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to use a shared or app path based on whether or + not a folder name was given. + + :param name: The name of the parameter/secret to build a path to. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret path. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The full path to a parameter/secret. + """ + if folder: + return self.make_shared_path(folder, env=env).format(name=name) + else: + return self.make_app_path(env=env).format(name=name) + + def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + """Gets the given parameter/secret from the storage this repo uses. + + Parameters/secrets can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to get a shared or app parameter based on whether or + not a folder name was given. + + Unless disabled by the `no_default` parameter, this method will attempt + to fetch a parameter using the `env=default` if one was not found for + the env in the current app properties. + + :param name: The name of the parameter/secret to get. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret. + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested parameter/secret or None if it was not found. + """ + if folder: + val = self.get_shared_param(name=name, folder=folder) + if val is not None: + return val + + if no_default: + return None + + return self.get_shared_param(name=name, folder=folder, env='default') + else: + val = self.get_app_param(name=name) + if val is not None: + return val + + if no_default: + return None + + return self.get_app_param(name=name, env='default') + + def replace(self, string: str, no_default: bool = False) -> str: + """Take in a containing a Fidelius parameter/secret configuration + expression (e.g. read from a config file), parses it, fetches the + relevant parameter/secret and returns. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + If the given string does not match a Fidilius expression, then it is + returned unchanged. + + :param string: The expression to replace with an actual parameter/secret + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested value + """ + m = self._EXPRESSION_PATTERN.match(string) + if m: + return self.get(m.group('name'), m.group('folder'), no_default=no_default) + return string diff --git a/fidelius/gateway/interface.py b/fidelius/gateway/interface.py index 36b6cde..884b6c7 100644 --- a/fidelius/gateway/interface.py +++ b/fidelius/gateway/interface.py @@ -3,70 +3,351 @@ 'IFideliusAdminRepo', ] import abc -from typing import * +from fidelius.structs import * class IFideliusRepo(abc.ABC): - _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' - _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' + @abc.abstractmethod + def __init__(self, app_props: FideliusAppProps, **kwargs): + """Initialise a new Fidelius Repository. + + :param app_props: The application properties to use. + """ + pass + + @abc.abstractmethod + @property + def app_props(self) -> FideliusAppProps: + """The current application properties. + """ + pass + + @abc.abstractmethod + def make_app_path(self, env: Optional[str] = None) -> str: + """The full path to application specific parameters/secrets. + """ + pass + + @abc.abstractmethod + def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: + """The full path to group shared parameters/secrets. + """ + pass @abc.abstractmethod - def __init__(self, app: str, group: str, env: str, **kwargs): - self._app = app - self._group = group - self._env = env + def get_expression_string(self, name: str, folder: Optional[str] = None) -> str: + """Return a Fidelius expression string (e.g. to use in configuration + files) which the `replace` method can parse and replace with + parameters/secrets fetched from the storage backend. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + + :param name: The name of the parameter/secret to create an expression for. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific expression. + :return: The Fidelius expression string. + """ + pass + + @abc.abstractmethod + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + """Gets an application specific parameter/secret. + + :param name: The name of the parameter/secret to get. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The requested parameter/secret or None if it was not found. + """ + pass @abc.abstractmethod def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: + """Gets the given parameter/secret from the storage this repo uses. + + Parameters/secrets can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to get a shared or app parameter based on whether or + not a folder name was given. + + Unless disabled by the `no_default` parameter, this method will attempt + to fetch a parameter using the `env=default` if one was not found for + the env in the current app properties. + + :param name: The name of the parameter/secret to get. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret. + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested parameter/secret or None if it was not found. + """ + pass + + @abc.abstractmethod + def get_full_path(self, name: str, folder: Optional[str] = None, env: Optional[str] = None) -> str: + """Gets the full path to a parameter/secret. + + Parameter/secret paths can be either application specific (use the + `app_path`) or shared (use the `shared_path`) and this method should + determine whether to use a shared or app path based on whether or + not a folder name was given. + + :param name: The name of the parameter/secret to build a path to. + :param folder: Optional name of shared parameters/secrets to use. This + is optional and only applies to shared + parameters/secrets. Leaving it blank (default) will + return the app specific parameter/secret path. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The full path to a parameter/secret. + """ + pass + + @abc.abstractmethod + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + """Gets a group shared parameter/secret. + + :param name: The name of the parameter/secret to get. + :param folder: The folder where the group shared parameter/secret is + located. + :param env: Optional env value override. Defaults to None, which uses + the env declaration from the current app properties. + :return: The requested parameter/secret or None if it was not found. + """ pass - def full_path(self, name: str, folder: Optional[str] = None) -> str: - if folder: - return self._SHARED_FULL_NAME.format(group=self._group, folder=folder, env=self._env, name=name) - else: - return self._APP_FULL_NAME.format(group=self._group, app=self._app, env=self._env, name=name) + @abc.abstractmethod + def replace(self, string: str, no_default: bool = False) -> str: + """Take in a containing a Fidelius parameter/secret configuration + expression (e.g. read from a config file), parses it, fetches the + relevant parameter/secret and returns. + + The default format of these expressions is: + - `${__FID__:PARAM_NAME}` for app params/secrets + - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER - def nameless_path(self, folder: Optional[str] = None) -> str: - return self.full_path(name='', folder=folder)[:-1] + :param string: The expression to replace with an actual parameter/secret + :param no_default: If True, does not try and get the default value if no + value was found for the current set environment. + :return: The requested value + """ + pass class IFideliusAdminRepo(IFideliusRepo): @abc.abstractmethod - def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + """Initialise a new Fidelius Admin Repository. + + :param app_props: The application properties to use. + :param tags: An optional set of meta-data tags to use when creating new + parameters (if supported by the underlying + parameter/secret storage). + """ + pass + + @abc.abstractmethod + @property + def tags(self) -> FideliusTags: + """The tags to use when creating new parameters. + """ pass @abc.abstractmethod def set_env(self, env: str): + """Sets the current app properties env to something else (for admin purposes) + """ pass @abc.abstractmethod - def create_param(self, name: str, value: str, description: Optional[str] = None): + def create_param(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new unencrypted application parameter. + + :param name: The parameter name. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ pass @abc.abstractmethod - def update_param(self, name: str, value: str, description: Optional[str] = None): + def update_param(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing unencrypted application parameter. + + :param name: The parameter name. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ pass @abc.abstractmethod - def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): + def delete_param(self, name: str, env: Optional[str] = None): + """Deletes an existing unencrypted application parameter. + + :param name: The parameter name. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ pass @abc.abstractmethod - def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): + def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new unencrypted group-shared parameter under the given + shared folder. + + :param name: The parameter name. + :param folder: The shared folder to place the parameter under. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ pass @abc.abstractmethod - def create_secret(self, name: str, value: str, description: Optional[str] = None): + def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing unencrypted group-shared parameter under the + given shared folder. + + :param name: The parameter name. + :param folder: The shared folder to place the parameter under. + :param value: The parameter value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the parameters full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ pass @abc.abstractmethod - def update_secret(self, name: str, value: str, description: Optional[str] = None): + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + """Deletes an existing unencrypted group-shared parameter under the + given shared folder. + + :param name: The parameter name. + :param folder: The shared folder parameter is placed under. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ pass @abc.abstractmethod - def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): + def create_secret(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new encrypted application secret. + + :param name: The secret name. + :param value: The secret value. + :param description: An optional description of the secret (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ pass @abc.abstractmethod - def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): + def update_secret(self, name: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing encrypted application secret. + + :param name: The secret name. + :param value: The secret value. + :param description: An optional description of the secret (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_secret(self, name: str, env: Optional[str] = None): + """Deletes an existing encrypted application secret. + + :param name: The parameter name. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ + pass + + @abc.abstractmethod + def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Creates a new encrypted group-shared secret under the given + shared folder. + + :param name: The secret name. + :param folder: The shared folder to place the secret under. + :param value: The secret value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + """Updates an existing encrypted group-shared secret under the + given shared folder. + + :param name: The secret name. + :param folder: The shared folder to place the secret under. + :param value: The secret value. + :param description: An optional description of the parameter (if + supported by the underlying parameter/secret + storage) + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + :return: A two tuple of strings containing the secret full path and + the default suggested configuration expression to use (for + convenience, which the `replace` method can parse "as-is") + """ + pass + + @abc.abstractmethod + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + """Deletes an existing encrypted group-shared secret under the + given shared folder. + + :param name: The secret name. + :param folder: The shared folder secret is placed under. + :param env: Optional env value override. Defaults to None, which uses + the env from the current app properties. + """ pass diff --git a/fidelius/gateway/paramstore/_paramstoreadmin.py b/fidelius/gateway/paramstore/_paramstoreadmin.py index 363b505..a97f929 100644 --- a/fidelius/gateway/paramstore/_paramstoreadmin.py +++ b/fidelius/gateway/paramstore/_paramstoreadmin.py @@ -13,7 +13,7 @@ class ParameterStoreAdmin(IFideliusAdminRepo, ParameterStore): def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): super().__init__(app, group, env) - self._tags = Tags(application=self._app, owner=owner, tier=env, finance=finance, **extra_tags) + self._tags = FideliusTags(application=self._app, owner=owner, tier=env, finance=finance, **extra_tags) def set_env(self, env: str): self._env = env diff --git a/fidelius/gateway/paramstore/_paramstorerepo.py b/fidelius/gateway/paramstore/_paramstorerepo.py index c5cedd8..b70999f 100644 --- a/fidelius/gateway/paramstore/_paramstorerepo.py +++ b/fidelius/gateway/paramstore/_paramstorerepo.py @@ -1,9 +1,9 @@ __all__ = [ - 'ParameterStore', + 'AwsParamStoreRepo', ] from fidelius.structs import * -from fidelius.gateway.interface import * +from fidelius.gateway._abstract import * import boto3 @@ -13,30 +13,39 @@ log = logging.getLogger(__name__) -class ParameterStore(IFideliusRepo): - _KEY_ID = os.environ.get('FIDELIUS_AWS_KEY_ARN', '') - _DEFAULT_REGION_NAME = os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') +class AwsParamStoreRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, + aws_access_key_id: str = None, + aws_secret_access_key: str = None, + aws_key_arn: str = None, + aws_region_name: str = None, + **kwargs): + super().__init__(app_props, **kwargs) - def __init__(self, app: str, group: str, env: str): - super().__init__(app, group, env) - if not self._KEY_ID: - raise RuntimeError('Fidelius requires the ARN for the KMS key to use to be in the FIDELIUS_AWS_KEY_ARN environment variable') + self._aws_key_arn = aws_key_arn or os.environ.get('FIDELIUS_AWS_KEY_ARN', '') + if not self._aws_key_arn: + raise EnvironmentError('Fidelius AwsParamStoreRepo requires the ARN for the KMS key argument when initialising or in the FIDELIUS_AWS_KEY_ARN environment variable') + + self._region_name = aws_region_name or os.environ.get('FIDELIUS_AWS_REGION_NAME', None) or os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') + + # TODO(thordurm@ccpgames.com>) 2024-04-09: Check for aws_access_key_id and/or aws_secret_access_key - self._app = app - self._group = group - self._env = env self._force_log_secrecy() self._ssm = boto3.client('ssm', - region_name=os.environ.get('FIDELIUS_AWS_REGION_NAME', self._DEFAULT_REGION_NAME), - aws_access_key_id=os.environ.get('FIDELIUS_AWS_ACCESS_KEY_ID', None), - aws_secret_access_key=os.environ.get('FIDELIUS_AWS_SECRET_ACCESS_KEY', None)) + region_name=self._region_name, + aws_access_key_id=aws_access_key_id or os.environ.get('FIDELIUS_AWS_ACCESS_KEY_ID', None) or os.environ.get('AWS_ACCESS_KEY_ID', None), + aws_secret_access_key=aws_secret_access_key or os.environ.get('FIDELIUS_AWS_SECRET_ACCESS_KEY', None) or os.environ.get('AWS_SECRET_ACCESS_KEY', None)) self._cache: Dict[str, str] = {} self._loaded: bool = False self._loaded_folders: Set[str] = set() - self._default_store: Optional[ParameterStore] = None - if self._env != 'default': - self._default_store = ParameterStore(self._app, self._group, 'default') + self._default_store: Optional[AwsParamStoreRepo] = None + if self.app_props.env != 'default': + self._default_store = AwsParamStoreRepo(app_props=FideliusAppProps(app=app_props.app, group=app_props.group, env='default'), + aws_access_key_id=aws_access_key_id, + aws_secret_access_key=aws_secret_access_key, + aws_key_arn=aws_key_arn, + aws_region_name=aws_region_name) def _full_path(self, name: str, folder: Optional[str] = None) -> str: if folder: diff --git a/fidelius/structs/__init__.py b/fidelius/structs/__init__.py index 0b83f41..8c7ec00 100644 --- a/fidelius/structs/__init__.py +++ b/fidelius/structs/__init__.py @@ -1,2 +1,2 @@ from ._base import * -from ._tags import * +from .api import * diff --git a/fidelius/structs/_appcfg.py b/fidelius/structs/_appprops.py similarity index 90% rename from fidelius/structs/_appcfg.py rename to fidelius/structs/_appprops.py index dde9d3e..9dcab8f 100644 --- a/fidelius/structs/_appcfg.py +++ b/fidelius/structs/_appprops.py @@ -1,13 +1,13 @@ __all__ = [ - 'AppProps', + 'FideliusAppProps', ] from ccptools.structs import * @dataclasses.dataclass -class AppProps: +class FideliusAppProps: """Application properties, used to get or set specific variations of - parameters/secrets.s + parameters/secrets. :ivar app: The name of the application itself (module name or slug) :ivar group: The app's "group" (think "business domain" or "namespace"). @@ -20,4 +20,3 @@ class AppProps: app: str group: str env: str = 'default' -æ \ No newline at end of file diff --git a/fidelius/structs/_base.py b/fidelius/structs/_base.py index 36f969b..b0a7460 100644 --- a/fidelius/structs/_base.py +++ b/fidelius/structs/_base.py @@ -1,3 +1 @@ -from typing import * - -import datetime +from ccptools.structs import * diff --git a/fidelius/structs/_tags.py b/fidelius/structs/_tags.py index fff0478..2e5b84b 100644 --- a/fidelius/structs/_tags.py +++ b/fidelius/structs/_tags.py @@ -1,10 +1,10 @@ __all__ = [ - 'Tags', + 'FideliusTags', ] from ._base import * -class Tags: +class FideliusTags: __slots__ = ('application', 'owner', 'tier', 'finance', '_other') def __init__(self, application: str, owner: str, tier: str = 'default', finance: str = 'COST', **kwargs): @@ -44,4 +44,5 @@ def to_dict(self) -> Dict[str, str]: return d def to_aws_format(self) -> List[Dict[str, str]]: + # TODO(thordurm@ccpgames.com>) 2024-04-09: Move to param-store implementation! return [{'Key': k, 'Value': v} for k, v in self.to_dict().items()] diff --git a/fidelius/structs/api.py b/fidelius/structs/api.py new file mode 100644 index 0000000..8461ca5 --- /dev/null +++ b/fidelius/structs/api.py @@ -0,0 +1,2 @@ +from ._tags import * +from ._appprops import * diff --git a/fidelius/utils/__init__.py b/fidelius/utils/__init__.py deleted file mode 100644 index acc496b..0000000 --- a/fidelius/utils/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .replacer import * diff --git a/fidelius/utils/replacer.py b/fidelius/utils/replacer.py deleted file mode 100644 index c9267bc..0000000 --- a/fidelius/utils/replacer.py +++ /dev/null @@ -1,15 +0,0 @@ -__all__ = [ - 'fidelius_replace', -] -import re -from fidelius.gateway.paramstore import ParameterStore - -_VAR_PATTERN = re.compile(r'\${__FID__:(?:(?P\w+):)?(?P[\w/-]+)}') - - -def fidelius_replace(value: str, pmstore: ParameterStore) -> str: - m = _VAR_PATTERN.match(value) - if m: - return pmstore.get(m.group('name'), m.group('shared_group')) - return value - diff --git a/tests/res/mock_params.json b/tests/res/mock_params.json new file mode 100644 index 0000000..adebc65 --- /dev/null +++ b/tests/res/mock_params.json @@ -0,0 +1,15 @@ +{ + "/fidelius/mygroup/default/apps/myapp/DB_PASSWORD": "default-pass", + "/fidelius/mygroup/local/apps/myapp/DB_PASSWORD": "local-pass", + "/fidelius/mygroup/test/apps/myapp/DB_PASSWORD": "test-pass", + "/fidelius/mygroup/prod/apps/myapp/DB_PASSWORD": "prod-pass", + + "/fidelius/mygroup/default/apps/myapp/DB_USERNAME": "svc_mock_user", + "/fidelius/mygroup/prod/apps/myapp/DB_USERNAME": "svc_mock_prod_user", + + "/fidelius/mygroup/default/apps/otherapp/DB_USERNAME": "svc_mock_other_user", + "/fidelius/mygroup/test/apps/otherapp/DB_USERNAME": "svc_mock_other_test_user", + + "/fidelius/mygroup/default/shared/database/DB_HOST": "dev-host.mock.cc", + "/fidelius/mygroup/prod/shared/database/DB_HOST": "real-host.mock.cc" +} \ No newline at end of file From 975fe4d31fdffe28eebac6fad20983513be50dce Mon Sep 17 00:00:00 2001 From: Thordur Matthiasson Date: Thu, 11 Apr 2024 07:52:43 +0000 Subject: [PATCH 3/8] Version 0.7.0-dev.3 - WIP Commit - Moving computers again --- fidelius/__init__.py | 2 +- fidelius/fideliusapi/__init__.py | 5 +- fidelius/gateway/__init__.py | 20 ++++++ fidelius/gateway/_abstract.py | 22 +++++- fidelius/gateway/interface.py | 6 +- fidelius/gateway/mock/_mockadmin.py | 100 ++++++++++++++++++++++++++++ fidelius/gateway/mock/_mockrepo.py | 34 +++++----- fidelius/gateway/mock/_std.py | 4 ++ fidelius/gateway/paramstore/_std.py | 1 + fidelius/structs/_appprops.py | 2 +- fidelius/structs/_errors.py | 22 ++++++ fidelius/structs/_tags.py | 4 ++ fidelius/structs/api.py | 1 + tests/test_mock.py | 26 ++++++++ 14 files changed, 220 insertions(+), 29 deletions(-) create mode 100644 fidelius/gateway/mock/_std.py create mode 100644 fidelius/gateway/paramstore/_std.py create mode 100644 fidelius/structs/_errors.py create mode 100644 tests/test_mock.py diff --git a/fidelius/__init__.py b/fidelius/__init__.py index 4eaf7c9..18d457f 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.7.0-dev.2' +__version__ = '0.7.0-dev.3' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/fideliusapi/__init__.py b/fidelius/fideliusapi/__init__.py index 5dee26c..569645b 100644 --- a/fidelius/fideliusapi/__init__.py +++ b/fidelius/fideliusapi/__init__.py @@ -1,4 +1,3 @@ from fidelius.structs.api import * -from fidelius.utils import * - -from fidelius.gateway.paramstore import * +from fidelius.gateway.interface import * +from fidelius.gateway import FideliusFactory diff --git a/fidelius/gateway/__init__.py b/fidelius/gateway/__init__.py index e69de29..529fb93 100644 --- a/fidelius/gateway/__init__.py +++ b/fidelius/gateway/__init__.py @@ -0,0 +1,20 @@ +__all__ = [ + 'FideliusFactory', +] + +from fidelius.structs import * +from .interface import * +from ccptools.tpu import strimp + +import logging +log = logging.getLogger(__name__) + + +class FideliusFactory: + @staticmethod + def get_class(impl: str = 'paramstore') -> Type[IFideliusRepo]: + return strimp.get_class(f'fidelius.gateway.{impl}._std.FideliusRepo', logger=log, reraise=True) + + @staticmethod + def get_admin_class(impl: str = 'paramstore') -> Type[IFideliusAdminRepo]: + return strimp.get_class(f'fidelius.gateway.{impl}._std.FideliusAdmin', logger=log, reraise=True) diff --git a/fidelius/gateway/_abstract.py b/fidelius/gateway/_abstract.py index b31810a..858e86d 100644 --- a/fidelius/gateway/_abstract.py +++ b/fidelius/gateway/_abstract.py @@ -1,5 +1,6 @@ __all__ = [ '_BaseFideliusRepo', + '_BaseFideliusAdminRepo', ] from .interface import * @@ -21,6 +22,7 @@ class _BaseFideliusRepo(IFideliusRepo, abc.ABC): _EXPRESSION_PATTERN = re.compile(r'\${__FID__:(?:(?P\w+):)?(?P[\w/-]+)}') def __init__(self, app_props: FideliusAppProps, **kwargs): + log.debug('_BaseFideliusRepo.__init__') self._app_props = app_props # Any kwargs should have been handled by implementation specific stuff, @@ -38,7 +40,7 @@ def app_props(self) -> FideliusAppProps: def make_app_path(self, env: Optional[str] = None) -> str: """The full path to application specific parameters/secrets. """ - return self._SHARED_PATH_FORMAT.format(group=self.app_props.group, + return self._APP_PATH_FORMAT.format(group=self.app_props.group, env=env or self.app_props.env, app=self.app_props.app, name='{name}') @@ -46,7 +48,7 @@ def make_app_path(self, env: Optional[str] = None) -> str: def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: """The full path to group shared parameters/secrets. """ - return self._APP_PATH_FORMAT.format(group=self.app_props.group, + return self._SHARED_PATH_FORMAT.format(group=self.app_props.group, env=env or self.app_props.env, folder=folder, name='{name}') @@ -155,3 +157,19 @@ def replace(self, string: str, no_default: bool = False) -> str: if m: return self.get(m.group('name'), m.group('folder'), no_default=no_default) return string + + +class _BaseFideliusAdminRepo(_BaseFideliusRepo, IFideliusAdminRepo, abc.ABC): + """Covers a lot of admin basic functionality common across most storage back-ends. + """ + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + log.debug('_BaseFideliusAdminRepo.__init__') + super().__init__(app_props, **kwargs) + self._tags = tags + + @property + def tags(self) -> Optional[FideliusTags]: + return self._tags + + def set_env(self, env: str): + self.app_props.env = env diff --git a/fidelius/gateway/interface.py b/fidelius/gateway/interface.py index 884b6c7..fbfda2b 100644 --- a/fidelius/gateway/interface.py +++ b/fidelius/gateway/interface.py @@ -15,8 +15,8 @@ def __init__(self, app_props: FideliusAppProps, **kwargs): """ pass - @abc.abstractmethod @property + @abc.abstractmethod def app_props(self) -> FideliusAppProps: """The current application properties. """ @@ -151,9 +151,9 @@ def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = N """ pass - @abc.abstractmethod @property - def tags(self) -> FideliusTags: + @abc.abstractmethod + def tags(self) -> Optional[FideliusTags]: """The tags to use when creating new parameters. """ pass diff --git a/fidelius/gateway/mock/_mockadmin.py b/fidelius/gateway/mock/_mockadmin.py index e69de29..dd4515b 100644 --- a/fidelius/gateway/mock/_mockadmin.py +++ b/fidelius/gateway/mock/_mockadmin.py @@ -0,0 +1,100 @@ +__all__ = [ + 'MockFideliusAdmin', +] + +from fidelius.gateway._abstract import * +from fidelius.structs import * + +import logging +log = logging.getLogger(__name__) + + +class MockFideliusAdmin(_BaseFideliusAdminRepo): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + """This mock version of the Fidelius Admin stores created and updated + params in memory only. + + Note that it does NOT extend the functionality of its non-Admin sibling, + the MockFideliusRepo and thus does not return a base64 encoded version + of every requested param/secret key name, but instead only uses its own + internal in-memory cache, and thus, `get` will not return anything that + hasn't been created first during that particular runtime. + + This is mainly intended for unit testing other packages and apps that + use Fidelius. + """ + log.debug('MockFideliusAdmin.__init__') + super().__init__(app_props, tags, **kwargs) + self._cache = {} + + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + return self._cache.get(self.get_full_path(name, env=env), None) + + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + return self._cache.get(self.get_full_path(name, folder, env=env), None) + + def _create(self, name: str, value: str, env: Optional[str] = None, folder: Optional[str] = None) -> (str, str): + key = self.get_full_path(name, folder=folder, env=env) + if key in self._cache: + raise FideliusParameterAlreadyExists(f'parameter already exists: {key}') + self._cache[key] = value + return key, self.get_expression_string(name, folder=folder) + + def _update(self, name: str, value: str, env: Optional[str] = None, folder: Optional[str] = None) -> (str, str): + key = self.get_full_path(name, folder=folder, env=env) + if key not in self._cache: + raise FideliusParameterNotFound(f'parameter not found: {key}') + self._cache[key] = value + return key, self.get_expression_string(name, folder=folder) + + def _delete(self, name: str, env: Optional[str] = None, folder: Optional[str] = None): + key = self.get_full_path(name, folder=folder, env=env) + if key not in self._cache: + raise FideliusParameterNotFound(f'parameter not found: {key}') + del self._cache[key] + + def create_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env) + + def update_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env) + + def delete_param(self, name: str, env: Optional[str] = None): + self._delete(name, env=env) + + def create_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, + env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env, folder=folder) + + def update_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, + env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env, folder=folder) + + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + self._delete(name, env=env, folder=folder) + + def create_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env) + + def update_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env) + + def delete_secret(self, name: str, env: Optional[str] = None): + self._delete(name, env=env) + + def create_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._create(name, value=value, env=env, folder=folder) + + def update_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + return self._update(name, value=value, env=env, folder=folder) + + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + self._delete(name, env=env, folder=folder) diff --git a/fidelius/gateway/mock/_mockrepo.py b/fidelius/gateway/mock/_mockrepo.py index f765dca..eda39a4 100644 --- a/fidelius/gateway/mock/_mockrepo.py +++ b/fidelius/gateway/mock/_mockrepo.py @@ -3,31 +3,27 @@ ] from fidelius.structs import * -from fidelius.gateway.interface import * +from fidelius.gateway._abstract import * -import json +import base64 import logging log = logging.getLogger(__name__) -class MockFideliusRepo(IFideliusRepo): - _APP_FULL_NAME = '/fidelius/{group}/{env}/apps/{app}/{name}' - _SHARED_FULL_NAME = '/fidelius/{group}/{env}/shared/{folder}/{name}' +class MockFideliusRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, **kwargs): + """This mock variation of the FideliusRepo simply returns a base64 + encoded version of the full path of the requested parameter/secret. - def __init__(self, app: str, group: str, env: str, pre_seeded_cache: Optional[Union[dict, str]] = None, **kwargs): - super().__init__(app, group, env) - self._cache: Dict[str, str] = {} - self._loaded: bool = False + This is mainly intended for unit testing other packages and apps that + use Fidelius. + """ + log.debug('MockFideliusRepo.__init__') + super().__init__(app_props, **kwargs) - self._pre_seeded_cache = pre_seeded_cache + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + return base64.encodebytes(self.get_full_path(name, env=env).encode('utf-8')).decode('utf-8').strip() - def _load_all(self, folder: Optional[str] = None): - if isinstance(self._pre_seeded_cache, dict): - self._cache = self._pre_seeded_cache - elif isinstance(self._pre_seeded_cache, str) and self._pre_seeded_cache.lower().endswith('.json'): - with open(self._pre_seeded_cache, 'r') as fin: - self._cache = json.load(fin) - - def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: - pass + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + return base64.encodebytes(self.get_full_path(name, folder=folder, env=env).encode('utf-8')).decode('utf-8').strip() diff --git a/fidelius/gateway/mock/_std.py b/fidelius/gateway/mock/_std.py new file mode 100644 index 0000000..3dc55cd --- /dev/null +++ b/fidelius/gateway/mock/_std.py @@ -0,0 +1,4 @@ +"""Here we import this modules implementation classes with generic names for the +Factory to use""" +from ._mockrepo import MockFideliusRepo as FideliusRepo +from ._mockadmin import MockFideliusAdmin as FideliusAdmin diff --git a/fidelius/gateway/paramstore/_std.py b/fidelius/gateway/paramstore/_std.py new file mode 100644 index 0000000..d1a9e0c --- /dev/null +++ b/fidelius/gateway/paramstore/_std.py @@ -0,0 +1 @@ +from ._paramstorerepo import \ No newline at end of file diff --git a/fidelius/structs/_appprops.py b/fidelius/structs/_appprops.py index 9dcab8f..d8fb0af 100644 --- a/fidelius/structs/_appprops.py +++ b/fidelius/structs/_appprops.py @@ -1,7 +1,7 @@ __all__ = [ 'FideliusAppProps', ] -from ccptools.structs import * +import dataclasses @dataclasses.dataclass diff --git a/fidelius/structs/_errors.py b/fidelius/structs/_errors.py new file mode 100644 index 0000000..5fc9218 --- /dev/null +++ b/fidelius/structs/_errors.py @@ -0,0 +1,22 @@ +__all__ = [ + 'FideliusError', + 'FideliusAdminError', + 'FideliusParameterNotFound', + 'FideliusParameterAlreadyExists', +] + + +class FideliusError(Exception): + pass + + +class FideliusAdminError(FideliusError): + pass + + +class FideliusParameterNotFound(FideliusAdminError, KeyError): + pass + + +class FideliusParameterAlreadyExists(FideliusAdminError, ValueError): + pass diff --git a/fidelius/structs/_tags.py b/fidelius/structs/_tags.py index 2e5b84b..bd405cb 100644 --- a/fidelius/structs/_tags.py +++ b/fidelius/structs/_tags.py @@ -32,6 +32,10 @@ def __setattr__(self, name: str, value: Optional[str]): def __delattr__(self, name: str): self.__setattr__(name, None) + def __repr__(self) -> str: + tags = ', '.join([f"{k}='{v}'" for k, v in self.to_dict().items()]) + return f'{self.__class__.__name__}({tags})' + def to_dict(self) -> Dict[str, str]: d = { 'application': self.application, diff --git a/fidelius/structs/api.py b/fidelius/structs/api.py index 8461ca5..63287ce 100644 --- a/fidelius/structs/api.py +++ b/fidelius/structs/api.py @@ -1,2 +1,3 @@ from ._tags import * from ._appprops import * +from ._errors import * diff --git a/tests/test_mock.py b/tests/test_mock.py new file mode 100644 index 0000000..83e2153 --- /dev/null +++ b/tests/test_mock.py @@ -0,0 +1,26 @@ +import unittest + +from fidelius.fideliusapi import * + +_app_props = FideliusAppProps(app='mock-app', group='somegroup', env='mock') + + +class TestMock(unittest.TestCase): + def test_mock(self): + fid = FideliusFactory.get_class('mock')(_app_props) + self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL2FwcHMvbW9jay1hcHAvbW9jay12YWx1ZQ==', fid.get('mock-value')) + self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL2FwcHMvbW9jay1hcHAvREJfUEFTU1dPUkQ=', fid.get('DB_PASSWORD')) + self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL3NoYXJlZC9zaGFyZWRwb3N0Z3Jlcy9EQl9IT1NU', fid.get('DB_HOST', 'sharedpostgres')) + + def test_mock_admin(self): + fia = FideliusFactory.get_admin_class('mock')(_app_props) + self.assertIsNone(fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_param('DB_PASSWORD') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_param('DB_PASSWORD', 'myBADpassword') + + key, expression = fia.create_param('DB_PASSWORD', 'myBADpassword') + From 65011e09bd98eba707266fe0ef09e1eba5da8baf Mon Sep 17 00:00:00 2001 From: Thordur Matthiasson Date: Thu, 11 Apr 2024 07:53:03 +0000 Subject: [PATCH 4/8] Version 0.7.0-dev.4 - WIP Commit - Moving computers again --- fidelius/gateway/file/__init__.py | 2 ++ fidelius/gateway/file/_fileadmin.py | 0 fidelius/gateway/file/_filerepo.py | 36 +++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+) create mode 100644 fidelius/gateway/file/__init__.py create mode 100644 fidelius/gateway/file/_fileadmin.py create mode 100644 fidelius/gateway/file/_filerepo.py diff --git a/fidelius/gateway/file/__init__.py b/fidelius/gateway/file/__init__.py new file mode 100644 index 0000000..a40e345 --- /dev/null +++ b/fidelius/gateway/file/__init__.py @@ -0,0 +1,2 @@ +from ._filerepo import * +from ._fileadmin import * diff --git a/fidelius/gateway/file/_fileadmin.py b/fidelius/gateway/file/_fileadmin.py new file mode 100644 index 0000000..e69de29 diff --git a/fidelius/gateway/file/_filerepo.py b/fidelius/gateway/file/_filerepo.py new file mode 100644 index 0000000..ad7e28b --- /dev/null +++ b/fidelius/gateway/file/_filerepo.py @@ -0,0 +1,36 @@ +__all__ = [ + 'MockFideliusRepo', +] + +from fidelius.structs import * +from fidelius.gateway._abstract import * + +import json + +import logging +log = logging.getLogger(__name__) + + +class MockFideliusRepo(_BaseFideliusRepo): + def __init__(self, app_props: FideliusAppProps, pre_seeded_cache: Optional[Union[dict, str]] = None, **kwargs): + super().__init__(app_props, **kwargs) + self._cache: Dict[str, str] = {} + self._loaded: bool = False + + self._pre_seeded_cache = pre_seeded_cache + + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + self._load_all() + return self._cache.get(self.get_full_path(name, env=env), None) + + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + self._load_all() + return self._cache.get(self.get_full_path(name, folder=folder, env=env), None) + + def _load_all(self): + if not self._loaded: + if isinstance(self._pre_seeded_cache, dict): + self._cache = self._pre_seeded_cache + elif isinstance(self._pre_seeded_cache, str) and self._pre_seeded_cache.lower().endswith('.json'): + with open(self._pre_seeded_cache, 'r') as fin: + self._cache = json.load(fin) From c6c17030ecd8483b4b50792eb8e92b910feb9903 Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Thu, 11 Apr 2024 15:16:52 +0000 Subject: [PATCH 5/8] Committing new Github Actions :) --- .github/workflows/publish-to-pypi.yml | 3 -- .github/workflows/unit-test.yml | 65 +++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/unit-test.yml diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 89a6f09..2fdfc61 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -18,9 +18,6 @@ jobs: python --version python -m pip install --upgrade pip pip install --upgrade setuptools wheel twine - # - name: Run Unit Tests - # run: | - # python -m unittest discover -v -f ./tests - name: Build and Package run: | python setup.py sdist bdist_wheel diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000..0eea8f9 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,65 @@ +name: Run Unit Tests + +on: [push, pull_request] + +jobs: + unit-tests: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade requirements.txt + - name: Run tests + run: | + python -m unittest discover -v -f ./tests/offline + + integration-tests: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + + services: + localstack: + image: localstack/localstack + ports: + - 4566:4566 + env: + SERVICES: ssm + DEFAULT_REGION: eu-west-1 + + env: + FIDELIUS_AWS_KEY_ARN: arn:aws:kms:eu-west-1:123456789012:alias/fidelius-key + FIDELIUS_AWS_REGION_NAME: eu-west-1 + FIDELIUS_AWS_ENDPOINT_URL: http://localhost:4566 + FIDELIUS_AWS_ACCESS_KEY_ID: somemadeupstuff + FIDELIUS_AWS_SECRET_ACCESS_KEY: notarealkey + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install Dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade requirements.txt + - name: Wait for LocalStack + run: | + until curl --fail http://localhost:4566/health; do sleep 5; done + - name: Run tests + run: | + python -m unittest discover -v -f ./tests/localstack From 1bd8ad6cae48d0aa4125d7198cb3bd7cc08fbe1f Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Thu, 11 Apr 2024 15:19:50 +0000 Subject: [PATCH 6/8] Committing new Github Actions again! --- .github/workflows/unit-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 0eea8f9..e9bf87e 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -19,7 +19,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install --upgrade requirements.txt + pip install --upgrade -r requirements.txt - name: Run tests run: | python -m unittest discover -v -f ./tests/offline @@ -56,7 +56,7 @@ jobs: - name: Install Dependencies run: | python -m pip install --upgrade pip - pip install --upgrade requirements.txt + pip install --upgrade -r requirements.txt - name: Wait for LocalStack run: | until curl --fail http://localhost:4566/health; do sleep 5; done From 4cf941d824294bd6620c91682129ac9424af708f Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Thu, 11 Apr 2024 15:23:03 +0000 Subject: [PATCH 7/8] Committing new Github Actions once again --- .github/workflows/unit-test.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index e9bf87e..a8ca1e9 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -57,9 +57,6 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade -r requirements.txt - - name: Wait for LocalStack - run: | - until curl --fail http://localhost:4566/health; do sleep 5; done - name: Run tests run: | python -m unittest discover -v -f ./tests/localstack From 96ba36d31b44248926921668c6e352f9dd4fcc84 Mon Sep 17 00:00:00 2001 From: ccp_zeulix Date: Thu, 11 Apr 2024 15:38:02 +0000 Subject: [PATCH 8/8] Version 1.0.0-beta.1 - Almost Stable ### Added - An interface for Fidelius gateway repos and admins to fulfil - A mock implementation of a Fidelius gateway repo and admin that use a simple singleton dict to store (and share) data during runtime - Unittests for the mock implementation - Unittests for the Parameter Store implementation using LocalStack - A Factory class to get different implementation classes - Methods to delete parameters - Config params for the `AwsParamStoreRepo` to use a custom AWS endpoint in order to hook up to stuff like LocalStack for testing and such ### Changed - The API a little bit so we're no longer backwards compatible (hence the major version bump to 1.0.0) - All config params can now be explicitly given to the `AwsParamStoreRepo` in addition to being picked up from environment variables if not supplied --- CHANGELOG.md | 20 +- README.md | 5 +- fidelius/__init__.py | 2 +- fidelius/gateway/_abstract.py | 24 +- fidelius/gateway/file/__init__.py | 2 - fidelius/gateway/file/_filerepo.py | 36 --- fidelius/gateway/interface.py | 10 +- fidelius/gateway/mock/_inmemcache.py | 19 ++ fidelius/gateway/mock/_mockadmin.py | 13 +- fidelius/gateway/mock/_mockrepo.py | 6 +- .../gateway/paramstore/_paramstoreadmin.py | 180 ++++++++---- .../gateway/paramstore/_paramstorerepo.py | 101 ++++--- fidelius/gateway/paramstore/_std.py | 3 +- fidelius/structs/_tags.py | 4 - .../localstack/__init__.py | 0 tests/localstack/test_paramstore.py | 260 ++++++++++++++++++ tests/offline/__init__.py | 0 tests/offline/test_mock.py | 235 ++++++++++++++++ tests/offline/test_singleton_dict.py | 80 ++++++ tests/test_mock.py | 26 -- tests/test_some_basic_stuff.py | 7 - 21 files changed, 829 insertions(+), 204 deletions(-) delete mode 100644 fidelius/gateway/file/__init__.py delete mode 100644 fidelius/gateway/file/_filerepo.py create mode 100644 fidelius/gateway/mock/_inmemcache.py rename fidelius/gateway/file/_fileadmin.py => tests/localstack/__init__.py (100%) create mode 100644 tests/localstack/test_paramstore.py create mode 100644 tests/offline/__init__.py create mode 100644 tests/offline/test_mock.py create mode 100644 tests/offline/test_singleton_dict.py delete mode 100644 tests/test_mock.py delete mode 100644 tests/test_some_basic_stuff.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 528896e..7472871 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,14 +5,26 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] - ? +## [1.0.0-beta.1] - 2024-04-11 ### Added - An interface for Fidelius gateway repos and admins to fulfil -- A mock implementation of a Fidelius gateway repo and admin that can read - in mock values from JSON files and mess with them in-memory, for use in - unit-tests of other code that uses Fidelius +- A mock implementation of a Fidelius gateway repo and admin that use a + simple singleton dict to store (and share) data during runtime +- Unittests for the mock implementation +- Unittests for the Parameter Store implementation using LocalStack +- A Factory class to get different implementation classes +- Methods to delete parameters +- Config params for the `AwsParamStoreRepo` to use a custom AWS endpoint in + order to hook up to stuff like LocalStack for testing and such + +### Changed + +- The API a little bit so we're no longer backwards compatible (hence the + major version bump to 1.0.0) +- All config params can now be explicitly given to the `AwsParamStoreRepo` + in addition to being picked up from environment variables if not supplied ## [0.6.0] - 2024-04-05 diff --git a/README.md b/README.md index d85887c..ca82bc1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,10 @@ package in mind but should work for other cases as well. **IMPORTANT:** This has been migrated more-or-less _"as-is"_ from CCP Tool's internal repo and hasn't yet been given the love it needs to be properly open-sourced and user friendly for other people _(unless you read though the -code and find it perfectly fits your use case)_. +code and find it perfectly fits your use case)_. + +**ALSO IMPORTANT:** This README hasn't been updated to reflect changes in +version 1.0.0 yet. Sowwie! :-/ ## What should be stored with Fidelius diff --git a/fidelius/__init__.py b/fidelius/__init__.py index 18d457f..420f4a1 100644 --- a/fidelius/__init__.py +++ b/fidelius/__init__.py @@ -1,4 +1,4 @@ -__version__ = '0.7.0-dev.3' +__version__ = '1.0.0-beta.1' __author__ = 'Thordur Matthiasson ' __license__ = 'MIT License' diff --git a/fidelius/gateway/_abstract.py b/fidelius/gateway/_abstract.py index 858e86d..4a62576 100644 --- a/fidelius/gateway/_abstract.py +++ b/fidelius/gateway/_abstract.py @@ -41,9 +41,9 @@ def make_app_path(self, env: Optional[str] = None) -> str: """The full path to application specific parameters/secrets. """ return self._APP_PATH_FORMAT.format(group=self.app_props.group, - env=env or self.app_props.env, - app=self.app_props.app, - name='{name}') + env=env or self.app_props.env, + app=self.app_props.app, + name='{name}') def make_shared_path(self, folder: str, env: Optional[str] = None) -> str: """The full path to group shared parameters/secrets. @@ -117,23 +117,30 @@ def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) value was found for the current set environment. :return: The requested parameter/secret or None if it was not found. """ + log.debug('_BaseFideliusRepo.get(name=%s, folder=%s, no_default=%s))', name, folder, no_default) if folder: val = self.get_shared_param(name=name, folder=folder) + log.debug('_BaseFideliusRepo.get->get_shared_param val=%s', val) if val is not None: return val if no_default: + log.debug('_BaseFideliusRepo.get->(shared) no_default STOP!') return None + log.debug('_BaseFideliusRepo.get->(shared) Lets try the default!!!') return self.get_shared_param(name=name, folder=folder, env='default') else: val = self.get_app_param(name=name) + log.debug('_BaseFideliusRepo.get->get_app_param val=%s', val) if val is not None: return val if no_default: + log.debug('_BaseFideliusRepo.get->(app) no_default STOP!') return None + log.debug('_BaseFideliusRepo.get->(app) Lets try the default!!!') return self.get_app_param(name=name, env='default') def replace(self, string: str, no_default: bool = False) -> str: @@ -145,17 +152,18 @@ def replace(self, string: str, no_default: bool = False) -> str: - `${__FID__:PARAM_NAME}` for app params/secrets - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER - If the given string does not match a Fidilius expression, then it is - returned unchanged. + An empty string is returned if the parameter was not found and if the + string does not match the expression format, it will be returned + unchanged. :param string: The expression to replace with an actual parameter/secret :param no_default: If True, does not try and get the default value if no value was found for the current set environment. - :return: The requested value + :return: The requested value, an empty string or the original string """ m = self._EXPRESSION_PATTERN.match(string) if m: - return self.get(m.group('name'), m.group('folder'), no_default=no_default) + return self.get(m.group('name'), m.group('folder'), no_default=no_default) or '' return string @@ -163,7 +171,7 @@ class _BaseFideliusAdminRepo(_BaseFideliusRepo, IFideliusAdminRepo, abc.ABC): """Covers a lot of admin basic functionality common across most storage back-ends. """ def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): - log.debug('_BaseFideliusAdminRepo.__init__') + log.debug('_BaseFideliusAdminRepo.__init__ (this should set tags?!?)') super().__init__(app_props, **kwargs) self._tags = tags diff --git a/fidelius/gateway/file/__init__.py b/fidelius/gateway/file/__init__.py deleted file mode 100644 index a40e345..0000000 --- a/fidelius/gateway/file/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from ._filerepo import * -from ._fileadmin import * diff --git a/fidelius/gateway/file/_filerepo.py b/fidelius/gateway/file/_filerepo.py deleted file mode 100644 index ad7e28b..0000000 --- a/fidelius/gateway/file/_filerepo.py +++ /dev/null @@ -1,36 +0,0 @@ -__all__ = [ - 'MockFideliusRepo', -] - -from fidelius.structs import * -from fidelius.gateway._abstract import * - -import json - -import logging -log = logging.getLogger(__name__) - - -class MockFideliusRepo(_BaseFideliusRepo): - def __init__(self, app_props: FideliusAppProps, pre_seeded_cache: Optional[Union[dict, str]] = None, **kwargs): - super().__init__(app_props, **kwargs) - self._cache: Dict[str, str] = {} - self._loaded: bool = False - - self._pre_seeded_cache = pre_seeded_cache - - def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: - self._load_all() - return self._cache.get(self.get_full_path(name, env=env), None) - - def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: - self._load_all() - return self._cache.get(self.get_full_path(name, folder=folder, env=env), None) - - def _load_all(self): - if not self._loaded: - if isinstance(self._pre_seeded_cache, dict): - self._cache = self._pre_seeded_cache - elif isinstance(self._pre_seeded_cache, str) and self._pre_seeded_cache.lower().endswith('.json'): - with open(self._pre_seeded_cache, 'r') as fin: - self._cache = json.load(fin) diff --git a/fidelius/gateway/interface.py b/fidelius/gateway/interface.py index fbfda2b..5be615a 100644 --- a/fidelius/gateway/interface.py +++ b/fidelius/gateway/interface.py @@ -131,10 +131,14 @@ def replace(self, string: str, no_default: bool = False) -> str: - `${__FID__:PARAM_NAME}` for app params/secrets - `${__FID__:FOLDER:PARAM_NAME}` for shared params/secrets in the given FOLDER + An empty string is returned if the parameter was not found and if the + string does not match the expression format, it will be returned + unchanged. + :param string: The expression to replace with an actual parameter/secret :param no_default: If True, does not try and get the default value if no value was found for the current set environment. - :return: The requested value + :return: The requested value, an empty string or the original string """ pass @@ -147,7 +151,9 @@ def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = N :param app_props: The application properties to use. :param tags: An optional set of meta-data tags to use when creating new parameters (if supported by the underlying - parameter/secret storage). + parameter/secret storage). Note that updating a parameter + does not update/change tags, they are only applied when + creating new parameters! """ pass diff --git a/fidelius/gateway/mock/_inmemcache.py b/fidelius/gateway/mock/_inmemcache.py new file mode 100644 index 0000000..8befe1a --- /dev/null +++ b/fidelius/gateway/mock/_inmemcache.py @@ -0,0 +1,19 @@ +__all__ = [ + '_SingletonDict', +] + +from ccptools.structs import * + + +class _SingletonDict(dict, metaclass=Singleton): + """Simple Singleton dict :) + + It's point is simply to act as a shared centralized store for the mock + stuff, mimicking how multiple instances of Fidelius Repos and/or Admin + Repos would nevertheless fetch data from the same source. + + This is just to "mock" the shared parameter/secret stuff. + + ...sneaky, right? + """ + pass diff --git a/fidelius/gateway/mock/_mockadmin.py b/fidelius/gateway/mock/_mockadmin.py index dd4515b..5684006 100644 --- a/fidelius/gateway/mock/_mockadmin.py +++ b/fidelius/gateway/mock/_mockadmin.py @@ -4,15 +4,17 @@ from fidelius.gateway._abstract import * from fidelius.structs import * +from ._mockrepo import * import logging log = logging.getLogger(__name__) -class MockFideliusAdmin(_BaseFideliusAdminRepo): +class MockFideliusAdmin(_BaseFideliusAdminRepo, MockFideliusRepo): def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): """This mock version of the Fidelius Admin stores created and updated - params in memory only. + params in memory only (although the cache is a singleton so multiple + instances of both admin and repo will be useing the same dict/data. Note that it does NOT extend the functionality of its non-Admin sibling, the MockFideliusRepo and thus does not return a base64 encoded version @@ -25,13 +27,6 @@ def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = N """ log.debug('MockFideliusAdmin.__init__') super().__init__(app_props, tags, **kwargs) - self._cache = {} - - def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: - return self._cache.get(self.get_full_path(name, env=env), None) - - def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: - return self._cache.get(self.get_full_path(name, folder, env=env), None) def _create(self, name: str, value: str, env: Optional[str] = None, folder: Optional[str] = None) -> (str, str): key = self.get_full_path(name, folder=folder, env=env) diff --git a/fidelius/gateway/mock/_mockrepo.py b/fidelius/gateway/mock/_mockrepo.py index eda39a4..7af693f 100644 --- a/fidelius/gateway/mock/_mockrepo.py +++ b/fidelius/gateway/mock/_mockrepo.py @@ -4,6 +4,7 @@ from fidelius.structs import * from fidelius.gateway._abstract import * +from ._inmemcache import _SingletonDict import base64 @@ -21,9 +22,10 @@ def __init__(self, app_props: FideliusAppProps, **kwargs): """ log.debug('MockFideliusRepo.__init__') super().__init__(app_props, **kwargs) + self._cache: _SingletonDict[str, str] = _SingletonDict() def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: - return base64.encodebytes(self.get_full_path(name, env=env).encode('utf-8')).decode('utf-8').strip() + return self._cache.get(self.get_full_path(name, env=env), None) def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: - return base64.encodebytes(self.get_full_path(name, folder=folder, env=env).encode('utf-8')).decode('utf-8').strip() + return self._cache.get(self.get_full_path(name, folder, env=env), None) diff --git a/fidelius/gateway/paramstore/_paramstoreadmin.py b/fidelius/gateway/paramstore/_paramstoreadmin.py index a97f929..79d168d 100644 --- a/fidelius/gateway/paramstore/_paramstoreadmin.py +++ b/fidelius/gateway/paramstore/_paramstoreadmin.py @@ -1,101 +1,161 @@ __all__ = [ - 'ParameterStoreAdmin', + 'AwsParameterStoreAdmin', ] from fidelius.structs import * -from fidelius.gateway.interface import * +from fidelius.gateway._abstract import * from ._paramstorerepo import * import logging log = logging.getLogger(__name__) -class ParameterStoreAdmin(IFideliusAdminRepo, ParameterStore): - def __init__(self, app: str, group: str, env: str, owner: str, finance: str = 'COST', **extra_tags): - super().__init__(app, group, env) - self._tags = FideliusTags(application=self._app, owner=owner, tier=env, finance=finance, **extra_tags) +class AwsParameterStoreAdmin(_BaseFideliusAdminRepo, AwsParamStoreRepo): + def __init__(self, app_props: FideliusAppProps, tags: Optional[FideliusTags] = None, **kwargs): + log.debug('AwsParameterStoreAdmin.__init__') + super().__init__(app_props, tags, **kwargs) - def set_env(self, env: str): - self._env = env - self._tags.tier = env + def create_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + res = self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=False, + encrypted=False) + return path, self.get_expression_string(name) - def _set_parameter(self, - full_name: str, - value: str, - encrypted: bool = False, - overwrite: bool = False, - description: Optional[str] = None) -> Dict: - kwargs = dict(Name=full_name, - Description=description or full_name, - Value=value, - Type='SecureString' if encrypted else 'String', - Overwrite=overwrite, - Tags=self._tags.to_aws_format(), - Tier='Standard') - if encrypted: - kwargs['KeyId'] = self._KEY_ID - - response = self._ssm.put_parameter(**kwargs) - return response + def update_param(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + if self.get_app_param(name, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') - def create_param(self, name: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name), + self._set_parameter(full_name=path, value=value, description=description, - overwrite=False, + overwrite=True, encrypted=False) + return path, self.get_expression_string(name) + + def delete_param(self, name: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, env=env)) - def update_param(self, name: str, value: str, description: Optional[str] = None): - if self.get(name=name, no_default=True): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=True, - encrypted=False) - else: - raise ValueError('that parameter does not exists yet, use create_param') - - def create_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), + def create_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + self._set_parameter(full_name=path, value=value, description=description, overwrite=False, encrypted=False) + return path, self.get_expression_string(name, folder=folder) + + def update_shared_param(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + if self.get_shared_param(name, folder=folder, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') - def update_shared_param(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), + self._set_parameter(full_name=path, value=value, description=description, overwrite=True, encrypted=False) + return path, self.get_expression_string(name, folder=folder) - def create_secret(self, name: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name), + def delete_shared_param(self, name: str, folder: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, folder=folder, env=env)) + + def create_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + self._set_parameter(full_name=path, value=value, description=description, overwrite=False, encrypted=True) + return path, self.get_expression_string(name) + + def update_secret(self, name: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, env=env) + if self.get_app_param(name, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') - def update_secret(self, name: str, value: str, description: Optional[str] = None): - if self.get(name=name, no_default=True): - self._set_parameter(full_name=self._full_path(name), - value=value, - description=description, - overwrite=True, - encrypted=True) - else: - raise ValueError('that secret does not exists yet, use create_secret') - - def create_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), + self._set_parameter(full_name=path, + value=value, + description=description, + overwrite=True, + encrypted=True) + return path, self.get_expression_string(name) + + def delete_secret(self, name: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, env=env)) + + def create_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + self._set_parameter(full_name=path, value=value, description=description, overwrite=False, encrypted=True) + return path, self.get_expression_string(name, folder=folder) + + def update_shared_secret(self, name: str, folder: str, value: str, + description: Optional[str] = None, env: Optional[str] = None) -> (str, str): + path = self.get_full_path(name, folder=folder, env=env) + if self.get_shared_param(name, folder=folder, env=env) is None: + raise FideliusParameterNotFound(f'parameter not found: {path}') - def update_shared_secret(self, name: str, folder: str, value: str, description: Optional[str] = None): - self._set_parameter(full_name=self._full_path(name, folder), + self._set_parameter(full_name=path, value=value, description=description, overwrite=True, encrypted=True) + return path, self.get_expression_string(name, folder=folder) + + def delete_shared_secret(self, name: str, folder: str, env: Optional[str] = None): + self._delete_parameter(full_name=self.get_full_path(name, folder=folder, env=env)) + + def _tags_to_aws_format(self) -> Optional[List[Dict[str, str]]]: + if self.tags: + return [{'Key': k, 'Value': v} for k, v in self.tags.to_dict().items()] + return None + + def _set_parameter(self, + full_name: str, + value: str, + encrypted: bool = False, + overwrite: bool = False, + description: Optional[str] = None) -> Dict: + kwargs = dict(Name=full_name, + Description=description or full_name, + Value=value, + Type='SecureString' if encrypted else 'String', + Overwrite=overwrite, + Tier='Standard') + if not overwrite: + tags = self._tags_to_aws_format() + if tags: + kwargs['Tags'] = tags + + if encrypted: + kwargs['KeyId'] = self._aws_key_arn + + try: + response = self._ssm.put_parameter(**kwargs) + return response + except self._ssm.exceptions.ParameterAlreadyExists: + raise FideliusParameterAlreadyExists(f'parameter already exists: {full_name}') + + except self._ssm.exceptions.ParameterNotFound: + raise FideliusParameterNotFound(f'parameter not found: {full_name}') + + def _delete_parameter(self, full_name: str) -> Dict: + try: + response = self._ssm.delete_parameter(Name=full_name) + return response + except self._ssm.exceptions.ParameterNotFound: + raise FideliusParameterNotFound(f'parameter not found: {full_name}') diff --git a/fidelius/gateway/paramstore/_paramstorerepo.py b/fidelius/gateway/paramstore/_paramstorerepo.py index b70999f..55556e1 100644 --- a/fidelius/gateway/paramstore/_paramstorerepo.py +++ b/fidelius/gateway/paramstore/_paramstorerepo.py @@ -19,8 +19,39 @@ def __init__(self, app_props: FideliusAppProps, aws_secret_access_key: str = None, aws_key_arn: str = None, aws_region_name: str = None, + aws_endpoint_url: str = None, + flush_cache_every_time: bool = False, **kwargs): + """Fidelius Admin Repo that uses AWS' Simple Systems Manager's Parameter Store as a back end. + + :param app_props: The current application properties. + :param aws_access_key_id: Optional AWS_ACCESS_KEY_ID which is otherwise + pulled from the FIDELIUS_AWS_ACCESS_KEY_ID or + AWS_ACCESS_KEY_ID environment variables. + :param aws_secret_access_key: Optional AWS_SECRET_ACCESS_KEY which is otherwise + pulled from the FIDELIUS_AWS_SECRET_ACCESS_KEY or + AWS_SECRET_ACCESS_KEY environment variables. + :param aws_key_arn: Optional ARN to an AWS KMS encryption key that'll be + used to encrypt secret parameters. If not provided + it'll be extracted from the FIDELIUS_AWS_KEY_ARN + environment variable and if that is missing as well, + an EnvironmentError is raised. + :param aws_region_name: Optional AWS region name, which is otherwise + extracted from the FIDELIUS_AWS_REGION_NAME or + AWS_DEFAULT_REGION environment variables or just + set to `eu-west-1` by default is completely + missing. + :param aws_endpoint_url: Optional custom AWS endpoint URL intended for + testing and development, e.g. by spinning up a + LocalStack container and pointing to that + instead of a live AWS environment. + :param flush_cache_every_time: Optional flat that'll flush the entire + cache before every operation if set to + True and is just intended for testing + purposes. + """ super().__init__(app_props, **kwargs) + self._flush_cache_every_time = flush_cache_every_time self._aws_key_arn = aws_key_arn or os.environ.get('FIDELIUS_AWS_KEY_ARN', '') if not self._aws_key_arn: @@ -28,71 +59,59 @@ def __init__(self, app_props: FideliusAppProps, self._region_name = aws_region_name or os.environ.get('FIDELIUS_AWS_REGION_NAME', None) or os.environ.get('AWS_DEFAULT_REGION', 'eu-west-1') - # TODO(thordurm@ccpgames.com>) 2024-04-09: Check for aws_access_key_id and/or aws_secret_access_key + self._aws_endpoint_url = aws_endpoint_url or os.environ.get('FIDELIUS_AWS_ENDPOINT_URL', '') self._force_log_secrecy() self._ssm = boto3.client('ssm', region_name=self._region_name, + endpoint_url=self._aws_endpoint_url or None, aws_access_key_id=aws_access_key_id or os.environ.get('FIDELIUS_AWS_ACCESS_KEY_ID', None) or os.environ.get('AWS_ACCESS_KEY_ID', None), aws_secret_access_key=aws_secret_access_key or os.environ.get('FIDELIUS_AWS_SECRET_ACCESS_KEY', None) or os.environ.get('AWS_SECRET_ACCESS_KEY', None)) self._cache: Dict[str, str] = {} - self._loaded: bool = False - self._loaded_folders: Set[str] = set() - self._default_store: Optional[AwsParamStoreRepo] = None - if self.app_props.env != 'default': - self._default_store = AwsParamStoreRepo(app_props=FideliusAppProps(app=app_props.app, group=app_props.group, env='default'), - aws_access_key_id=aws_access_key_id, - aws_secret_access_key=aws_secret_access_key, - aws_key_arn=aws_key_arn, - aws_region_name=aws_region_name) - - def _full_path(self, name: str, folder: Optional[str] = None) -> str: - if folder: - return self._SHARED_FULL_NAME.format(group=self._group, folder=folder, env=self._env, name=name) - else: - return self._APP_FULL_NAME.format(group=self._group, app=self._app, env=self._env, name=name) - - def _nameless_path(self, folder: Optional[str] = None) -> str: - return self._full_path(name='', folder=folder)[:-1] - - def _force_log_secrecy(self): + self._loaded_paths: Set[str] = set() + + def _nameless_path(self, folder: Optional[str] = None, env: Optional[str] = None) -> str: + return self.get_full_path(name='', folder=folder, env=env)[:-1] + + @staticmethod + def _force_log_secrecy(): # We don't allow debug or less logging of botocore's HTTP requests cause # those logs have unencrypted passwords in them! botolog = logging.getLogger('botocore') if botolog.level < logging.INFO: botolog.setLevel(logging.INFO) - def _load_all(self, folder: Optional[str] = None): + def _load_path(self, folder: Optional[str] = None, env: Optional[str] = None): + log.debug('AwsParamStoreRepo._load_path(folder=%s, env=%s)', folder, env) self._force_log_secrecy() - if folder: - if folder in self._loaded_folders: - return - else: - if self._loaded: - return + + # This is stuff for unit-testing only! + if self._flush_cache_every_time: + self._loaded_paths = set() + self._cache = {} + + path = self._nameless_path(folder, env) + if path in self._loaded_paths: + return response = self._ssm.get_parameters_by_path( - Path=self._nameless_path(folder), + Path=path, Recursive=True, WithDecryption=True ) + for p in response.get('Parameters', []): key = p.get('Name') if key: self._cache[key] = p.get('Value') - if folder: - self._loaded_folders.add(folder) - else: - self._loaded = True + self._loaded_paths.add(path) - def get(self, name: str, folder: Optional[str] = None, no_default: bool = False) -> Optional[str]: - self._load_all(folder) - return self._cache.get(self._full_path(name, folder), - None if no_default else self._get_default(name, folder)) + def get_app_param(self, name: str, env: Optional[str] = None) -> Optional[str]: + self._load_path(env=env) + return self._cache.get(self.get_full_path(name, env=env), None) - def _get_default(self, name: str, folder: Optional[str] = None) -> Optional[str]: - if self._default_store: - return self._default_store.get(name, folder) - return None + def get_shared_param(self, name: str, folder: str, env: Optional[str] = None) -> Optional[str]: + self._load_path(folder=folder, env=env) + return self._cache.get(self.get_full_path(name, folder, env=env), None) diff --git a/fidelius/gateway/paramstore/_std.py b/fidelius/gateway/paramstore/_std.py index d1a9e0c..87552f2 100644 --- a/fidelius/gateway/paramstore/_std.py +++ b/fidelius/gateway/paramstore/_std.py @@ -1 +1,2 @@ -from ._paramstorerepo import \ No newline at end of file +from ._paramstorerepo import AwsParamStoreRepo as FideliusRepo +from ._paramstoreadmin import AwsParameterStoreAdmin as FideliusAdmin diff --git a/fidelius/structs/_tags.py b/fidelius/structs/_tags.py index bd405cb..848f471 100644 --- a/fidelius/structs/_tags.py +++ b/fidelius/structs/_tags.py @@ -46,7 +46,3 @@ def to_dict(self) -> Dict[str, str]: if self._other: d.update(self._other) return d - - def to_aws_format(self) -> List[Dict[str, str]]: - # TODO(thordurm@ccpgames.com>) 2024-04-09: Move to param-store implementation! - return [{'Key': k, 'Value': v} for k, v in self.to_dict().items()] diff --git a/fidelius/gateway/file/_fileadmin.py b/tests/localstack/__init__.py similarity index 100% rename from fidelius/gateway/file/_fileadmin.py rename to tests/localstack/__init__.py diff --git a/tests/localstack/test_paramstore.py b/tests/localstack/test_paramstore.py new file mode 100644 index 0000000..99b67f3 --- /dev/null +++ b/tests/localstack/test_paramstore.py @@ -0,0 +1,260 @@ +import unittest + +from fidelius.fideliusapi import * +from typing import * + +import os + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + +_APP_PROPS = FideliusAppProps(app='mock-app', group='tempunittestgroup', env='mock') +_APP_PROPS_TWO = FideliusAppProps(app='other-app', group='tempunittestgroup', env='mock') +_APP_PROPS_THREE = FideliusAppProps(app='mock-app', group='tempunittestgroup', env='test') + + +def _extract_param_names(response: Dict) -> List[str]: + return [p.get('Name') for p in response.get('Parameters', [])] + + +def _get_param_names_by_path(ssm, path: str) -> List[str]: + return _extract_param_names(ssm.get_parameters_by_path(Path=path, + Recursive=True, + WithDecryption=False)) + + +def _delete_all_params(): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + ssm = fia._ssm # noqa + paths = [ + '/fidelius/tempunittestgroup/', + ] + param_set = set() + for p in paths: + param_set.update(_get_param_names_by_path(ssm, p)) + + if param_set: + res = ssm.delete_parameters(Names=list(param_set)) + log.debug('_delete_all_params -> %s', res) + + +class TestParamstore(unittest.TestCase): + @classmethod + def setUpClass(cls): + # Check if certain environment variables are set + env_var = os.getenv('FIDELIUS_AWS_ENDPOINT_URL') + if not env_var: + raise unittest.SkipTest("Environment variable 'FIDELIUS_AWS_ENDPOINT_URL' not set. " + "Skipping all tests in TestParamstore.") + # Just in case...! + _delete_all_params() + + def test_paramstore(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('MY_DB_NAME', 'brain') + fia.create_secret('MY_DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('MY_DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('MY_DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('MY_DB_NAME')) + self.assertEqual('myBADpassword', fid.get('MY_DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('MY_DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('MY_DB_USERNAME', 'dbstuff')) + _delete_all_params() + + def test_paramstore_admin_params(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_TWO')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_param('DB_PASSWORD_TWO') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_param('DB_PASSWORD_TWO', 'myBADpassword') + + key, expression = fia.create_param('DB_PASSWORD_TWO', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_TWO', key) + self.assertEqual('${__FID__:DB_PASSWORD_TWO}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_TWO')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_param('DB_PASSWORD_TWO', 'myWORSEpassword') + + key, expression = fia.update_param('DB_PASSWORD_TWO', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_TWO', key) + self.assertEqual('${__FID__:DB_PASSWORD_TWO}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_TWO')) + + fia.delete_param('DB_PASSWORD_TWO') + + self.assertIsNone(fia.get('DB_PASSWORD_TWO')) + _delete_all_params() + + def test_paramstore_admin_secrets(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_THREE')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_secret('DB_PASSWORD_THREE') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_secret('DB_PASSWORD_THREE', 'myBADpassword') + + key, expression = fia.create_secret('DB_PASSWORD_THREE', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_THREE', key) + self.assertEqual('${__FID__:DB_PASSWORD_THREE}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_THREE')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_secret('DB_PASSWORD_THREE', 'myWORSEpassword') + + key, expression = fia.update_secret('DB_PASSWORD_THREE', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/apps/mock-app/DB_PASSWORD_THREE', key) + self.assertEqual('${__FID__:DB_PASSWORD_THREE}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_THREE')) + + fia.delete_secret('DB_PASSWORD_THREE') + + self.assertIsNone(fia.get('DB_PASSWORD_THREE')) + _delete_all_params() + + def test_paramstore_admin_shared_params(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia2 = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS_TWO, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FOUR', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FOUR}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_param('DB_PASSWORD_FOUR', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FOUR', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FOUR}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + + fia.delete_shared_param('DB_PASSWORD_FOUR', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD_FOUR', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FOUR', 'dbfolder')) + _delete_all_params() + + def test_paramstore_admin_shared_secrets(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia2 = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS_TWO, flush_cache_every_time=True) + self.assertIsNone(fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FIVE', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FIVE}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_secret('DB_PASSWORD_FIVE', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/tempunittestgroup/mock/shared/dbfolder/DB_PASSWORD_FIVE', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD_FIVE}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + + fia.delete_shared_secret('DB_PASSWORD_FIVE', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD_FIVE', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD_FIVE', 'dbfolder')) + _delete_all_params() + + def test_paramstore_defaults(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('DB_NAME', 'brain', env='default') + fia.create_secret('DB_PASSWORD', 'defaultPassword', env='default') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.localhost', env='default') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock', env='default') + + fia.create_secret('DB_PASSWORD', 'mockPassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + + fid2 = FideliusFactory.get_class('paramstore')(_APP_PROPS_THREE, flush_cache_every_time=True) + self.assertIsNone(fid2.get('mock-value')) + self.assertEqual('brain', fid2.get('DB_NAME')) + self.assertEqual('defaultPassword', fid2.get('DB_PASSWORD')) + self.assertEqual('braindb.localhost', fid2.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid2.get('DB_USERNAME', 'dbstuff')) + + self.assertIsNone(fid.get('DB_NAME', no_default=True)) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD', no_default=True)) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff', no_default=True)) + self.assertIsNone(fid.get('DB_USERNAME', 'dbstuff', no_default=True)) + _delete_all_params() + + def test_paramstore_replacer(self): + fia = FideliusFactory.get_admin_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + fia.create_param('DB2_NAME', 'brain') + fia.create_secret('DB2_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB2_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB2_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('paramstore')(_APP_PROPS, flush_cache_every_time=True) + self.assertEqual('brain', fid.replace('${__FID__:DB2_NAME}')) + self.assertEqual('myBADpassword', fid.replace('${__FID__:DB2_PASSWORD}')) + self.assertEqual('braindb.mock.cc', fid.replace('${__FID__:dbstuff:DB2_HOST}')) + self.assertEqual('svc-mock', fid.replace('${__FID__:dbstuff:DB2_USERNAME}')) + self.assertEqual('', fid.replace('${__FID__:I_DONT_EXIST}')) + self.assertEqual('I am incorrectly formatted!', fid.replace('I am incorrectly formatted!')) + _delete_all_params() diff --git a/tests/offline/__init__.py b/tests/offline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/offline/test_mock.py b/tests/offline/test_mock.py new file mode 100644 index 0000000..80791cf --- /dev/null +++ b/tests/offline/test_mock.py @@ -0,0 +1,235 @@ +import unittest + +from fidelius.fideliusapi import * + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + +_APP_PROPS = FideliusAppProps(app='mock-app', group='somegroup', env='mock') +_APP_PROPS_TWO = FideliusAppProps(app='other-app', group='somegroup', env='mock') +_APP_PROPS_THREE = FideliusAppProps(app='mock-app', group='somegroup', env='test') + +from fidelius.gateway.mock._inmemcache import _SingletonDict # noqa + + +def _clear_cache(): + _SingletonDict().clear() + + +class TestMock(unittest.TestCase): + def test_mock(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain') + fia.create_secret('DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('myBADpassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + _clear_cache() + + def test_mock_admin_params(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + self.assertIsNone(fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_param('DB_PASSWORD') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_param('DB_PASSWORD', 'myBADpassword') + + key, expression = fia.create_param('DB_PASSWORD', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_param('DB_PASSWORD', 'myWORSEpassword') + + key, expression = fia.update_param('DB_PASSWORD', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD')) + + fia.delete_param('DB_PASSWORD') + + self.assertIsNone(fia.get('DB_PASSWORD')) + _clear_cache() + + def test_mock_admin_secrets(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + self.assertIsNone(fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_secret('DB_PASSWORD') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_secret('DB_PASSWORD', 'myBADpassword') + + key, expression = fia.create_secret('DB_PASSWORD', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_secret('DB_PASSWORD', 'myWORSEpassword') + + key, expression = fia.update_secret('DB_PASSWORD', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/apps/mock-app/DB_PASSWORD', key) + self.assertEqual('${__FID__:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD')) + + fia.delete_secret('DB_PASSWORD') + + self.assertIsNone(fia.get('DB_PASSWORD')) + _clear_cache() + + def test_mock_admin_shared_params(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia2 = FideliusFactory.get_admin_class('mock')(_APP_PROPS_TWO) + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_param('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_param('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_param('DB_PASSWORD', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_param('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD', 'dbfolder')) + + fia.delete_shared_param('DB_PASSWORD', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + _clear_cache() + + def test_mock_admin_shared_secrets(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia2 = FideliusFactory.get_admin_class('mock')(_APP_PROPS_TWO) + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterNotFound): + fia.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia2.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + with self.assertRaises(FideliusParameterNotFound): + fia.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + with self.assertRaises(FideliusParameterNotFound): + fia2.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + + key, expression = fia.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myBADpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myBADpassword', fia.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myBADpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + + with self.assertRaises(FideliusParameterAlreadyExists): + fia.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + with self.assertRaises(FideliusParameterAlreadyExists): + fia2.create_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + + key, expression = fia2.update_shared_secret('DB_PASSWORD', 'dbfolder', 'myWORSEpassword') + self.assertEqual('/fidelius/somegroup/mock/shared/dbfolder/DB_PASSWORD', key) + self.assertEqual('${__FID__:dbfolder:DB_PASSWORD}', expression) + + self.assertEqual('myWORSEpassword', fia2.get('DB_PASSWORD', 'dbfolder')) + self.assertEqual('myWORSEpassword', fia.get('DB_PASSWORD', 'dbfolder')) + + fia.delete_shared_secret('DB_PASSWORD', 'dbfolder') + + self.assertIsNone(fia.get('DB_PASSWORD', 'dbfolder')) + self.assertIsNone(fia2.get('DB_PASSWORD', 'dbfolder')) + _clear_cache() + + def test_mock_defaults(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain', env='default') + fia.create_secret('DB_PASSWORD', 'defaultPassword', env='default') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.localhost', env='default') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock', env='default') + + fia.create_secret('DB_PASSWORD', 'mockPassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertIsNone(fid.get('mock-value')) + self.assertEqual('brain', fid.get('DB_NAME')) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD')) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid.get('DB_USERNAME', 'dbstuff')) + + fid2 = FideliusFactory.get_class('mock')(_APP_PROPS_THREE) + self.assertIsNone(fid2.get('mock-value')) + self.assertEqual('brain', fid2.get('DB_NAME')) + self.assertEqual('defaultPassword', fid2.get('DB_PASSWORD')) + self.assertEqual('braindb.localhost', fid2.get('DB_HOST', 'dbstuff')) + self.assertEqual('svc-mock', fid2.get('DB_USERNAME', 'dbstuff')) + + self.assertIsNone(fid.get('DB_NAME', no_default=True)) + self.assertEqual('mockPassword', fid.get('DB_PASSWORD', no_default=True)) + self.assertEqual('braindb.mock.cc', fid.get('DB_HOST', 'dbstuff', no_default=True)) + self.assertIsNone(fid.get('DB_USERNAME', 'dbstuff', no_default=True)) + _clear_cache() + + def test_mock_replacer(self): + _clear_cache() + fia = FideliusFactory.get_admin_class('mock')(_APP_PROPS) + fia.create_param('DB_NAME', 'brain') + fia.create_secret('DB_PASSWORD', 'myBADpassword') + fia.create_shared_param('DB_HOST', 'dbstuff', 'braindb.mock.cc') + fia.create_shared_secret('DB_USERNAME', 'dbstuff', 'svc-mock') + + fid = FideliusFactory.get_class('mock')(_APP_PROPS) + self.assertEqual('brain', fid.replace('${__FID__:DB_NAME}')) + self.assertEqual('myBADpassword', fid.replace('${__FID__:DB_PASSWORD}')) + self.assertEqual('braindb.mock.cc', fid.replace('${__FID__:dbstuff:DB_HOST}')) + self.assertEqual('svc-mock', fid.replace('${__FID__:dbstuff:DB_USERNAME}')) + self.assertEqual('', fid.replace('${__FID__:I_DONT_EXIST}')) + self.assertEqual('I am incorrectly formatted!', fid.replace('I am incorrectly formatted!')) + _clear_cache() diff --git a/tests/offline/test_singleton_dict.py b/tests/offline/test_singleton_dict.py new file mode 100644 index 0000000..9042540 --- /dev/null +++ b/tests/offline/test_singleton_dict.py @@ -0,0 +1,80 @@ +import unittest + +from fidelius.gateway.mock._inmemcache import _SingletonDict # noqa + +import logging +log = logging.getLogger(__file__) +logging.basicConfig(level=logging.DEBUG) + + +class TestSingletonDict(unittest.TestCase): + def test_singleton_dict(self): + d = _SingletonDict() + d.clear() # Just in case! + + d2 = _SingletonDict() + + self.assertFalse(bool(d)) + self.assertFalse(bool(d2)) + + self.assertFalse('foo' in d) + self.assertFalse('foo' in d2) + + self.assertIsNone(d.get('foo')) + self.assertIsNone(d2.get('foo')) + + with self.assertRaises(KeyError): + _ = d['foo'] + + with self.assertRaises(KeyError): + _ = d2['foo'] + + d['foo'] = 'bar' + + self.assertTrue(bool(d)) + self.assertTrue(bool(d2)) + + self.assertTrue('foo' in d) + self.assertTrue('foo' in d2) + + self.assertEqual('bar', d.get('foo')) + self.assertEqual('bar', d2.get('foo')) + + self.assertEqual('bar', d['foo']) + self.assertEqual('bar', d2['foo']) + + d3 = _SingletonDict() + + self.assertTrue(bool(d3)) + self.assertTrue('foo' in d3) + self.assertEqual('bar', d3.get('foo')) + self.assertEqual('bar', d3['foo']) + + d3['foo'] = 'not bar' + + self.assertEqual('not bar', d['foo']) + self.assertEqual('not bar', d2['foo']) + self.assertEqual('not bar', d3['foo']) + + del d2['foo'] + + self.assertFalse(bool(d)) + self.assertFalse(bool(d2)) + self.assertFalse(bool(d3)) + + self.assertFalse('foo' in d) + self.assertFalse('foo' in d2) + self.assertFalse('foo' in d3) + + self.assertIsNone(d.get('foo')) + self.assertIsNone(d2.get('foo')) + self.assertIsNone(d3.get('foo')) + + with self.assertRaises(KeyError): + _ = d['foo'] + + with self.assertRaises(KeyError): + _ = d2['foo'] + + with self.assertRaises(KeyError): + _ = d3['foo'] diff --git a/tests/test_mock.py b/tests/test_mock.py deleted file mode 100644 index 83e2153..0000000 --- a/tests/test_mock.py +++ /dev/null @@ -1,26 +0,0 @@ -import unittest - -from fidelius.fideliusapi import * - -_app_props = FideliusAppProps(app='mock-app', group='somegroup', env='mock') - - -class TestMock(unittest.TestCase): - def test_mock(self): - fid = FideliusFactory.get_class('mock')(_app_props) - self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL2FwcHMvbW9jay1hcHAvbW9jay12YWx1ZQ==', fid.get('mock-value')) - self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL2FwcHMvbW9jay1hcHAvREJfUEFTU1dPUkQ=', fid.get('DB_PASSWORD')) - self.assertEqual('L2ZpZGVsaXVzL3NvbWVncm91cC9tb2NrL3NoYXJlZC9zaGFyZWRwb3N0Z3Jlcy9EQl9IT1NU', fid.get('DB_HOST', 'sharedpostgres')) - - def test_mock_admin(self): - fia = FideliusFactory.get_admin_class('mock')(_app_props) - self.assertIsNone(fia.get('DB_PASSWORD')) - - with self.assertRaises(FideliusParameterNotFound): - fia.delete_param('DB_PASSWORD') - - with self.assertRaises(FideliusParameterNotFound): - fia.update_param('DB_PASSWORD', 'myBADpassword') - - key, expression = fia.create_param('DB_PASSWORD', 'myBADpassword') - diff --git a/tests/test_some_basic_stuff.py b/tests/test_some_basic_stuff.py deleted file mode 100644 index 374e0e4..0000000 --- a/tests/test_some_basic_stuff.py +++ /dev/null @@ -1,7 +0,0 @@ -import unittest - -from fidelius.gateway.mock import * - - -class TestSomeBasicStuff(unittest.TestCase): - pass \ No newline at end of file