New config.EnvExtendedInterpolation; release 1.8.0

Author: pcisar

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.8.0] - 2024-05-03
+
+### Added
+
+- New `EnvExtendedInterpolation` class in `cnfig` module that extends `configparser.ExtendedInterpolation`
+  with special handling for "env" section that returns value of specified environment
+  variable, or empty string if such variable is not defined.
+
 ## [1.7.2] - 2024-02-20
 
 ### Fixed

docs/changelog.txt

@@ -2,6 +2,16 @@
 Changelog
 #########
 
+Version 1.8.0
+=============
+
+* `~firebird.base.config` module:
+
+  - New `.EnvExtendedInterpolation` class that extends `configparser.ExtendedInterpolation`
+    with special handling for "env" section that returns value of specified environment
+    variable, or empty string if such variable is not defined.
+
+
 Version 1.7.2
 =============
 

docs/conf.py

@@ -20,7 +20,7 @@
 # -- Project information -----------------------------------------------------
 
 project = 'Firebird-base'
-copyright = '2020-2023, The Firebird Project'
+copyright = '2020-2024, The Firebird Project'
 author = 'Pavel Císař'
 
 # The short X.Y version

docs/config.txt

@@ -33,16 +33,21 @@ The framework is based around two classes:
   `.PyCode` and `.PyCallable`. It also provides special options `ConfigOption` and
   `ConfigListOption`.
 
-Additionally, the `.ApplicationDirectoryScheme` abstract base class defines set of mostly
+Additionally, the `.DirectoryScheme` abstract base class defines set of mostly
 used application directories. The function `.get_directory_scheme()` could be then used
 to obtain instance that implements platform-specific standards for file-system location
 for these directories. Currently, only "Windows", "Linux" and "MacOS" directory schemes
 are supported.
 
-.. note::
-    You may use `platform.system` call to determine the scheme name suitable for platform
+.. tip::
+    You may use `.get_directory_scheme()` function to get the scheme suitable for platform
     where your application is running.
 
+.. tip::
+    If your configurations contain secrets like passwords or access tokens, that would be
+    read from files via `configparser`, you should consider to use `.EnvExtendedInterpolation`
+    that has support for option values defined via environment variables.
+
 Usage
 -----
 First, you need to define your own configuration.
@@ -288,7 +293,7 @@ The protobuf message is defined in :file:`/proto/config.proto`.
       cfg_msg = ConfigProto()
 
    Because the proto file is NOT registered in `.protobuf` registry, you must register
-   it manually. The proto file is listed in `setup.cfg` under *"firebird.base.protobuf"*
+   it manually. The proto file is listed in `pyproject.toml` under *"firebird.base.protobuf"*
    entrypoint, so use `load_registered('firebird.base.protobuf')` for its registration.
 
    .. code-block::
@@ -331,6 +336,11 @@ Application Directory Scheme
 .. autoclass:: MacOSDirectoryScheme
 .. autofunction:: get_directory_scheme
 
+Configparser interpolation
+==========================
+
+.. autoclass:: EnvExtendedInterpolation
+
 Config
 ======
 .. autoclass:: Config

docs/introduction.txt

@@ -69,10 +69,11 @@ configuration that supports:
 * configuration options of various data type, including lists and other complex types
 * validation
 * direct manipulation of configuration values
-* reading from (and writing into) configuration in `configparser` format
+* reading from (and writing into) configuration in `configparser` format, including support
+  for option values defined via environment variables (see `.EnvExtendedInterpolation`)
 * exchanging configuration (for example between processes) using Google protobuf messages
 
-Additionally, the `.ApplicationDirectoryScheme` abstract base class defines set of mostly
+Additionally, the `.DirectoryScheme` abstract base class defines set of mostly
 used application directories. The function `.get_directory_scheme()` could be then used
 to obtain instance that implements platform-specific standards for file-system location
 for these directories. Currently, only "Windows", "Linux" and "MacOS" directory schemes

docs/requirements.txt

@@ -1,4 +1,4 @@
 sphinx-bootstrap-theme>=0.8.1
 sphinx-autodoc-typehints>=1.24.0
-sphinx>=5.3
+sphinx==7.2.6
 .

pyproject.toml

@@ -81,7 +81,7 @@ python = ["3.8", "3.9", "3.10", "3.11", "3.12"]
 detached = false
 platforms = ["linux"]
 dependencies = [
-  "Sphinx>=7.2.6",
+  "Sphinx==7.2.6",
   "sphinx-bootstrap-theme>=0.8.1",
   "sphinx-autodoc-typehints>=1.24.0",
   "doc2dash>=3.0.0"

src/firebird/base/__about__.py

@@ -1,4 +1,4 @@
 # SPDX-FileCopyrightText: 2020-present The Firebird Projects <www.firebirdsql.org>
 #
 # SPDX-License-Identifier: MIT
-__version__ = "1.7.2"
+__version__ = "1.8.0"

src/firebird/base/config.py

@@ -55,7 +55,10 @@
 from pathlib import Path
 from uuid import UUID
 from decimal import Decimal, DecimalException
-from configparser import ConfigParser, DEFAULTSECT
+from configparser import (ConfigParser, DEFAULTSECT, ExtendedInterpolation,
+                          MAX_INTERPOLATION_DEPTH, InterpolationDepthError,
+                          InterpolationSyntaxError, NoSectionError, NoOptionError,
+                          InterpolationMissingOptionError)
 from inspect import signature, Signature, Parameter
 from enum import Enum, Flag
 import os
@@ -140,6 +143,74 @@ def _power_of_two(value):
         return False
     return value == 2 ** (value.bit_length() - 1)
 
+class EnvExtendedInterpolation(ExtendedInterpolation):
+    """.. versionadded:: 1.8.0
+
+    Modified version of `configparser.ExtendedInterpolation` class that adds special
+    handling for "env" section that returns value of specified environment variable,
+    or empty string if such variable is not defined.
+
+    Example::
+
+       ${env:path} is reference to PATH environment variable.
+    """
+    def _interpolate_some(self, parser, option, accum, rest, section, map,
+                          depth):
+        rawval = parser.get(section, option, raw=True, fallback=rest)
+        if depth > MAX_INTERPOLATION_DEPTH:
+            raise InterpolationDepthError(option, section, rawval)
+        while rest:
+            p = rest.find("$")
+            if p < 0:
+                accum.append(rest)
+                return
+            if p > 0:
+                accum.append(rest[:p])
+                rest = rest[p:]
+            # p is no longer used
+            c = rest[1:2]
+            if c == "$":
+                accum.append("$")
+                rest = rest[2:]
+            elif c == "{":
+                m = self._KEYCRE.match(rest)
+                if m is None:
+                    raise InterpolationSyntaxError(option, section,
+                        "bad interpolation variable reference %r" % rest)
+                path = m.group(1).split(':')
+                rest = rest[m.end():]
+                sect = section
+                opt = option
+                try:
+                    if len(path) == 1:
+                        opt = parser.optionxform(path[0])
+                        v = map[opt]
+                    elif len(path) == 2:
+                        sect = path[0]
+                        opt = parser.optionxform(path[1])
+                        if sect == 'env':
+                            v = os.getenv(opt.upper(), '')
+                        else:
+                            v = parser.get(sect, opt, raw=True)
+                    else:
+                        raise InterpolationSyntaxError(
+                            option, section,
+                            "More than one ':' found: %r" % (rest,))
+                except (KeyError, NoSectionError, NoOptionError):
+                    raise InterpolationMissingOptionError(
+                        option, section, rawval, ":".join(path)) from None
+                if "$" in v:
+                    self._interpolate_some(parser, opt, accum, v, sect,
+                                           dict(parser.items(sect, raw=True)),
+                                           depth + 1)
+                else:
+                    accum.append(v)
+            else:
+                raise InterpolationSyntaxError(
+                    option, section,
+                    "'$' must be followed by '$' or '{', "
+                    "found: %r" % (rest,))
+
 class DirectoryScheme:
     """Class that provide paths to typically used application directories.
 

tests/test_config.py

@@ -47,7 +47,7 @@
 from enum import IntEnum, IntFlag, Flag, auto
 from dataclasses import dataclass
 from inspect import signature
-from configparser import ConfigParser, ExtendedInterpolation
+from configparser import ConfigParser
 from firebird.base.types import Error, ZMQAddress, MIME, PyExpr, PyCode, PyCallable
 from firebird.base.strconv import convert_to_str
 from firebird.base import config
@@ -108,7 +108,7 @@ class BaseConfigTest(TestCase):
     "Base class for firebird.base.config unit tests"
     def setUp(self):
         self.proto: config.ConfigProto = config.ConfigProto()
-        self.conf: ConfigParser = ConfigParser(interpolation=ExtendedInterpolation())
+        self.conf: ConfigParser = ConfigParser(interpolation=config.EnvExtendedInterpolation())
     def tearDown(self):
         pass
     def setConf(self, conf_str):
@@ -3384,6 +3384,31 @@ def test_08_linux_home_change(self, *args):
         self.assertEqual(scheme.user_sync, Path('~/.local/sync/test_app').expanduser())
         self.assertEqual(scheme.user_cache, Path('~/.cache/test_app').expanduser())
 
+class TestInterpolation(BaseConfigTest):
+    "Unit tests for firebird.base.config.EnvExtendedInterpolation"
+    CONFIG = """[base]
+base_value = BASE
+
+[my-config]
+value_str = VALUE
+value_int = 1
+base_value = ${base:base_value}
+value_env_1 = ${env:mysecret}
+value_env_2 = ${env:not-present}
+value_env_path = ${env:path}
+"""
+    @mock.patch.dict(os.environ, {'MYSECRET': 'secret'})
+    def test_01(self):
+        cfg = ConfigParser(interpolation=config.EnvExtendedInterpolation())
+        cfg.read_string(self.CONFIG)
+        self.assertEqual(cfg['my-config']['value_str'], 'VALUE')
+        self.assertEqual(cfg['my-config']['value_int'], '1')
+        self.assertEqual(cfg['my-config']['base_value'], 'BASE')
+        self.assertEqual(cfg['my-config']['value_env_1'], 'secret')
+        self.assertEqual(cfg['my-config']['value_env_2'], '')
+        self.assertEqual(cfg['my-config']['value_env_path'], os.getenv('PATH'))
+
+
 if __name__ == '__main__':
     unittest_main()