jinjaPlugin

#!/usr/bin/python3

import logging
import os
import base64
import sys
import copy
import yaml
from collections import OrderedDict
from jinja2 import Environment
from functools import reduce
import operator

# yaml-helpers: to avoid reordering of fields when read yaml, we need to use OrderedDict
# see
# https://github.com/yaml/pyyaml/issues/110 and
# https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts
# string scalar formatter is taken from:
# https://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data
def _yaml_orderedLoad(arg, _all=False, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    if _all == True:
        return yaml.load_all(arg, OrderedLoader)
    return yaml.load(arg, OrderedLoader)

def _yaml_orderedDump(data, _all=False, stream=None, Dumper=yaml.Dumper, **kwds):
    class OrderedDumper(Dumper):
        pass
    def str_presenter(dumper, data):
        if len(data.splitlines()) > 1:  # check for multiline string
            return dumper.represent_scalar('tag:yaml.org,2002:str', data, style='|')
        return dumper.represent_scalar('tag:yaml.org,2002:str', data)
    OrderedDumper.add_representer(str, str_presenter)

    def _dict_representer(dumper, data):
        return dumper.represent_mapping(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            data.items())
    OrderedDumper.add_representer(OrderedDict, _dict_representer)
    if _all == True:
        result = ''
        for y in data:
            result+=yaml.dump(y, stream, OrderedDumper, **kwds)
            result+='---\n'
        return result
    return yaml.dump(data, stream, OrderedDumper, **kwds)

def yaml_load(y, _all=False):
    return _yaml_orderedLoad(y, _all=_all)

def yaml_dump(y, _all=False):
    return _yaml_orderedDump(y, _all=_all, default_flow_style=False)

# symmetric-encryption-helper: the basic functionality to decrypt/encrypt configential data
# see https://nitratine.net/blog/post/encryption-and-decryption-in-python/#what-is-symmetric-encryption
def _crypt_getKeyFromPassword(password_provided):
    from cryptography.hazmat.backends import default_backend
    from cryptography.hazmat.primitives import hashes
    from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC

    password = password_provided.encode() # Convert to type bytes
    salt = b'salt_' # CHANGE THIS - recommend using a key from os.urandom(16), must be of type bytes
    kdf = PBKDF2HMAC(
        algorithm=hashes.SHA256(),
        length=32,
        salt=salt,
        iterations=100000,
        backend=default_backend()
    )
    return base64.urlsafe_b64encode(kdf.derive(password))

def _crypt_getKey():
    return _crypt_getKeyFromPassword(os.environ['JINJA_CRYPT_PASSWORD'])

def crypt_encrypt(s):
    from cryptography.fernet import Fernet
    f = Fernet(_crypt_getKey())
    return base64.b64encode(f.encrypt(s.encode())).decode()

def crypt_decrypt(e):
    from cryptography.fernet import Fernet
    f = Fernet(_crypt_getKey())
    return f.decrypt(base64.b64decode(e.encode())).decode()

# vault-helper:
# see more https://www.vaultproject.io/docs/commands/
# https://github.com/hvac/hvac 
# and https://hvac.readthedocs.io/en/stable/overview.html#getting-started
def vault_getClient():
    import hvac
    # TODO: read some extended params from file if needed (in home dir or provided via env)
    return hvac.Client(url = os.environ['VAULT_ADDR'],
            token=os.environ['VAULT_TOKEN'])

# taken from https://stackoverflow.com/questions/14692690/access-nested-dictionary-items-via-a-list-of-keys
# can also use https://pypi.org/project/dpath/ 
def _dict_getByPath(dataDict, path):
    mapList = [int(i) if i.isdigit() else i for i in path.split('/')]
    try:
        return reduce(operator.getitem, mapList, dataDict)
    except:
        #TODO: catch only corrent exceptions
        logger.error("filter_fieldExits error: {}\n".format(str(sys.exc_info()[0])))
    return None

def dict_getByPath(dataDict, pathComplex, _all=False):
    val = dataDict
    pathList = pathComplex.split('|')
    for i in range(len(pathList)):
        res = _dict_getByPath(val, pathList[i])
        if res is None:
            return res
        if i == (len(pathList) - 1):
            return res
        val = yaml_load(res, _all=_all)   # _all allows to parse yaml with many yamls as string in the field

def operation_set(value):
    def _set(obj, index):
        logger.debug("_set obj={} index={} value={}\n".format(str(obj), str(index), str(value)))
        obj[index] = value
    return logger_wrap(_set)

def operation_del():
    def _delete(obj, index):
        del obj[index]
    return logger_wrap(_delete)

def operation_merge(value):
    def _merge(obj, index):
        if index == '':
            obj.update(value)
        else:
            obj[index].update(value)
    return logger_wrap(_merge)

def dict_modifyByPath(dataDict, pathComplex, operation, _all=False):
    res = dataDict
    pathList = pathComplex.split('|')
    if (len(pathList) > 0):
        mapList = [int(i) if i.isdigit() else i for i in pathList[0].split('/')]
        res = copy.deepcopy(dataDict)
        if len(pathList) == 1:
            logger.debug("operation={} mapList={} res={}\n".format(str(operation), str(mapList), str(res)))
            reduced = reduce(operator.getitem, mapList[:-1], res)
            logger.debug("reduced={}\n".format(str(reduced)))
            operation(reduced, mapList[-1])
        else:
            org = reduce(operator.getitem, mapList[:-1], dataDict)[mapList[-1]]
            reduce(operator.getitem, mapList[:-1], res)[mapList[-1]] = \
                    yaml_dump(dict_modifyByPath(yaml_load(org, _all=_all), '|'.join(pathList[1:]), operation, _all=_all), _all=_all)
    return res

def filter_(value):
    def _value(dataDict):
        logger.debug("closure value={}\n".format(str(value)))
        return value
    return logger_wrap(_value)

def filter_fieldExist(path):
    def _exist(dataDict):
        logger.debug("closure path={}\n".format(str(path)))
        return not dict_getByPath(dataDict, path) is None
    return logger_wrap(_exist)
        
def filter_fieldEqual(path, val):
    def _equal(dataDict):
        logger.debug("closure path={} val={}\n".format(str(path),str(val)))
        path_val = dict_getByPath(dataDict, path)
        if (not path_val is None) and (path_val == val):
            return True
        return False
    return logger_wrap(_equal)

def filter_operatorNot(filterToInvert):
    def _not(dataDict):
        return not filterToInvert(dataDict)
    return logger_wrap(_not)

def filter_operatorAnd(*args):
    if len(args) < 2:
        raise ValueError('filter_operatorAnd needs at least 2 arguments')
    def _and(dataDict):
        for i in args:
            if not i(dataDict):
                return False
        return True
    return logger_wrap(_and)

def filter_operatorOr(*args):
    if len(args) < 2:
        raise ValueError('filter_operatorOr needs at least 2 arguments')
    def _or(dataDict):
        for i in args:
            if i(dataDict):
                return True
        return False
    return logger_wrap(_or)

def filter_getMatchingIndexes(dataDictArray, filterToApply):
    result = []
    for i in range(len(dataDictArray)):
        if filterToApply(dataDictArray[i]):
            result.append(i)
    return result

def filter_modifyMatching(dataDictArray, filterToApply, *modify):
    resultArray = []
    for i in  range(len(dataDictArray)):
        if filterToApply(dataDictArray[i]):
            res = dataDictArray[i]
            for i in modify:
                res = i(res)
            resultArray.append(res)
        else:
            resultArray.append(dataDictArray[i])
    return resultArray

def modify_byPath(pathComplex, operation, _all=False):
    def _byPath(obj):
        logger.debug("closure pathComplex={} operation={} _all={}\n".format(str(pathComplex),str(operation),str(_all)))
        return dict_modifyByPath(obj, pathComplex, operation, _all=_all)
    return logger_wrap(_byPath)

def logger_wrap(fn):
    def wrapper(*args, **kwargs):
        logger.debug("{} args=\n\t{}\n\tkwargs={}\n".format(fn.__name__, str(args), str(kwargs)))
        result = fn(*args, **kwargs)
        logger.debug("{} result=\n\t{}\n".format(fn.__name__, str(result)))
        return result
    return wrapper

def get_functions():
    return {
        'yaml_load': logger_wrap(yaml_load),
        'yaml_dump': logger_wrap(yaml_dump),
        'crypt_encrypt': logger_wrap(crypt_encrypt),
        'crypt_decrypt': logger_wrap(crypt_decrypt),
        'vault_getClient': logger_wrap(vault_getClient),
        'dict_getByPath': logger_wrap(dict_getByPath),
        'operation_set': logger_wrap(operation_set),
        'operation_del': logger_wrap(operation_del),
        'operation_merge': logger_wrap(operation_merge),
        'dict_modifyByPath': logger_wrap(dict_modifyByPath),
        'filter_fieldExist': logger_wrap(filter_fieldExist),
        'filter_fieldEqual': logger_wrap(filter_fieldEqual),
        'filter_operatorNot': logger_wrap(filter_operatorNot),
        'filter_operatorAnd': logger_wrap(filter_operatorAnd),
        'filter_operatorOr': logger_wrap(filter_operatorOr),
        'filter_getMatchingIndexes': logger_wrap(filter_getMatchingIndexes),
        'filter_modifyMatching': logger_wrap(filter_modifyMatching),
        'modify_byPath': logger_wrap(modify_byPath), }
#-------------------------------------------------------------------
def get_input():
    input = []
    for i in yaml_load(sys.stdin, _all=True):
        input.append(i)
    return input

def render_JinjaDocument(global_obj):

    apiVersion = global_obj["config"].get("apiVersion")
    kind = global_obj["config"].get("kind")
    sources = global_obj["config"].get("sources")
    jinjaTemplate = global_obj["config"].get("jinjaTemplate")

    if apiVersion is None or kind is None or not kind in ["JinjaGenerator", "JinjaTransformer"] or jinjaTemplate is None:
        sys.exit("Can't render Jinja from this document {} - exiting".format(str(global_obj["config"])))

    if not sources is None:
        for source in sources:

            source_path = source.get('path')
            source_root = source.get('root')
            source__all = source.get('_all')

            if source_path is None:
                sys.exit("source doesn't have a mandatory field path: {} - exiting", source)

            if source_root is None:
                source_root = global_obj['config_root']

            if source__all is None:
                source__all = False

            source["object"] = yaml_load(process_config_from_file(os.path.realpath(os.path.join(global_obj['config_root'], source_root)), source_path), _all=source__all)
            if source["object"] is None:
                sys.exit("couldn't read yaml - for source {} exiting".format(str(source)))

    logger.debug("rendering with global_obj={}\n".format(str(global_obj)))

    env = Environment()

    global_obj['fn'] = get_functions()
    env.filters['yaml_dump'] = logger_wrap(yaml_dump)
    env.filters['filter_modifyMatching'] = logger_wrap(filter_modifyMatching)

    template = env.from_string(jinjaTemplate)
    rendered = template.render(global_obj)

    logger.debug("rendered ={}\n".format(str(rendered)))

    return rendered

def process_config_from_file(config_root, config_path):
    logger.debug("process_config_from_file config_root={}, config_path:\n{}\n".format(str(config_root), str(config_path)))

    global_obj = {
        'config_root': config_root}

    with open(os.path.join(config_root, config_path)) as f:
        global_obj["config"] = yaml_load(f) #TODO: decode(utf-8)?
    if  global_obj["config"] is None:
        sys.exit("wasn't able to read {}".format(str(config_path)))

    config_kind = global_obj["config"].get("kind")
    if config_kind is None or not config_kind in ["JinjaGenerator"]:
            return yaml_dump(global_obj["config"])

    return render_JinjaDocument(global_obj)

def process_config_from_string(config_root, config_string):
    logger.debug("process_config_from_string config_root={}, config_string:\n{}\n".format(str(config_root), str(config_string)))

    global_obj = {
        'config_root': config_root}

    global_obj["config"] = yaml_load(config_string)
    if global_obj["config"] is None:
        sys.exit("wasn't able to read config")
    logger.debug("config:\n{}\n".format(str(global_obj["config"])))

    config_kind = global_obj["config"].get("kind")
    if config_kind is None or not config_kind in ["JinjaGenerator", "JinjaTransformer"]:
        sys.exit("kind is incorrect: {} - exiting".format(str(config_kind)))

    if config_kind == "JinjaTransformer":
        global_obj["resources"] = get_input()

    return render_JinjaDocument(global_obj)


# configure logger
logger = logging.getLogger('jinjaPlugin')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
if not os.environ.get('JINJA_DEBUG') is None and int(os.environ['JINJA_DEBUG']) > 0:
    ch.setLevel(logging.DEBUG)
else:
    ch.setLevel(logging.ERROR)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

# do the actual work and output the result
print(
    process_config_from_string(
        os.environ['KUSTOMIZE_PLUGIN_CONFIG_ROOT'],
        os.environ['KUSTOMIZE_PLUGIN_CONFIG_STRING']
    )
)