diff --git a/ocm_util.py b/ocm_util.py index f6713c9b..9cb226de 100644 --- a/ocm_util.py +++ b/ocm_util.py @@ -1,6 +1,8 @@ import collections.abc import logging +import urllib.parse +import ci.util import cnudie.iter import cnudie.retrieve_async import dso.model @@ -64,7 +66,7 @@ def iter_local_blob_content( ) -async def find_artefact_node( +async def find_artefact_node_async( component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, artefact: dso.model.ComponentArtefactId, absent_ok: bool=False, @@ -119,3 +121,57 @@ async def find_artefact_node( if not absent_ok: raise ValueError(f'could not find OCM node for {artefact=}') + + +def to_absolute_access( + access: ocm.OciAccess | ocm.RelativeOciAccess, + ocm_repo: ocm.OciOcmRepository=None, +) -> ocm.OciAccess: + if access.type is ocm.AccessType.OCI_REGISTRY: + return access + + if access.type is ocm.AccessType.RELATIVE_OCI_REFERENCE: + if not '://' in ocm_repo.baseUrl: + base_url = urllib.parse.urlparse(f'x://{ocm_repo.baseUrl}').netloc + else: + base_url = urllib.parse.urlparse(ocm_repo.baseUrl).netloc + + return ocm.OciAccess( + imageReference=ci.util.urljoin(base_url, access.reference), + ) + + raise NotImplementedError(access.type) + + +def find_artefact_node( + component: ocm.Component, + component_descriptor_lookup, + artefact_name: str, + artefact_version: str, + artefact_type: str, + node_filter: collections.abc.Callable | None, + absent_ok: bool=False, + recursion_depth: int=-1, +) -> cnudie.iter.ArtefactNode | None: + for rnode in cnudie.iter.iter( + component=component, + lookup=component_descriptor_lookup, + node_filter=node_filter, + recursion_depth=recursion_depth, + ): + rnode: cnudie.iter.ResourceNode + if not ( + rnode.resource.name == artefact_name + and rnode.resource.version == artefact_version + and rnode.resource.type == artefact_type + ): + continue + + return rnode + + else: + if absent_ok: + return None + + raise ValueError(f'no ocm node found for {artefact_name=} {artefact_version=} \ + {artefact_type=}') diff --git a/odg_operator/__init__.py b/odg_operator/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/odg_operator/odg_controller.py b/odg_operator/odg_controller.py new file mode 100644 index 00000000..28cc1a21 --- /dev/null +++ b/odg_operator/odg_controller.py @@ -0,0 +1,249 @@ +import argparse +import collections +import collections.abc +import http +import logging +import os + +import dacite +import kubernetes.client +import kubernetes.client.rest +import kubernetes.watch +import urllib3 +import yaml + +import ci.log +import cnudie.iter +import oci.client + +import k8s.util +import lookups +import odg_operator.odg_model as odgm + + +ci.log.configure_default_logging() +logger = logging.getLogger(__name__) +own_dir = os.path.abspath(os.path.dirname(__file__)) +CUSTOMER_CLEANUP_FINALIZER = 'open-delivery-gear.ocm.software/customer-cluster-cleanup' +ODG_COMPONENT_NAME = 'ocm.software/ocm-gear' + + +def find_extension_definition( + extension_definitions: list[odgm.ExtensionDefinition], + extension_name: str, + absent_ok: bool=False, +) -> odgm.ExtensionDefinition | None: + for extension_definition in extension_definitions: + if extension_definition.name == extension_name: + return extension_definition + + if absent_ok: + return None + + raise ValueError(f'unknown extension-definition for {extension_name=}') + + +def iter_missing_dependencies( + requested: collections.abc.Container[odgm.ExtensionDefinition], + known: collections.abc.Container[odgm.ExtensionDefinition], +) -> collections.abc.Generator[odgm.ExtensionDefinition, None, None]: + ''' + recursively add known extensions until all dependencies are included. + assumes extension-definitions are consistent. + ''' + seen = set([e.name for e in requested]) + + def resolve( + dependencies, + ): + for dependency in dependencies: + if dependency in seen: + continue + + missing_extension_definition = find_extension_definition( + extension_definitions=known, + extension_name=dependency, + ) + + yield missing_extension_definition + seen.add(missing_extension_definition.name) + yield from resolve(missing_extension_definition.dependencies) + + for extension_definition in requested: + yield from resolve(extension_definition.dependencies) + + +def outputs_as_jsonpath( + outputs_by_extension: dict, +) -> dict: + ''' + convert outputs as templated by extensions to lookup dict ready to use with `jsonpaths_ng`. + ''' + output_lookup = collections.defaultdict(lambda: collections.defaultdict(dict)) + for name, outputs in outputs_by_extension.items(): + _outputs = {} + for output in outputs: + output: odgm.ExtensionOutput + _outputs[output.name] = output.value + output_lookup['dependencies'][name]['outputs'] = _outputs + return dict(output_lookup) + + +def reconcile( + extension_definitions: list[odgm.ExtensionDefinition], + component_descriptor_lookup, + group: str= odgm.ODGExtensionMeta.group, + plural: str = odgm.ODGMeta.plural, +): + ''' + watches for events of ODG custom-resource + creates, updates and deletes ODG installations using managed-resources + ''' + + logger.info(f'watching for events: {group=} {plural=}') + + try: + for event in kubernetes.watch.Watch().stream( + kubernetes_api.custom_kubernetes_api.list_cluster_custom_object, + group=group, + version='v1', + plural=plural, + resource_version=resource_version, + timeout_seconds=0, + ): + metadata = event['object'].get('metadata') + odg_name = metadata['name'] + logger.info(f'{event["type"]} "{odg_name}" in "{metadata["namespace"]}"') + + requested_extension_definitions = [ + find_extension_definition( + extension_definitions=extension_definitions, + extension_name=extension_name, + ) + for extension_name in event['object']['spec']['extensions'] + ] + + requested_extension_definitions.extend(list( + iter_missing_dependencies( + requested=requested_extension_definitions, + known=extension_definitions, + ) + )) + + context = event['object']['spec']['context'] + + outputs_for_extension = dict([ + ( + extension_definition.name, + extension_definition.templated_outputs(context), + ) + for extension_definition in requested_extension_definitions + ]) + outputs_jsonpath = outputs_as_jsonpath(outputs_for_extension) + + extension_instances = [ + odgm.ExtensionInstance.from_definition( + extension_definition=extension_definition, + outputs=outputs_jsonpath, + component_descriptor_lookup=component_descriptor_lookup, + ) + for extension_definition in requested_extension_definitions + ] + + import pprint + for extension_instance in extension_instances: + pprint.pprint(extension_instance) + + # TODO: create managed resources + + except kubernetes.client.rest.ApiException as e: + if e.status == http.HTTPStatus.GONE: + resource_version = '' + logger.info('API resource watching expired, will start new watch') + else: + raise e + + except urllib3.exceptions.ProtocolError: + # this is a known error which has no impact on the functionality, thus rather be + # degregated to a warning or even info + # [ref](https://github.com/kiwigrid/k8s-sidecar/issues/233#issuecomment-1332358459) + resource_version = '' + logger.info('API resource watching received protocol error, will start new watch') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--kubeconfig') + parser.add_argument('--extension-definition-file') + parser.add_argument( + '--extension', + dest='extensions', + action='append', + default=[], + help='can be specified multiple times, \ + expected format: ::' + ) + parsed = parser.parse_args() + + oci_client = oci.client.Client( + credentials_lookup=lambda **kwargs: None, # consume public oci-images only + ) + component_descriptor_lookup = lookups.init_component_descriptor_lookup( + cache_dir='./cache/ocm', + oci_client=oci_client, + ) + + extension_definitions = [] + + if parsed.extension_definition_file: + with open(parsed.extension_definition_file) as f: + extensions_raw = yaml.safe_load_all(f) + extension_definitions.extend([ + dacite.from_dict( + data=extension_raw, + data_class=odgm.ExtensionDefinition, + ) + for extension_raw in extensions_raw + ]) + + for extension in parsed.extensions: + extension: str + component_id, artefact_name = extension.rsplit(':', 1) + component = component_descriptor_lookup(component_id).component + for resource_node in cnudie.iter.iter( + component=component, + recursion_depth=0, + node_filter=cnudie.iter.Filter.resources, + ): + if ( + resource_node.resource.type == 'odg-extension' + and resource_node.resource.name == artefact_name + ): + break + else: + raise ValueError(f'no odg-extension found in {extension}') + + resource_node: cnudie.iter.ResourceNode + odg_extension_raw = oci_client.blob( + image_reference=resource_node.component.current_ocm_repo.component_version_oci_ref( + name=resource_node.component.name, + version=resource_node.component.version, + ), + digest=resource_node.resource.access.localReference, + stream=False, + ).json() + extension_definitions.append(dacite.from_dict( + data=odg_extension_raw, + data_class=odgm.ExtensionDefinition, + )) + + logger.info(f'known extension definitions: {[e.name for e in extension_definitions]}') + + kubernetes_api = k8s.util.kubernetes_api(kubeconfig_path=parsed.kubeconfig) + resource_version = '' + + while True: + reconcile( + extension_definitions=extension_definitions, + component_descriptor_lookup=component_descriptor_lookup, + ) diff --git a/odg_operator/odg_model.py b/odg_operator/odg_model.py new file mode 100644 index 00000000..86d21ab3 --- /dev/null +++ b/odg_operator/odg_model.py @@ -0,0 +1,198 @@ +import dataclasses +import enum +import string + +import jsonpath_ng +import cnudie.iter +import oci +import oci.client +import ocm + +import odg_operator.odg_util as odgu +import ocm_util + + +class ManagedResourceClasses(enum.StrEnum): + INTERNAL = 'internal' + EXTERNAL = 'external' + + +@dataclasses.dataclass +class ManagedResourceMeta: + # see: https://github.com/gardener/gardener/blob/master/docs/concepts/resource-manager.md + group: str = 'resources.gardener.cloud' + version: str = 'v1alpha1' + plural: str = 'managedresources' + kind: str = 'ManagedResource' + apiVersion: str = 'resources.gardener.cloud/v1alpha1' + + +@dataclasses.dataclass +class ODGExtensionMeta: + group: str = 'open-delivery-gear.ocm.software' + version: str = 'v1' + plural: str = 'odges' + kind: str = 'ODGE' + + @staticmethod + def apiVersion() -> str: + return f'{ODGExtensionMeta.group}/{ODGExtensionMeta.version}' + + +@dataclasses.dataclass +class ODGMeta: + group: str = 'open-delivery-gear.ocm.software' + version: str = 'v1' + plural: str = 'odgs' + kind: str = 'ODG' + + @staticmethod + def apiVersion() -> str: + return f'{ODGMeta.group}/{ODGMeta.version}' + + +@dataclasses.dataclass +class OcmArtefactReference: + name: str + version: str + + +@dataclasses.dataclass +class InstallationOcmReference: + ref_name: str + name: str + version: str + artefact: OcmArtefactReference + mappings: list[OcmArtefactReference] + + +@dataclasses.dataclass +class InstallationValues: + helm_reference: str + helm_attribute: str + value: str + + +@dataclasses.dataclass +class ExtensionInstallation: + ocm_references: list[InstallationOcmReference] = dataclasses.field(default_factory=list) + values: list[InstallationValues] = dataclasses.field(default_factory=list) + + +@dataclasses.dataclass +class ExtensionOutput: + name: str + value: str + + +@dataclasses.dataclass +class ExtensionDefinition: + name: str + installation: ExtensionInstallation + outputs: list[ExtensionOutput] = dataclasses.field(default_factory=list) + dependencies: list[str] = dataclasses.field(default_factory=list) + + def templated_outputs(self, context: dict) -> list[ExtensionOutput]: + return [ + ExtensionOutput( + name=output.name, + value=string.Template(output.value).substitute(context), + ) + for output in self.outputs + ] + + def __hash__(self): + return hash(self.name) + + +@dataclasses.dataclass +class ExtensionInstance: + name: str + artefacts: list[ocm.Resource] + values: list[InstallationValues] + + @staticmethod + def from_definition( + extension_definition: ExtensionDefinition, + outputs: dict, + component_descriptor_lookup, + absent_ok: bool=False, + ) -> 'ExtensionInstance': + values = [ + dataclasses.replace( + value_ref, + value=jsonpath_ng.parse(value_ref.value).find(outputs)[0].value + ) + for value_ref in extension_definition.installation.values + ] + + extension_installation_resources = [] + for ocm_ref in extension_definition.installation.ocm_references: + component_descriptor = component_descriptor_lookup(f'{ocm_ref.name}:{ocm_ref.version}') + component: ocm.Component = component_descriptor.component + resource_node = ocm_util.find_artefact_node( + component=component, + absent_ok=absent_ok, + component_descriptor_lookup=component_descriptor_lookup, + artefact_name=ocm_ref.artefact.name, + artefact_version=ocm_ref.artefact.version, + artefact_type=ocm.ArtefactType.HELM_CHART, + node_filter=cnudie.iter.Filter.resources, + recursion_depth=0, + ) + if ( + absent_ok + and not resource_node + ): + continue + + extension_installation_resources.append(resource_node.resource) + + oci_client = oci.client.Client( + credentials_lookup=lambda **kwargs: None, # only support public images for now + ) + for mapping in ocm_ref.mappings: + resource_node: cnudie.iter.ResourceNode = ocm_util.find_artefact_node( + component=component, + absent_ok=absent_ok, + component_descriptor_lookup=component_descriptor_lookup, + artefact_name=ocm_ref.artefact.name, + artefact_version=ocm_ref.artefact.version, + artefact_type='helmchart-imagemap', + node_filter=cnudie.iter.Filter.resources, + recursion_depth=0, + ) + + if ( + absent_ok + and not resource_node + ): + continue + + image_mappings = oci_client.blob( + image_reference=resource_node.component.current_ocm_repo.component_version_oci_ref( # noqa: E501 + name=resource_node.component.name, + version=resource_node.component.version, + ), + digest=resource_node.resource.access.localReference, + # imagemaps are typically small, so it should be okay to read into memory + stream=False, + ).json()['imageMapping'] + + image_mappings = odgu.resolved_image_mappings( + image_mappings=image_mappings, + component=component, + ) + + for path, value in image_mappings.items(): + values.append(InstallationValues( + helm_reference=mapping.name, + helm_attribute=path, + value=value, + )) + + return ExtensionInstance( + name=extension_definition.name, + artefacts=extension_installation_resources, + values=values, + ) diff --git a/odg_operator/odg_util.py b/odg_operator/odg_util.py new file mode 100644 index 00000000..e9492d3f --- /dev/null +++ b/odg_operator/odg_util.py @@ -0,0 +1,39 @@ +import oci.model +import ocm + +import ocm_util + + +def resolved_image_mappings( + image_mappings: list[dict], + component: ocm.Component, +) -> dict: + for image_mapping in image_mappings: + resource_name = image_mapping['resource']['name'] + for resource in component.resources: + if resource.name != resource_name: + continue + if not resource.type is ocm.ArtefactType.OCI_IMAGE: + continue + break # found it + else: + raise ValueError(f'did not find oci-image w/ {resource_name=} in {component=}') + + resource.access = ocm_util.to_absolute_access( + access=resource.access, + ocm_repo=component.current_ocm_repo, + ) + image_ref = oci.model.OciImageReference(resource.access.imageReference) + + if image_ref.has_mixed_tag: + # special-handling, as OciImageReference will - for backwards-compatibility - always + # return digest-tag for "mixed tags" + symbolic_tag, digest_tag = image_ref.parsed_mixed_tag + tag = f'{symbolic_tag}@{digest_tag}' + else: + tag = image_ref.tag + + return { + image_mapping['repository']: image_ref.ref_without_tag, + image_mapping['tag']: tag, + } diff --git a/paths.py b/paths.py index ce186663..c534fe46 100644 --- a/paths.py +++ b/paths.py @@ -32,6 +32,11 @@ _odg_path = os.path.join(_own_dir, 'odg') +test_resources_extension_definitions = os.path.join( + _own_dir, + 'test/resources/extension-definitions.yaml', +) + def features_cfg_candidates() -> collections.abc.Generator[str, None, None]: if features_cfg_path := os.environ.get('FEATURES_CFG_PATH'): diff --git a/rescore/artefacts.py b/rescore/artefacts.py index 1b022cb2..18f3d127 100644 --- a/rescore/artefacts.py +++ b/rescore/artefacts.py @@ -759,7 +759,7 @@ async def get(self): ) if odg.findings.FindingType.VULNERABILITY in type_filter: - artefact_node = await ocm_util.find_artefact_node( + artefact_node = await ocm_util.find_artefact_node_async( component_descriptor_lookup=self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP], artefact=artefact, absent_ok=True, diff --git a/test/resources/extension-definitions.yaml b/test/resources/extension-definitions.yaml new file mode 100644 index 00000000..11a6ae06 --- /dev/null +++ b/test/resources/extension-definitions.yaml @@ -0,0 +1,55 @@ +name: delivery-service +installation: + ocm_references: + - ref_name: delivery-service + name: ocm.software/ocm-gear/delivery-service + version: 0.1142.0 + artefact: + name: delivery-service + version: 0.1142.0 + mappings: [] + values: + - helm_reference: delivery-service + helm_attribute: db-url + value: dependencies.delivery-db.outputs.db-url +outputs: +- name: delivery-service-url + value: api.${base_url} +dependencies: +- delivery-db +--- +name: delivery-dashboard +installation: + ocm_references: + - ref_name: delivery-dashboard + name: ocm.software/ocm-gear/delivery-dashboard + version: 0.386.0 + artefact: + name: delivery-dashboard + version: 0.386.0 + mappings: + - name: delivery-dashboard + version: 0.386.0 + values: + - helm_reference: delivery-dashboard + helm_attribute: backend-url + value: dependencies.delivery-service.outputs.delivery-service-url +outputs: [] +dependencies: +- delivery-service +--- +name: delivery-db +installation: + ocm_references: + - ref_name: delivery-database + name: ocm.software/ocm-gear/delivery-database + version: 0.7.0 + artefact: + name: postgresql + version: 16.6.1 + mappings: [] + values: [] +outputs: +- name: db-url + value: db.${base_url} +dependencies: [] \ No newline at end of file diff --git a/test/test_modg_extension.py b/test/test_modg_extension.py new file mode 100644 index 00000000..c5d62aec --- /dev/null +++ b/test/test_modg_extension.py @@ -0,0 +1,69 @@ +import unittest.mock + +import dacite +import jsonpath_ng +import pytest +import yaml + +import odg_operator.odg_controller as odgc +import odg_operator.odg_model as odgm + +import paths + + +@pytest.fixture() +def extension_definitions(): + with open(paths.test_resources_extension_definitions, 'r') as f: + return list(yaml.safe_load_all(f)) + + +@pytest.fixture +def component_mock(): + mock = unittest.mock.Mock() + mock.component.resources = [] + mock.component.sources = [] + mock.component.componentReferences = [] + mock.component.find_label = lambda name: [] + return mock + + +def test_extensions(extension_definitions, component_mock): + ds, dd, db = [ + dacite.from_dict( + data=raw, + data_class=odgm.ExtensionDefinition, + ) + for raw in extension_definitions + ] + + missing = set(odgc.iter_missing_dependencies( + requested=(dd,), + known=(ds, dd, db), + )) + assert missing == set([ds, db]) + + context = { + 'base_url': 'my-domain.com', + } + + outputs = dict([ + (extension_definition.name, extension_definition.templated_outputs(context)) + for extension_definition in (dd, ds, db) + ]) + + ds_outputs = outputs['delivery-service'] + assert ds_outputs[0].name == 'delivery-service-url' + assert ds_outputs[0].value == 'api.my-domain.com' + + output_paths = odgc.outputs_as_jsonpath(outputs) + path = jsonpath_ng.parse('dependencies.delivery-service.outputs.delivery-service-url') + assert path.find(output_paths)[0].value == 'api.my-domain.com' + + dd_instance: odgm.ExtensionInstance = odgm.ExtensionInstance.from_definition( + extension_definition=dd, + outputs=output_paths, + # we know ocm lookup works + component_descriptor_lookup=lambda _: component_mock, + absent_ok=True, + ) + assert dd_instance.values[0].value == 'api.my-domain.com'