From a7e211163ff22e39bd64d8029d8499ed69b91229 Mon Sep 17 00:00:00 2001 From: TuanAnh17N Date: Fri, 25 Apr 2025 14:06:49 +0200 Subject: [PATCH 1/4] drop compliance summary os id --- app.py | 7 - cache_manager.py | 22 -- compliance_summary/__init__.py | 253 ++---------------- compliance_summary/artefact_metadata_cfg.yaml | 20 -- components.py | 10 - consts.py | 1 - metadata.py | 22 +- odg/findings.py | 1 - odg/findings_cfg.yaml | 17 -- odg/profiles.yaml | 1 - paths.py | 3 - swagger/swagger.yaml | 1 - test/test_compliance_summary.py | 171 ------------ 13 files changed, 26 insertions(+), 503 deletions(-) delete mode 100644 compliance_summary/artefact_metadata_cfg.yaml diff --git a/app.py b/app.py index fc001e61..7da03c85 100755 --- a/app.py +++ b/app.py @@ -13,7 +13,6 @@ import artefacts import compliance_tests -import compliance_summary as cs import components import consts import ctx_util @@ -36,7 +35,6 @@ import service_extensions import special_component import sprint -import util ci.log.configure_default_logging(print_thread_id=True) @@ -140,10 +138,6 @@ def add_app_context_vars( addressbook_github_mappings = [] addressbook_source = None - artefact_metadata_cfg_by_type = cs.artefact_metadata_cfg_by_type( - artefact_metadata_cfg=util.parse_yaml_file(paths.artefact_metadata_cfg), - ) - component_with_tests_callback = features.get_feature( features.FeatureTests, ).get_component_with_tests @@ -193,7 +187,6 @@ def add_app_context_vars( app[consts.APP_ADDRESSBOOK_ENTRIES] = addressbook_entries app[consts.APP_ADDRESSBOOK_GITHUB_MAPPINGS] = addressbook_github_mappings app[consts.APP_ADDRESSBOOK_SOURCE] = addressbook_source - app[consts.APP_ARTEFACT_METADATA_CFG] = artefact_metadata_cfg_by_type app[consts.APP_BASE_URL] = base_url app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] = component_descriptor_lookup app[consts.APP_COMPONENT_WITH_TESTS_CALLBACK] = component_with_tests_callback diff --git a/cache_manager.py b/cache_manager.py index 6689679f..433a693e 100644 --- a/cache_manager.py +++ b/cache_manager.py @@ -31,15 +31,12 @@ import ctx_util import deliverydb import deliverydb.model as dm -import eol import k8s.logging -import k8s.model import k8s.util import lookups import odg.extensions_cfg import odg.findings import paths -import util logger = logging.getLogger(__name__) @@ -139,9 +136,7 @@ def interval_min(column: sqlalchemy.DateTime) -> sqlalchemy.sql.elements.BinaryE async def prefill_compliance_summary_cache( component_id: ocm.ComponentIdentity, component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, - eol_client: eol.EolClient, finding_cfgs: collections.abc.Sequence[odg.findings.Finding], - artefact_metadata_cfg_by_type: dict[str, compliance_summary.ArtefactMetadataCfg], db_session: sqlasync.session.AsyncSession, ): logger.info(f'Updating compliance summary for {component_id.name}:{component_id.version}') @@ -154,8 +149,6 @@ async def prefill_compliance_summary_cache( datasource=dso.model.Datatype.datatype_to_datasource(finding_cfg.type), db_session=db_session, component_descriptor_lookup=component_descriptor_lookup, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg_by_type.get(finding_cfg.type), ) @@ -164,9 +157,7 @@ async def prefill_compliance_summary_caches( component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, version_lookup: cnudie.retrieve_async.VersionLookupByComponent, oci_client: oci.client_async.Client, - eol_client: eol.EolClient, finding_cfgs: collections.abc.Sequence[odg.findings.Finding], - artefact_metadata_cfg_by_type: dict[str, compliance_summary.ArtefactMetadataCfg], invalid_semver_ok: bool, db_session: sqlasync.session.AsyncSession, ): @@ -211,9 +202,7 @@ async def prefill_compliance_summary_caches( await prefill_compliance_summary_cache( component_id=component_id, component_descriptor_lookup=component_descriptor_lookup, - eol_client=eol_client, finding_cfgs=finding_cfgs, - artefact_metadata_cfg_by_type=artefact_metadata_cfg_by_type, db_session=db_session, ) @@ -238,9 +227,7 @@ async def prefill_function_caches( component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, version_lookup: cnudie.retrieve_async.VersionLookupByComponent, oci_client: oci.client_async.Client, - eol_client: eol.EolClient, finding_cfgs: collections.abc.Sequence[odg.findings.Finding], - artefact_metadata_cfg_by_type: dict[str, compliance_summary.ArtefactMetadataCfg], invalid_semver_ok: bool, db_session: sqlasync.session.AsyncSession, ): @@ -254,9 +241,7 @@ async def prefill_function_caches( component_descriptor_lookup=component_descriptor_lookup, version_lookup=version_lookup, oci_client=oci_client, - eol_client=eol_client, finding_cfgs=finding_cfgs, - artefact_metadata_cfg_by_type=artefact_metadata_cfg_by_type, invalid_semver_ok=invalid_semver_ok, db_session=db_session, ) @@ -361,7 +346,6 @@ async def main(): ) oci_client = lookups.semver_sanitising_oci_client_async(secret_factory) - eol_client = eol.EolClient() component_descriptor_lookup = lookups.init_component_descriptor_lookup_async( cache_dir=parsed_arguments.cache_dir, @@ -374,10 +358,6 @@ async def main(): default_absent_ok=True, ) - artefact_metadata_cfg_by_type = compliance_summary.artefact_metadata_cfg_by_type( - artefact_metadata_cfg=util.parse_yaml_file(paths.artefact_metadata_cfg), - ) - db_session = await deliverydb.sqlalchemy_session(db_url) try: cache_size_bytes = await db_size(db_session=db_session) @@ -398,9 +378,7 @@ async def main(): component_descriptor_lookup=component_descriptor_lookup, version_lookup=version_lookup, oci_client=oci_client, - eol_client=eol_client, finding_cfgs=finding_cfgs, - artefact_metadata_cfg_by_type=artefact_metadata_cfg_by_type, invalid_semver_ok=parsed_arguments.invalid_semver_ok, db_session=db_session, ) diff --git a/compliance_summary/__init__.py b/compliance_summary/__init__.py index c9088b69..49cf16bf 100644 --- a/compliance_summary/__init__.py +++ b/compliance_summary/__init__.py @@ -6,23 +6,16 @@ import functools import logging -import awesomeversion -import dacite import sqlalchemy.ext.asyncio as sqlasync import cnudie.retrieve import cnudie.retrieve_async import dso.model import ocm -import unixutil.model as um -import delivery.model -import delivery.util as du import deliverydb.cache import deliverydb.util -import eol import odg.findings -import osinfo import rescore.utility @@ -64,147 +57,6 @@ def rescored_severity_if_any( return most_specific_rescoring.data.severity -@dataclasses.dataclass(frozen=True) -class SeverityMappingBase: - severityName: str - - -@dataclasses.dataclass(frozen=True) -class OsStatusMapping(SeverityMappingBase): - status: list[str] - - async def match( - self, - finding: dso.model.ArtefactMetadata, - **kwargs, - ) -> str | None: - - eol_client = kwargs['eol_client'] - - class OsStatus(enum.Enum): - ''' - values used to map severity, see `compliance_summary/artefact_metadata_cfg.yaml` - ''' - NO_BRANCH_INFO = 'noBranchInfo' - NO_RELEASE_INFO = 'noReleaseInfo' - UNABLE_TO_COMPARE_VERSION = 'unableToCompareVersion' - IS_EOL = 'isEol' - UPDATE_AVAILABLE_FOR_BRANCH = 'updateAvailableForBranch' - GREATEST_BRANCH_VERSION = 'greatestBranchVersion' - EMPTY_OS_ID = 'emptyOsId' - - def empty_os_id( - os_id: um.OperatingSystemId, - ) -> bool: - if not any([ - field - for field in os_id.__dict__.values() - ]): - return True - return False - - def determine_status(release_infos: list[delivery.model.OsReleaseInfo]) -> OsStatus: - branch_info = du.find_branch_info( - os_id=os_id, - os_infos=release_infos, - ) - - if not branch_info: - return OsStatus.NO_BRANCH_INFO - - is_eol = du.branch_reached_eol( - os_id=os_id, - os_infos=release_infos, - ) - - try: - update_avilable = du.update_available( - os_id=os_id, - os_infos=release_infos, - ) - except awesomeversion.exceptions.AwesomeVersionCompareException: - return OsStatus.UNABLE_TO_COMPARE_VERSION - - if is_eol: - return OsStatus.IS_EOL - - if update_avilable: - return OsStatus.UPDATE_AVAILABLE_FOR_BRANCH - - return OsStatus.GREATEST_BRANCH_VERSION - - def severity_for_os_status(os_status: OsStatus) -> str | None: - severity_name = self.severityName - for mapping_status in self.status: - if mapping_status == os_status.value: - return severity_name - return None - - os_id = finding.data.os_info - - if empty_os_id(os_id): - return severity_for_os_status(OsStatus.EMPTY_OS_ID) - - release_infos = osinfo.os_release_infos( - os_id=eol.normalise_os_id(os_id.ID), - eol_client=eol_client, - ) - - if not release_infos: - logger.debug(f'did not find release-info for {os_id=}') - return severity_for_os_status(OsStatus.NO_RELEASE_INFO) - - return severity_for_os_status(determine_status(release_infos)) - - -@dataclasses.dataclass(frozen=True) -class ArtefactMetadataCfg: - ''' - Represents configuration for a single ArtefactMetadataType. - `categories` classifies an ArtefactMetadataType, e.g. compliance or structureInfo. - `severityMappings` holds mapping configuration to a single severity string. - Each evaluated ArtefactMetadataType has its own Mapping dataclass. - ''' - type: str - severityMappings: list[OsStatusMapping] | None - categories: list[str] = dataclasses.field(default_factory=list) - - async def match( - self, - finding: dso.model.ArtefactMetadata, - **kwargs, - ) -> ComplianceEntryCategorisation | str | None: - ''' - matches finding against severityMappings - ''' - for severity_mapping in self.severityMappings: - if (severity := await severity_mapping.match(finding, **kwargs)): - return severity - - raise RuntimeError(f'no severity mapping for {finding.meta.type=}') - - -def artefact_metadata_cfg_by_type(artefact_metadata_cfg: dict) -> dict[str, ArtefactMetadataCfg]: - ''' - parse raw cfg, raise on duplicate artefact metadata type names - ''' - - cfg_by_type = {} - - for artefact_metadata_cfg_raw in artefact_metadata_cfg['artefactMetadataCfg']: - cfg = dacite.from_dict( - data_class=ArtefactMetadataCfg, - data=artefact_metadata_cfg_raw, - ) - - if cfg_by_type.get(cfg.type): - raise RuntimeError(f'duplicate artefact metadata cfg for {cfg.type=}') - - cfg_by_type[cfg.type] = cfg - - return cfg_by_type - - class ComplianceScanStatus: NO_DATA = 'no_data' OK = 'ok' @@ -245,14 +97,10 @@ class ComponentComplianceSummary: async def severity_for_finding( finding: dso.model.ArtefactMetadata, - artefact_metadata_cfg: ArtefactMetadataCfg | None=None, rescorings: collections.abc.Iterable[dso.model.ArtefactMetadata]=tuple(), - eol_client: eol.EolClient | None=None, ) -> ComplianceEntryCategorisation | str | None: ''' Severity for known `ArtefactMetadata`. - - Raises `RuntimeError` if no severity mapping could be applied. ''' if rescorings: loop = asyncio.get_running_loop() @@ -264,23 +112,13 @@ async def severity_for_finding( if rescored_severity: return rescored_severity - if hasattr(finding.data, 'severity'): - # these types have the severity already stored in their data field - # no need to do separate severity mapping - return finding.data.severity - - return await artefact_metadata_cfg.match( - finding=finding, - eol_client=eol_client, - ) + return finding.data.severity async def calculate_summary_entry( finding_cfg: odg.findings.Finding, findings: collections.abc.Iterable[dso.model.ArtefactMetadata], rescorings: collections.abc.Iterable[dso.model.ArtefactMetadata], - eol_client: eol.EolClient, - artefact_metadata_cfg: ArtefactMetadataCfg | None=None, ) -> ComplianceSummaryEntry: ''' returns most severe (highest semantic value) `ComplianceSummaryEntry` @@ -292,8 +130,6 @@ async def calculate_summary_entry( severity_name = await severity_for_finding( finding=finding, rescorings=rescorings, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg, ) categorisation = finding_cfg.categorisation_by_id(severity_name) @@ -319,8 +155,6 @@ async def compliance_summary_entry( scan_exists: bool, findings: collections.abc.Sequence[dso.model.ArtefactMetadata], rescorings: collections.abc.Sequence[dso.model.ArtefactMetadata], - eol_client: eol.EolClient, - artefact_metadata_cfg: ArtefactMetadataCfg, ) -> ComplianceSummaryEntry: if not scan_exists: return ComplianceSummaryEntry( @@ -344,8 +178,6 @@ async def compliance_summary_entry( finding_cfg=finding_cfg, findings=findings, rescorings=rescorings, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg, ) @@ -356,8 +188,6 @@ async def artefact_datatype_summary( artefact_scan_infos: collections.abc.Sequence[dso.model.ArtefactMetadata], findings: collections.abc.Sequence[dso.model.ArtefactMetadata], rescorings: collections.abc.Sequence[dso.model.ArtefactMetadata], - eol_client: eol.EolClient, - artefact_metadata_cfg: ArtefactMetadataCfg, ) -> ComplianceSummaryEntry: findings_for_artefact = [ finding for finding in findings @@ -386,20 +216,15 @@ async def artefact_datatype_summary( ) ] - if not dso.model.Datasource.has_scan_info(datasource): - # TODO remove this conditional branch once all datasources emit scan info objects - scan_exists = bool(findings_for_artefact) - + for artefact_scan_info in artefact_scan_infos: + if ( + artefact_scan_info.artefact.artefact_kind is artefact.artefact_kind + and artefact_scan_info.artefact.artefact == artefact.artefact + ): + scan_exists = True + break else: - for artefact_scan_info in artefact_scan_infos: - if ( - artefact_scan_info.artefact.artefact_kind is artefact.artefact_kind - and artefact_scan_info.artefact.artefact == artefact.artefact - ): - scan_exists = True - break - else: - scan_exists = False + scan_exists = False return await compliance_summary_entry( finding_cfg=finding_cfg, @@ -407,8 +232,6 @@ async def artefact_datatype_summary( scan_exists=scan_exists, findings=findings_for_artefact, rescorings=rescorings_for_artefact, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg, ) @@ -420,8 +243,6 @@ async def artefact_datatype_summary( exclude_kwargs=( 'finding_cfg', 'component_descriptor_lookup', - 'eol_client', - 'artefact_metadata_cfg', 'ocm_repo', ), ) @@ -432,8 +253,6 @@ async def component_datatype_summaries( datasource: str, db_session: sqlasync.session.AsyncSession, component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, - eol_client: eol.EolClient, - artefact_metadata_cfg: ArtefactMetadataCfg, ocm_repo: ocm.OciOcmRepository | None=None, shortcut_cache: bool=False, ) -> list[tuple[dso.model.ComponentArtefactId, ComplianceSummaryEntry]]: @@ -445,49 +264,33 @@ async def component_datatype_summaries( else: component = (await component_descriptor_lookup(component)).component - if not dso.model.Datasource.has_scan_info(datasource): - # TODO remove this conditional branch once all datasources emit scan info objects - artefact_scan_infos = None + artefact_scan_infos = await deliverydb.util.findings_for_component( + component=component, + finding_type=dso.model.Datatype.ARTEFACT_SCAN_INFO, + datasource=datasource, + db_session=db_session, + ) + if artefact_scan_infos: findings = await deliverydb.util.findings_for_component( component=component, finding_type=finding_type, datasource=datasource, db_session=db_session, ) - - # the remaining datasources do not support rescorings so retrieval of the same - # can be safely skipped in this case - rescorings = [] - else: - artefact_scan_infos = await deliverydb.util.findings_for_component( + # if no scan exists, we don't have to query for findings + findings = [] + + if findings: + rescorings = await deliverydb.util.rescorings_for_component( component=component, - finding_type=dso.model.Datatype.ARTEFACT_SCAN_INFO, - datasource=datasource, + finding_type=finding_type, db_session=db_session, ) - - if artefact_scan_infos: - findings = await deliverydb.util.findings_for_component( - component=component, - finding_type=finding_type, - datasource=datasource, - db_session=db_session, - ) - else: - # if no scan exists, we don't have to query for findings - findings = [] - - if findings: - rescorings = await deliverydb.util.rescorings_for_component( - component=component, - finding_type=finding_type, - db_session=db_session, - ) - else: - # if no findings exist, we don't have to query for rescorings - rescorings = [] + else: + # if no findings exist, we don't have to query for rescorings + rescorings = [] summaries = [] for artefact in component.resources + component.sources: @@ -506,8 +309,6 @@ async def component_datatype_summaries( artefact_scan_infos=artefact_scan_infos, findings=findings, rescorings=rescorings, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg, ) summaries.append(( @@ -543,8 +344,6 @@ async def component_compliance_summary( finding_cfgs: collections.abc.Sequence[odg.findings.Finding], db_session: sqlasync.session.AsyncSession, component_descriptor_lookup: cnudie.retrieve_async.ComponentDescriptorLookupById, - eol_client: eol.EolClient, - artefact_metadata_cfg_by_type: dict[str, ArtefactMetadataCfg], ocm_repo: ocm.OciOcmRepository | None=None, shortcut_cache: bool=False, ) -> ComponentComplianceSummary: @@ -559,8 +358,6 @@ async def component_compliance_summary( datasource=dso.model.Datatype.datatype_to_datasource(finding_cfg.type), db_session=db_session, component_descriptor_lookup=component_descriptor_lookup, - eol_client=eol_client, - artefact_metadata_cfg=artefact_metadata_cfg_by_type.get(finding_cfg.type), ocm_repo=ocm_repo, shortcut_cache=shortcut_cache, ) diff --git a/compliance_summary/artefact_metadata_cfg.yaml b/compliance_summary/artefact_metadata_cfg.yaml deleted file mode 100644 index d248773a..00000000 --- a/compliance_summary/artefact_metadata_cfg.yaml +++ /dev/null @@ -1,20 +0,0 @@ -artefactMetadataCfg: -- type: os_ids - categories: - - compliance - severityMappings: - - severityName: CRITICAL - status: - - isEol - - severityName: MEDIUM - status: - - updateAvailableForBranch - - severityName: UNKNOWN - status: - - emptyOsId - - noBranchInfo - - noReleaseInfo - - unableToCompareVersion - - severityName: CLEAN - status: - - greatestBranchVersion diff --git a/components.py b/components.py index 7f60085b..4bfe75ea 100644 --- a/components.py +++ b/components.py @@ -2,7 +2,6 @@ import dataclasses import datetime import dataclasses_json -import enum import http import io import logging @@ -11,9 +10,7 @@ import aiohttp.web import dacite.exceptions -import sqlalchemy as sa import sqlalchemy.ext.asyncio as sqlasync -import sqlalchemy.orm.query as sq import yaml import cnudie.iter @@ -32,12 +29,9 @@ import config import consts import deliverydb.cache -import deliverydb.model as dm -import deliverydb.util import features import lookups import responsibles -import responsibles.github_statistics import responsibles.labels import util import yp @@ -1290,9 +1284,7 @@ async def get(self): $ref: '#/definitions/ComplianceSummary' ''' params = self.request.rel_url.query - artefact_metadata_cfg_by_type = self.request.app[consts.APP_ARTEFACT_METADATA_CFG] component_descriptor_lookup = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] - eol_client = self.request.app[consts.APP_EOL_CLIENT] invalid_semver_ok = self.request.app[consts.APP_INVALID_SEMVER_OK] version_filter_callback = self.request.app[consts.APP_VERSION_FILTER_CALLBACK] version_lookup = self.request.app[consts.APP_VERSION_LOOKUP] @@ -1349,8 +1341,6 @@ async def get(self): finding_cfgs=finding_cfgs, db_session=db_session, component_descriptor_lookup=component_descriptor_lookup, - eol_client=eol_client, - artefact_metadata_cfg_by_type=artefact_metadata_cfg_by_type, ocm_repo=ocm_repo, shortcut_cache=shortcut_cache, ) for component in components diff --git a/consts.py b/consts.py index a82beada..7bcbb47c 100644 --- a/consts.py +++ b/consts.py @@ -7,7 +7,6 @@ APP_ADDRESSBOOK_ENTRIES = 'addressbook_entries' APP_ADDRESSBOOK_GITHUB_MAPPINGS = 'addressbook_github_mappings' APP_ADDRESSBOOK_SOURCE = 'addressbook_source' -APP_ARTEFACT_METADATA_CFG = 'artefact_metadata_cfg' APP_BASE_URL = 'base_url' APP_COMPONENT_DESCRIPTOR_LOOKUP = 'component_descriptor_lookup' APP_COMPONENT_WITH_TESTS_CALLBACK = 'component_with_tests_callback' diff --git a/metadata.py b/metadata.py index d35e97a2..b3612384 100644 --- a/metadata.py +++ b/metadata.py @@ -1,5 +1,4 @@ import collections.abc -import dataclasses import datetime import http @@ -11,7 +10,6 @@ import dso.model import ocm -import compliance_summary as cs import consts import deliverydb.cache as dc import deliverydb.model as dm @@ -84,9 +82,7 @@ async def post(self): items: $ref: '#/definitions/ArtefactMetadata' ''' - artefact_metadata_cfg_by_type = self.request.app[consts.APP_ARTEFACT_METADATA_CFG] component_descriptor_lookup = self.request.app[consts.APP_COMPONENT_DESCRIPTOR_LOOKUP] - eol_client = self.request.app[consts.APP_EOL_CLIENT] params = self.request.rel_url.query body = await self.request.json() @@ -204,23 +200,7 @@ def result_dict( return finding_dict - cfg = artefact_metadata_cfg_by_type.get(finding.meta.type) - - if not cfg: - return result_dict(finding) - - severity = await cs.severity_for_finding( - finding=finding, - artefact_metadata_cfg=cfg, - eol_client=eol_client, - ) - if not severity: - return result_dict(finding) - - return result_dict( - finding=finding, - meta=dict(**dataclasses.asdict(finding.meta), severity=severity), - ) + return result_dict(finding) db_session: sqlasync.session.AsyncSession = self.request[consts.REQUEST_DB_SESSION] db_stream = await db_session.stream(db_statement) diff --git a/odg/findings.py b/odg/findings.py index f6eaade2..e3501fb0 100644 --- a/odg/findings.py +++ b/odg/findings.py @@ -75,7 +75,6 @@ class FindingType(enum.StrEnum): LICENSE = 'finding/license' MALWARE = 'finding/malware' OSID = 'finding/osid' - OS_IDS = 'os_ids' SAST = 'finding/sast' VULNERABILITY = 'finding/vulnerability' FALCO = 'finding/falco' diff --git a/odg/findings_cfg.yaml b/odg/findings_cfg.yaml index fb0eef15..fe98515e 100644 --- a/odg/findings_cfg.yaml +++ b/odg/findings_cfg.yaml @@ -139,23 +139,6 @@ ratings: - not-compliant -- type: os_ids - issues: - enable_issues: False - categorisations: - - id: UNKNOWN - display_name: UNKNOWN - value: -1 - - id: CLEAN - display_name: CLEAN - value: 0 - - id: MEDIUM - display_name: MEDIUM - value: 2 - - id: CRITICAL - display_name: CRITICAL - value: 8 - - type: finding/inventory filter: - semantics: include diff --git a/odg/profiles.yaml b/odg/profiles.yaml index abfaa7c0..2a9d1aa0 100644 --- a/odg/profiles.yaml +++ b/odg/profiles.yaml @@ -6,7 +6,6 @@ - finding/sast - finding/diki - finding/crypto - - os_ids - finding/osid special_component_ids: - 03e237b4-434e-4e1c-b786-5ceb6cc76c1c diff --git a/paths.py b/paths.py index ce186663..9934117e 100644 --- a/paths.py +++ b/paths.py @@ -23,9 +23,6 @@ 'test/resources/gardener_org_members.yaml', ) -_compliance_summary_path = os.path.join(_own_dir, 'compliance_summary') -artefact_metadata_cfg = os.path.join(_compliance_summary_path, 'artefact_metadata_cfg.yaml') - _features_path = os.path.join(_own_dir, 'features') swagger_path = os.path.join(_own_dir, 'swagger', 'swagger.yaml') diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index bc641408..63d2fcbb 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -161,7 +161,6 @@ definitions: - finding/diki - finding/sast - finding/osid - - os_ids - rescorings - compliance/snapshots - meta/artefact_scan_info diff --git a/test/test_compliance_summary.py b/test/test_compliance_summary.py index 1e321562..14a94be8 100644 --- a/test/test_compliance_summary.py +++ b/test/test_compliance_summary.py @@ -1,10 +1,8 @@ import logging import pytest -import unittest.mock import ci.log import dso.model -import unixutil.model import compliance_summary as cs import odg.findings @@ -15,75 +13,6 @@ ci.log.configure_default_logging(stdout_level=logging.ERROR) -@pytest.fixture() -def eol_client(): - def cycles( - product: str, - absent_ok: bool = False, - ): - return [ - { - 'cycle': '9.99', - 'latest': '9.99.9', - 'eol': '9999-12-30', - }, - { - 'cycle': '3.11', - 'latest': '3.11.13', - 'eol': '2021-11-01', - }, - ] - - api_mock = unittest.mock.Mock() - api_mock.cycles = cycles - - return api_mock - - -@pytest.fixture() -def artefact_metadata_cfg_by_type(): - cfg_raw = { - 'artefactMetadataCfg': [ - { - 'type': 'os_ids', - 'categories': [ - 'compliance', - ], - 'severityMappings': [ - { - 'severityName': 'CRITICAL', - 'status': [ - 'isEol', - ], - }, - { - 'severityName': 'MEDIUM', - 'status': [ - 'updateAvailableForBranch', - ], - }, - { - 'severityName': 'UNKNOWN', - 'status': [ - 'emptyOsId', - 'noBranchInfo', - 'unableToCompareVersion', - ], - }, - { - 'severityName': 'CLEAN', - 'status': [ - 'greatestBranchVersion', - ], - }, - ] - }, - ] - } - - return cs.artefact_metadata_cfg_by_type(artefact_metadata_cfg=cfg_raw) - - @pytest.fixture def component_artefact_id() -> dso.model.ComponentArtefactId: return dso.model.ComponentArtefactId( @@ -218,106 +147,6 @@ async def test_malware(component_artefact_id): )).categorisation == 'BLOCKER' -@pytest.mark.asyncio -async def test_os_id( - eol_client, - artefact_metadata_cfg_by_type, - component_artefact_id, -): - type = odg.findings.FindingType.OS_IDS - meta = dso.model.Metadata( - datasource=None, - type=type, - ) - - finding_cfg = odg.findings.Finding.from_file( - path=paths.findings_cfg_path(), - finding_type=type, - ) - - assert (await cs.calculate_summary_entry( - finding_cfg=finding_cfg, - findings=[dso.model.ArtefactMetadata( - artefact=component_artefact_id, - meta=meta, - data=dso.model.OsID( - os_info=unixutil.model.OperatingSystemId(), - ), - )], - rescorings=[], - artefact_metadata_cfg=artefact_metadata_cfg_by_type[type], - eol_client=eol_client, - )).categorisation == 'UNKNOWN' - - assert (await cs.calculate_summary_entry( - finding_cfg=finding_cfg, - findings=[dso.model.ArtefactMetadata( - artefact=component_artefact_id, - meta=meta, - data=dso.model.OsID( - os_info=unixutil.model.OperatingSystemId( - VERSION_ID='9.99.1', - ID='fooOs', - ), - ), - )], - rescorings=[], - artefact_metadata_cfg=artefact_metadata_cfg_by_type[type], - eol_client=eol_client, - )).categorisation == 'MEDIUM' - - assert (await cs.calculate_summary_entry( - finding_cfg=finding_cfg, - findings=[dso.model.ArtefactMetadata( - artefact=component_artefact_id, - meta=meta, - data=dso.model.OsID( - os_info=unixutil.model.OperatingSystemId( - VERSION_ID='9.99.9', - ID='fooOs', - ), - ), - )], - rescorings=[], - artefact_metadata_cfg=artefact_metadata_cfg_by_type[type], - eol_client=eol_client, - )).categorisation is cs.ComplianceEntryCategorisation.CLEAN - - assert (await cs.calculate_summary_entry( - finding_cfg=finding_cfg, - findings=[dso.model.ArtefactMetadata( - artefact=component_artefact_id, - meta=meta, - data=dso.model.OsID( - os_info=unixutil.model.OperatingSystemId( - VERSION_ID='3.11.5', - ID='fooOs', - ), - ), - )], - rescorings=[], - artefact_metadata_cfg=artefact_metadata_cfg_by_type[type], - eol_client=eol_client, - )).categorisation == 'CRITICAL' - - assert (await cs.calculate_summary_entry( - finding_cfg=finding_cfg, - findings=[dso.model.ArtefactMetadata( - artefact=component_artefact_id, - meta=meta, - data=dso.model.OsID( - os_info=unixutil.model.OperatingSystemId( - VERSION_ID='bar--foo', - ID='fooOs', - ), - ), - )], - rescorings=[], - artefact_metadata_cfg=artefact_metadata_cfg_by_type[type], - eol_client=eol_client, - )).categorisation == 'UNKNOWN' - - @pytest.mark.asyncio async def test_licenses(component_artefact_id): meta = dso.model.Metadata( From 1d3ce84fee6e26f59ab4493e5445cb70e49a9228 Mon Sep 17 00:00:00 2001 From: TuanAnh17N Date: Fri, 25 Apr 2025 14:07:39 +0200 Subject: [PATCH 2/4] remove unused route --- app.py | 5 -- components.py | 175 -------------------------------------------------- 2 files changed, 180 deletions(-) diff --git a/app.py b/app.py index 7da03c85..bbec1ccf 100755 --- a/app.py +++ b/app.py @@ -285,11 +285,6 @@ def add_routes( handler=components.ComplianceSummary, ) - app.router.add_view( - path='/components/metadata', - handler=components.ComponentMetadata, - ) - app.router.add_view( path='/delivery/sprint-infos', handler=sprint.SprintInfos, diff --git a/components.py b/components.py index 4bfe75ea..165f14ec 100644 --- a/components.py +++ b/components.py @@ -1352,178 +1352,3 @@ async def get(self): }, dumps=util.dict_to_json_factory, ) - - -class Select(enum.Enum): - GREATEST = 'greatestVersion' - LATEST = 'latestData' - - -class ComponentMetadata(aiohttp.web.View): - required_features = (features.FeatureDeliveryDB,) - - def _latest_metadata_query( - self, - metadata_types: collections.abc.Iterable[str], - component_version: str | None, - component_name: str, - ) -> sq.Query: - # The following is a nested query. First, in the subquery, the relevant entries in the table - # are partitioned by metadata, sorted, and a rank-column is assigned to each row in the - # resulting partitions. - # Then, the actual query returns the rows where rank equals '1' from the subquery. - subquery = sa.select( - dm.ArtefactMetaData, - sa.func.rank().over( - order_by=dm.ArtefactMetaData.creation_date.desc(), - partition_by=dm.ArtefactMetaData.type, - ).label('rank') - ).where( - sa.and_( - dm.ArtefactMetaData.component_name == component_name, - ( - # if no version is given, set this predicate to 'True' to avoid filtering - dm.ArtefactMetaData.component_version == component_version - if component_version else True - ), - # similarly, return results for all metadata types if no collection is given - dm.ArtefactMetaData.type.in_(metadata_types) if metadata_types else True, - ) - ).subquery() - return sa.select(subquery).where(subquery.c.rank == 1) - - def _metadata_query_for_version( - self, - metadata_types: collections.abc.Iterable[str], - component_version: str, - component_name: str, - ) -> sq.Query: - subquery = sa.select(dm.ArtefactMetaData).where( - sa.and_( - dm.ArtefactMetaData.component_name == component_name, - # return results for all metadata types if no collection (or an empty one) is given - dm.ArtefactMetaData.type.in_(metadata_types) if metadata_types else True, - dm.ArtefactMetaData.component_version == component_version, - ), - ).subquery() - return sa.select(subquery) - - async def get(self): - ''' - --- - description: - Returns a list of artefact-metadata for the given component with optional filters. One of - `select` and `version` must be given. However, if `select` is given as `greatestVersion`, - `version` must _not_ be given. - tags: - - Artefact metadata - produces: - - application/json - parameters: - - in: query - name: name - type: string - required: true - - in: query - name: version - type: string - required: false - description: The component version to consider. - - in: query - name: type - schema: - $ref: '#/definitions/Datatype' - required: false - description: - The metadata-types to retrieve. Can be given multiple times. If no type is given, all - relevant metadata will be returned. - - in: query - name: select - type: string - enum: - - greatestVersion - - latestDat - required: false - - in: query - name: version_filter - type: string - enum: - - all - - releases_only - required: false - responses: - "200": - description: Successful operation. - schema: - type: array - items: - $ref: '#/definitions/ArtefactMetadata' - ''' - params = self.request.rel_url.query - invalid_semver_ok = self.request.app[consts.APP_INVALID_SEMVER_OK] - version_filter_callback = self.request.app[consts.APP_VERSION_FILTER_CALLBACK] - version_lookup = self.request.app[consts.APP_VERSION_LOOKUP] - - component_name = util.param(params, 'name', required=True) - component_version = util.param(params, 'version') - - version_filter = util.param(params, 'version_filter', default=version_filter_callback()) - util.get_enum_value_or_raise(version_filter, config.VersionFilter) - - data_types = params.getall('type', default=[]) - select = util.param(params, 'select') - - if select: - try: - select = Select(select) - except ValueError: - raise aiohttp.web.HTTPBadRequest( - reason='Invalid parameter', - text=( - 'The value of the parameter select must be a one of ' - f'{[m.value for m in Select]}' - ), - ) - - if not component_version and not select: - raise aiohttp.web.HTTPBadRequest(text='One of "version" and "select" must be given.') - - if component_version and select and select is Select.GREATEST: - raise aiohttp.web.HTTPBadRequest( - text=f'"select" must not be "{Select.GREATEST.value}" if "version" is given', - ) - - db_session: sqlasync.session.AsyncSession = self.request.get(consts.REQUEST_DB_SESSION) - - if select is Select.GREATEST: - component_version = await greatest_component_version( - component_name=component_name, - version_lookup=version_lookup, - version_filter=version_filter, - invalid_semver_ok=invalid_semver_ok, - db_session=db_session, - ) - - if component_version: - db_statement = self._metadata_query_for_version( - metadata_types=data_types, - component_version=component_version, - component_name=component_name, - ) - else: - db_statement = self._latest_metadata_query( - metadata_types=data_types, - component_version=component_version, - component_name=component_name, - ) - - db_stream = await db_session.stream(db_statement) - - return aiohttp.web.json_response( - data=[ - deliverydb.util.db_artefact_metadata_to_dict(result) - async for partition in db_stream.partitions(size=50) - for result in partition - ], - dumps=util.dict_to_json_factory, - ) From 2fa93fe3c8b7a12bdf316f2813a64b24fb900c82 Mon Sep 17 00:00:00 2001 From: TuanAnh17N Date: Fri, 9 May 2025 15:14:25 +0200 Subject: [PATCH 3/4] adjust usage of dataclass after mving it in cc-utils --- osid_extension/__main__.py | 9 ++++----- osid_extension/scan.py | 8 ++++---- osid_extension/util.py | 8 ++++---- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/osid_extension/__main__.py b/osid_extension/__main__.py index ffca5f1b..179bc0a2 100755 --- a/osid_extension/__main__.py +++ b/osid_extension/__main__.py @@ -19,7 +19,6 @@ import oci.model import ocm import tarutil -import unixutil.model as um import consts import cnudie.retrieve @@ -63,7 +62,7 @@ def handle_termination_signal(*args): def determine_os_status( - osid: um.OperatingSystemId, + osid: dso.model.OperatingSystemId, eol_client: eol.EolClient, ) -> tuple[dso.model.OsStatus, str | None, datetime.datetime | None]: ''' @@ -127,7 +126,7 @@ def determine_os_status( def determine_osid( resource: ocm.Resource, oci_client: oci.client.Client, -) -> um.OperatingSystemId | None: +) -> dso.model.OperatingSystemId | None: if resource.type != ocm.ArtefactType.OCI_IMAGE: return @@ -147,7 +146,7 @@ def determine_osid( def base_image_osid( oci_client: oci.client.Client, resource: ocm.Resource, -) -> um.OperatingSystemId: +) -> dso.model.OperatingSystemId: image_reference = resource.access.imageReference manifest = oci_client.manifest( @@ -182,7 +181,7 @@ def base_image_osid( def create_artefact_metadata( artefact: dso.model.ComponentArtefactId, osid_finding_config: odg.findings.Finding, - osid: um.OperatingSystemId | None, + osid: dso.model.OperatingSystemId | None, eol_client: eol.EolClient, relation: ocm.ResourceRelation, time_now: datetime.datetime | None = None, diff --git a/osid_extension/scan.py b/osid_extension/scan.py index 4e7c2663..f9027f3c 100644 --- a/osid_extension/scan.py +++ b/osid_extension/scan.py @@ -3,9 +3,9 @@ import dacite -import version +import dso.model -import unixutil.model as um +import version def _parse_os_release( @@ -62,7 +62,7 @@ def _parse_debian_version( def determine_osinfo( tarfh: tarfile.TarFile -) -> um.OperatingSystemId | None: +) -> dso.model.OperatingSystemId | None: ''' tries to determine the operating system identification, roughly as specified by https://www.freedesktop.org/software/systemd/man/os-release.html @@ -133,6 +133,6 @@ def determine_osinfo( return None return dacite.from_dict( - data_class=um.OperatingSystemId, + data_class=dso.model.OperatingSystemId, data=os_info, ) diff --git a/osid_extension/util.py b/osid_extension/util.py index d5a45ee3..731f1ca8 100644 --- a/osid_extension/util.py +++ b/osid_extension/util.py @@ -2,7 +2,7 @@ import awesomeversion -import unixutil.model as um +import dso.model import osinfo.model @@ -11,7 +11,7 @@ def find_branch_info( - osid: um.OperatingSystemId, + osid: dso.model.OperatingSystemId, os_infos: list[osinfo.model.OsReleaseInfo], ) -> osinfo.model.OsReleaseInfo | None: os_version = osid.VERSION_ID @@ -42,7 +42,7 @@ def version_candidates(): def branch_reached_eol( - osid: um.OperatingSystemId, + osid: dso.model.OperatingSystemId, os_infos: list[osinfo.model.OsReleaseInfo], ) -> bool: if not osid.ID: @@ -57,7 +57,7 @@ def branch_reached_eol( def update_available( - osid: um.OperatingSystemId, + osid: dso.model.OperatingSystemId, os_infos: list[osinfo.model.OsReleaseInfo], ignore_if_patchlevel_is_next_to_greatest=False, ) -> bool: From 79b32144e9274d9879a5f9c45aa5228441a04d3a Mon Sep 17 00:00:00 2001 From: TuanAnh17N Date: Fri, 9 May 2025 15:17:13 +0200 Subject: [PATCH 4/4] fix: drop unused eolclient argument from method calls --- test/test_compliance_summary.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/test_compliance_summary.py b/test/test_compliance_summary.py index 14a94be8..b89294b6 100644 --- a/test/test_compliance_summary.py +++ b/test/test_compliance_summary.py @@ -59,7 +59,6 @@ async def test_vulnerability(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation is cs.ComplianceEntryCategorisation.CLEAN assert (await cs.calculate_summary_entry( @@ -82,7 +81,6 @@ async def test_vulnerability(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation == 'CRITICAL' @@ -119,7 +117,6 @@ async def test_malware(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation is cs.ComplianceEntryCategorisation.CLEAN assert (await cs.calculate_summary_entry( @@ -143,7 +140,6 @@ async def test_malware(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation == 'BLOCKER' @@ -176,7 +172,6 @@ async def test_licenses(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation is cs.ComplianceEntryCategorisation.CLEAN assert (await cs.calculate_summary_entry( @@ -196,5 +191,4 @@ async def test_licenses(component_artefact_id): ), )], rescorings=[], - eol_client=None, )).categorisation == 'BLOCKER'