From b918021ad7db114d90a8bb46c5641867943f329e Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Wed, 17 Apr 2024 15:46:17 +0200 Subject: [PATCH] add squad-compare-compat This script shows the differences between compat and non compat between build_name '*-compat' and replace('-compat', ''). Example output: non-compat | compat | build_version/device/build_name/testsuite/test -------------------------------------------------------------------- fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/log-parser-test/check-kernel-oops fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs/fs_fill fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs_bind/fs_bind05_sh fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs_bind/fs_bind13_sh pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap02 pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap13 pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap14 Signed-off-by: Anders Roxell --- squad-compare-compat | 144 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 squad-compare-compat diff --git a/squad-compare-compat b/squad-compare-compat new file mode 100755 index 0000000..dde12f8 --- /dev/null +++ b/squad-compare-compat @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# vim: set ts=4 +# +# Copyright 2024-present Linaro Limited +# +# SPDX-License-Identifier: MIT + + +import argparse +import logging +import os +import sys +import yaml +from urllib import request +from squad_client.core.api import SquadApi +from squad_client.core.models import Squad +from squad_client.shortcuts import download_tests as download +from squad_client.shortcuts import get_build + +squad_host_url = "https://qa-reports.linaro.org/" +SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url)) + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger() + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compare builds within SQUAD") + + parser.add_argument( + "--gpb", + required=True, + action="append", + nargs=3, + help="squad group,project,build", + ) + + parser.add_argument( + "--environments", help="Filter on environments (separated by ',')" + ) + + parser.add_argument( + "--suites", help="Filter on suites (separated by ',')" + ) + + parser.add_argument( + "--filename", help="Name of the output file where results will be written" + ) + + parser.add_argument( + "--debug", + action="store_true", + default=False, + help="Display debug messages", + ) + + return parser.parse_args() + + +def download_tests(project, build, environments, suites, output_filename): + download(project, build, environments, suites, "{test.environment.slug}/{test.test_run.metadata.build_name}/{test.name} {test.status}", output_filename,) + + +def run(): + args = parse_args() + if args.debug: + logger.setLevel(level=logging.DEBUG) + + known_issue_files = "kselftests-production.yaml kvm-unit-tests.yaml libhugetlbfs-production.yaml ltp-production.yaml network-basic-tests.yaml packetdrill-tests.yaml perf.yaml spectre-meltdown-checker.yaml v4l2-compliance.yaml" + known_issues_patterns = [] + group_name, project_name, build_name = args.gpb[0] + for file in known_issue_files.split(): + + x = request.urlopen(f'https://raw.githubusercontent.com/Linaro/qa-reports-known-issues/master/{file}') + issues = yaml.safe_load(x) + for tmp in issues["projects"][0]["known_issues"]: + environments = [] + projects = [] + for key, value in tmp.items(): + if 'environments' in tmp.keys(): + environments = tmp["environments"] + else: + for matrix in tmp["matrix_apply"]: + environments.append(matrix["environments"]) + + if 'projects' in tmp.keys(): + projects = tmp["projects"] + else: + for matrix in tmp["matrix_apply"]: + projects.append(matrix["projects"]) + if "test_names" in key: + for argh in value: + if f'{group_name}/{project_name}' in projects: + for env in environments: + known_issues_patterns.append(f"{env}/*{argh.replace('.', '?')}") + elif "test_name" in key: + if f'{group_name}/{project_name}' in projects: + for env in environments: + known_issues_patterns.append(f"{env}/*{value.replace('.', '?')}") + + table_filename = 'table-compat.txt' + headings = "non-compat | compat | build_version/device/build_name/testsuite/test\n" + headings = f"{headings}--------------------------------------------------------------------\n" + with open(table_filename, 'w') as fp: + fp.write(headings) + for (group_name, project_name, build_name) in args.gpb: + group = Squad().group(group_name) + project = group.project(project_name) + build = get_build(build_name, project) + + environments = None + if args.environments: + environments = [project.environment(e) for e in args.environments.split(",")] + + suites = None + if args.suites: + suites = [] + for s in args.suites.split(","): + suites += project.suites(slug__startswith=s).values() + + file = f"{group.slug}-{project.slug}-{build.version}".replace('~', '') + file_with_ending = os.path.join(file + '.txt') + download_tests(project, build, environments, suites, file_with_ending) + file_open = open(file_with_ending, 'r') + file_lines = file_open.readlines() + + compatlist = {} + for line in file_lines: + test_name, test_result = line.split() + build_name = test_name.split("/")[1] + if build_name.endswith("-compat"): + compatlist[test_name.replace("-compat", "")] = test_result + for line in file_lines: + test_name, test_result = line.split() + if test_name in compatlist.keys(): + if test_result != compatlist[test_name]: + fp.write(f"{test_result} | {compatlist[test_name]} : {build.version}/{test_name}\n") + + logger.debug(f"group: {group}, project: {project}, build: {build}") + + +if __name__ == "__main__": + sys.exit(run())