add squad-compare-compat

This script shows the differences between compat and non compat between
build_name '*-compat' and replace('-compat', '').

Example output:
non-compat | compat | build_version/device/build_name/testsuite/test
--------------------------------------------------------------------
fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/log-parser-test/check-kernel-oops
fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs/fs_fill
fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs_bind/fs_bind05_sh
fail | pass : next-20240403/e850-96/gcc-13-lkftconfig/ltp-fs_bind/fs_bind13_sh
pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap02
pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap13
pass | skip : next-20240403/e850-96/gcc-13-lkftconfig/ltp-hugetlb/hugemmap14

Signed-off-by: Anders Roxell <[email protected]>

squad-compare-compat

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+# vim: set ts=4
+#
+# Copyright 2024-present Linaro Limited
+#
+# SPDX-License-Identifier: MIT
+
+
+import argparse
+import logging
+import os
+import sys
+import yaml
+from urllib import request
+from squad_client.core.api import SquadApi
+from squad_client.core.models import Squad
+from squad_client.shortcuts import download_tests as download
+from squad_client.shortcuts import get_build
+
+squad_host_url = "https://qa-reports.linaro.org/"
+SquadApi.configure(cache=3600, url=os.getenv("SQUAD_HOST", squad_host_url))
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Compare builds within SQUAD")
+
+    parser.add_argument(
+        "--gpb",
+        required=True,
+        action="append",
+        nargs=3,
+        help="squad group,project,build",
+    )
+
+    parser.add_argument(
+        "--environments", help="Filter on environments (separated by ',')"
+    )
+
+    parser.add_argument(
+        "--suites", help="Filter on suites (separated by ',')"
+    )
+
+    parser.add_argument(
+        "--filename", help="Name of the output file where results will be written"
+    )
+
+    parser.add_argument(
+        "--debug",
+        action="store_true",
+        default=False,
+        help="Display debug messages",
+    )
+
+    return parser.parse_args()
+
+
+def download_tests(project, build, environments, suites, output_filename):
+    download(project, build, environments, suites, "{test.environment.slug}/{test.test_run.metadata.build_name}/{test.name} {test.status}", output_filename,)
+
+
+def run():
+    args = parse_args()
+    if args.debug:
+        logger.setLevel(level=logging.DEBUG)
+
+    known_issue_files = "kselftests-production.yaml kvm-unit-tests.yaml libhugetlbfs-production.yaml ltp-production.yaml network-basic-tests.yaml packetdrill-tests.yaml perf.yaml spectre-meltdown-checker.yaml v4l2-compliance.yaml"
+    known_issues_patterns = []
+    group_name, project_name, build_name = args.gpb[0]
+    for file in known_issue_files.split():
+
+        x = request.urlopen(f'https://raw.githubusercontent.com/Linaro/qa-reports-known-issues/master/{file}')
+        issues = yaml.safe_load(x)
+        for tmp in issues["projects"][0]["known_issues"]:
+            environments = []
+            projects = []
+            for key, value in tmp.items():
+                if 'environments' in tmp.keys():
+                    environments = tmp["environments"]
+                else:
+                    for matrix in tmp["matrix_apply"]:
+                        environments.append(matrix["environments"])
+
+                if 'projects' in tmp.keys():
+                    projects = tmp["projects"]
+                else:
+                    for matrix in tmp["matrix_apply"]:
+                        projects.append(matrix["projects"])
+                if "test_names" in key:
+                    for argh in value:
+                        if f'{group_name}/{project_name}' in projects:
+                            for env in environments:
+                                known_issues_patterns.append(f"{env}/*{argh.replace('.', '?')}")
+                elif "test_name" in key:
+                    if f'{group_name}/{project_name}' in projects:
+                        for env in environments:
+                            known_issues_patterns.append(f"{env}/*{value.replace('.', '?')}")
+
+    table_filename = 'table-compat.txt'
+    headings = "non-compat | compat | build_version/device/build_name/testsuite/test\n"
+    headings = f"{headings}--------------------------------------------------------------------\n"
+    with open(table_filename, 'w') as fp:
+        fp.write(headings)
+        for (group_name, project_name, build_name) in args.gpb:
+            group = Squad().group(group_name)
+            project = group.project(project_name)
+            build = get_build(build_name, project)
+
+            environments = None
+            if args.environments:
+                environments = [project.environment(e) for e in args.environments.split(",")]
+
+            suites = None
+            if args.suites:
+                suites = []
+                for s in args.suites.split(","):
+                    suites += project.suites(slug__startswith=s).values()
+
+            file = f"{group.slug}-{project.slug}-{build.version}".replace('~', '')
+            file_with_ending = os.path.join(file + '.txt')
+            download_tests(project, build, environments, suites, file_with_ending)
+            file_open = open(file_with_ending, 'r')
+            file_lines = file_open.readlines()
+
+            compatlist = {}
+            for line in file_lines:
+                test_name, test_result = line.split()
+                build_name = test_name.split("/")[1]
+                if build_name.endswith("-compat"):
+                    compatlist[test_name.replace("-compat", "")] = test_result
+            for line in file_lines:
+                test_name, test_result = line.split()
+                if test_name in compatlist.keys():
+                    if test_result != compatlist[test_name]:
+                        fp.write(f"{test_result} | {compatlist[test_name]} : {build.version}/{test_name}\n")
+
+        logger.debug(f"group: {group}, project: {project}, build: {build}")
+
+
+if __name__ == "__main__":
+    sys.exit(run())