python: Add a simple TuxMake wrapper for kernel.org toolchains

Signed-off-by: Nathan Chancellor <nathan@kernel.org>

Author: nathanchance

fish/functions/tuxmake_bld_all.fish

@@ -0,0 +1,7 @@
+#!/usr/bin/env fish
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2023 Nathan Chancellor
+
+function tuxmake_bld_all -d "Wrapper for tuxmake_bld_all.py"
+    $PYTHON_SCRIPTS_FOLDER/tuxmake_bld_all.py $argv
+end

python/.gitignore

@@ -1 +1,2 @@
 __pycache__/
+tuxmake/

python/lib/utils.py

@@ -16,10 +16,36 @@ def print_cmd(command):
     print(f"$ {' '.join([shlex.quote(str(elem)) for elem in command])}")
 
 
+def print_duration(seconds):
+    days, seconds = divmod(int(seconds), 60 * 60 * 24)
+    hours, seconds = divmod(seconds, 60 * 60)
+    minutes, seconds = divmod(seconds, 60)
+
+    parts = []
+    if days:
+        parts += [f"{days}d"]
+    if hours:
+        parts += [f"{hours}h"]
+    if minutes:
+        parts += [f"{minutes}m"]
+    parts += [f"{seconds}s"]
+
+    return f"{' '.join(parts)}"
+
+
+def print_header(string):
+    border = ''.join(["=" for _ in range(0, len(string) + 6)])
+    print_cyan(f"\n{border}\n== {string} ==\n{border}\n")
+
+
 def print_color(color, string):
     print(f"{color}{string}\033[0m", flush=True)
 
 
+def print_cyan(msg):
+    print_color('\033[01;36m', msg)
+
+
 def print_green(msg):
     print_color('\033[01;32m', msg)
 

python/scripts/korg_gcc.py

@@ -26,9 +26,14 @@ def korg_gcc_canonicalize_target(value):
     suffix = ''
     if value == 'arm':
         suffix = '-gnueabi'
-    elif value == 'arm64':  # in case the kernel ARCH value is passed in
-        value = 'aarch64'
-    return f"{value}-linux{suffix}"
+
+    # in case the kernel ARCH value is passed in, we make an educated guess as
+    # to what the user intended
+    kernel_to_gcc = {
+        'arm64': 'aarch64',
+        'riscv': 'riscv64',
+    }
+    return f"{kernel_to_gcc.get(value, value)}-linux{suffix}"
 
 
 def supported_korg_gcc_arches():

python/scripts/tuxmake_bld_all.py

@@ -0,0 +1,206 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Copyright (C) 2023 Nathan Chancellor
+
+from argparse import ArgumentParser
+from pathlib import Path
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+import korg_gcc
+
+# Until https://gitlab.com/Linaro/tuxmake/-/merge_requests/283 is merged and released
+if not (tuxmake_src := Path(__file__).resolve().parents[1].joinpath('tuxmake')).exists():
+    git_clone_cmd = [
+        'git', 'clone', '-b', 'allow-override-make-vars', 'https://gitlab.com/Linaro/tuxmake.git',
+        tuxmake_src
+    ]
+    subprocess.run(git_clone_cmd, capture_output=True, check=True)
+
+sys.path.insert(0, str(tuxmake_src.parent))
+# pylint: disable-next=wrong-import-position
+import lib.utils  # noqa: E402
+
+sys.path.insert(0, str(tuxmake_src))
+# pylint: disable-next=import-error,no-name-in-module,wrong-import-position
+import tuxmake.build  # noqa: E402
+
+
+def interrupt_handler(_signum, _frame):
+    sys.exit(130)
+
+
+def parse_arguments():
+    parser = ArgumentParser(description='Do a series of builds with GCC from kernel.org')
+
+    supported_architectures = [
+        'arm',
+        'arm64',
+        'i386',
+        'mips',
+        'powerpc',
+        's390',
+        'riscv',
+        'x86_64',
+    ]
+    parser.add_argument('-a',
+                        '--architectures',
+                        choices=supported_architectures,
+                        default=supported_architectures,
+                        help='Architectures to build (default: all supported architectures)',
+                        metavar='TARGETS',
+                        nargs='+')
+
+    suppported_toolchains = [f"gcc-{ver}" for ver in korg_gcc.supported_korg_gcc_versions()]
+    parser.add_argument('-t',
+                        '--toolchains',
+                        choices=suppported_toolchains,
+                        default=[suppported_toolchains[-1]],
+                        help='Toolchains to build kernels with (default: latest GCC release)',
+                        metavar='TOOLCHAINS',
+                        nargs='+')
+
+    supported_targets = ['def', 'all']
+    parser.add_argument('-T',
+                        '--targets',
+                        choices=supported_targets,
+                        default=supported_targets,
+                        help='Targets to build (default: all supported targets)',
+                        metavar='TARGETS',
+                        nargs='+')
+
+    default_kernel_source = Path('.').resolve()
+    parser.add_argument('-C',
+                        '--directory',
+                        default=default_kernel_source,
+                        help='Kernel source to build (default: current working directory)',
+                        metavar='SOURCE')
+
+    parser.add_argument(
+        '-o',
+        '--output-dir',
+        help='Output folder for build artifacts (default: build folder in kernel source)')
+
+    parser.add_argument('--use-ccache',
+                        action='store_true',
+                        help='Use ccache for builds (default: no caching)')
+
+    return parser.parse_args()
+
+
+def get_kconfigs_for_target(targets):
+    kconfigs = {
+        'all': ['allmodconfig', 'allnoconfig'],  # allyesconfig is not super useful and slow
+        'def': ['defconfig'],
+    }
+
+    return [kconfig for target in targets for kconfig in kconfigs[target]]
+
+
+def get_make_variables(target_arch, toolchain):
+    make_variables = {}
+
+    if 'gcc' in toolchain:
+        version = int(toolchain.split('-')[1])
+        make_variables['CROSS_COMPILE'] = korg_gcc.get_gcc_cross_compile(version, target_arch)
+        if target_arch == 'arm64':
+            make_variables['CROSS_COMPILE_COMPAT'] = korg_gcc.get_gcc_cross_compile(version, 'arm')
+
+    return make_variables
+
+
+def get_targets(kconfig):
+    targets = ['default']
+    if kconfig == 'defconfig':
+        targets += ['kernel']
+    return targets
+
+
+def build_one(tree, output_dir, target_arch, toolchain, wrapper, kconfig, results_file):
+    bld_str = f"ARCH={target_arch} {kconfig} {toolchain}"
+
+    lib.utils.print_header(bld_str)
+
+    config_output_dir = f"{output_dir}/{target_arch}/{kconfig}"
+    Path(config_output_dir).mkdir(exist_ok=True, parents=True)
+
+    # pylint: disable-next=c-extension-no-member
+    result = tuxmake.build.build(tree=tree,
+                                 output_dir=config_output_dir,
+                                 target_arch=target_arch,
+                                 wrapper=wrapper,
+                                 kconfig=kconfig,
+                                 make_variables=get_make_variables(target_arch, toolchain),
+                                 targets=get_targets(kconfig))
+
+    duration = 0
+    passed = True
+    for info in result.status.values():
+        duration += info.duration
+        passed &= info.passed
+
+    res_str = 'PASS' if passed else 'FAIL'
+    duration_str = lib.utils.print_duration(duration)
+    with results_file.open(encoding='utf-8', mode='a') as file:
+        file.write(f"{bld_str}: {res_str} in {duration_str}\n")
+
+
+def process_results(results_file, start_time):
+    print()
+
+    failed = []
+    passed = []
+    for line in results_file.read_text(encoding='utf-8').splitlines(keepends=True):
+        (passed if 'PASS' in line else failed).append(line)
+
+    if passed:
+        print('Successful builds:\n')
+        print(''.join(passed))
+
+    if failed:
+        print('Failed builds:\n')
+        print(''.join(failed))
+
+    print(f"Total build time: {lib.utils.print_duration(time.time() - start_time)}")
+
+
+def build_all(linux_folder, out_folder, architectures, targets, toolchains, use_ccache,
+              results_file):
+    for toolchain in toolchains:
+        for target_arch in architectures:
+            for kconfig in get_kconfigs_for_target(targets):
+                build_one(tree=linux_folder,
+                          output_dir=out_folder,
+                          target_arch=target_arch,
+                          toolchain=toolchain,
+                          wrapper='ccache' if use_ccache and shutil.which('ccache') else None,
+                          kconfig=kconfig,
+                          results_file=results_file)
+
+
+if __name__ == '__main__':
+    signal.signal(signal.SIGINT, interrupt_handler)
+
+    args = parse_arguments()
+
+    if not (output := args.output_dir):
+        output = Path(args.directory, 'build')
+
+    if (output := Path(output).resolve()).exists():
+        shutil.rmtree(output)
+
+    results = Path(output, 'results.log')
+    start = time.time()
+
+    build_all(linux_folder=Path(args.directory).resolve(),
+              out_folder=output,
+              architectures=args.architectures,
+              targets=args.targets,
+              toolchains=args.toolchains,
+              use_ccache=args.use_ccache,
+              results_file=results)
+
+    process_results(results, start)

requirements.txt

@@ -1 +1,2 @@
 requests
+tuxmake