diff --git a/.github/actions/bake-test-push/action.yml b/.github/actions/bake-test-push/action.yml index 1660da06..95c31e74 100644 --- a/.github/actions/bake-test-push/action.yml +++ b/.github/actions/bake-test-push/action.yml @@ -97,9 +97,6 @@ runs: id: build uses: docker/bake-action@v4 with: - set: | - *.cache-from=type=gha - *.cache-to=type=gha,compression=zstd targets: "${{ inputs.target }}" push: false @@ -111,8 +108,5 @@ runs: - name: Push - ${{ inputs.push-image }} uses: docker/bake-action@v4 with: - set: | - *.cache-from=type=gha - *.cache-to=type=gha,compression=zstd targets: "${{ inputs.target }}" push: ${{ inputs.push-image }} diff --git a/.github/workflows/build-bake.yaml b/.github/workflows/build-bake.yaml index 4c49a7b1..d77c4327 100644 --- a/.github/workflows/build-bake.yaml +++ b/.github/workflows/build-bake.yaml @@ -25,6 +25,15 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 id: setup-buildx @@ -41,6 +50,16 @@ jobs: dockerhub-token: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' + - name: Export artifacts + run: | + just export-artifacts base-images + + - name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: base-images + path: ./.out + connect: needs: [base] name: Connect @@ -63,6 +82,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: @@ -96,6 +134,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: @@ -161,6 +218,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: @@ -194,6 +270,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: @@ -227,6 +322,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: @@ -239,7 +353,6 @@ jobs: gcp-json: '${{ secrets.GCP_ARTIFACT_REGISTRY_JSON }}' workbench-for-google-cloud-workstations: - needs: [base] name: Workbench for Google Cloud Workstations runs-on: ubuntu-latest-4x @@ -293,6 +406,25 @@ jobs: with: buildkitd-config: ./share/buildkitd.toml + - name: Set up Just + uses: extractions/setup-just@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download artifacts + uses: actions/download-artifact@v2 + with: + name: base-images + path: ./.out + + - name: Import artifacts + run: | + just import-artifacts + - name: Build, Test, and Push uses: ./.github/actions/bake-test-push with: diff --git a/Justfile b/Justfile index e49a6742..a0f98b58 100644 --- a/Justfile +++ b/Justfile @@ -136,6 +136,14 @@ run product tag="": -e RSPM_VERSION="${RSPM_VERSION}" \ {{product}} +# Export/import targets + +export-artifacts target build_definition="docker-bake.hcl": + python3 {{justfile_directory()}}/tools/export_bake_artifacts.py --target "{{target}}" --file "{{build_definition}}" + +import-artifacts: + python3 {{justfile_directory()}}/tools/import_bake_artifacts.py + # Helper targets # just _get-tag-safe-version 2022.07.2+576.pro12 diff --git a/tools/export_bake_artifacts.py b/tools/export_bake_artifacts.py new file mode 100644 index 00000000..05fb0a44 --- /dev/null +++ b/tools/export_bake_artifacts.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +""" +Export bake artifacts as tar files for later reuse. + +./export_bake_artifacts.py --file --target --output-path +""" + +import argparse +import json +import subprocess +from pathlib import Path + +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Export one or more bake artifacts to tar files for reuse" +) +parser.add_argument("--file", default="docker-bake.hcl") +parser.add_argument("--target", default="default") +parser.add_argument("--output-path", default=PROJECT_DIR / ".out") + + +def get_bake_plan(bake_file="docker-bake.hcl", target="default"): + cmd = ["docker", "buildx", "bake", "-f", str(PROJECT_DIR / bake_file), "--print", target] + p = subprocess.run(cmd, capture_output=True) + if p.returncode != 0: + print(f"Failed to get bake plan: {p.stderr}") + exit(1) + return json.loads(p.stdout.decode("utf-8")) + + +def build_export_command(target_name, target_spec, output_path): + output_file = Path(output_path) / f"{target_name}.tar" + cmd = [ + "docker", + "image", + "save", + "--output", + f"{output_file}", + " ".join(target_spec["tags"]), + ] + return cmd + + +def run_cmd(target_name, cmd): + p = subprocess.run(" ".join(cmd), shell=True) + if p.returncode != 0: + print(f"{target_name} failed to export: {p.returncode}") + return p.returncode + + +def main(): + args = parser.parse_args() + plan = get_bake_plan(args.file, args.target) + output = args.output_path + if not Path(output).exists(): + Path(output).mkdir(parents=True) + print(f"Exporting {len(plan['target'].keys())} targets: {plan['target'].keys()}") + for target_name, target_spec in plan["target"].items(): + print(f"Exporting {target_name}") + cmd = build_export_command(target_name, target_spec, output) + run_cmd(target_name, cmd) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/tools/import_bake_artifacts.py b/tools/import_bake_artifacts.py new file mode 100644 index 00000000..6f3c026c --- /dev/null +++ b/tools/import_bake_artifacts.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +""" +Run tests against bake artifacts by group/target and build definition. + +./test_bake_artifacts.py --file --target +""" + +import argparse +import json +import subprocess +from pathlib import Path + +PROJECT_DIR = Path(__file__).resolve().parents[1] + + +parser = argparse.ArgumentParser( + description="Import one or more bake artifacts from tar files for reuse" +) +parser.add_argument("--archive-path", type=Path, default=PROJECT_DIR / ".out") + + +def main(): + args = parser.parse_args() + if args.archive_path: + for archive in args.archive_path.glob("*.tar"): + print(f"Importing {archive}") + cmd = ["docker", "image", "load", "--input", archive] + p = subprocess.run(cmd) + if p.returncode != 0: + print(f"Failed to import {archive}: {p.returncode}") + + +if __name__ == "__main__": + main() diff --git a/tools/test_bake_artifacts.py b/tools/test_bake_artifacts.py index dd05a691..dd2834c1 100644 --- a/tools/test_bake_artifacts.py +++ b/tools/test_bake_artifacts.py @@ -1,3 +1,10 @@ +#!/usr/bin/env python3 +""" +Run tests against bake artifacts by group/target and build definition. + +./test_bake_artifacts.py --file --target +""" + import argparse import json import re