Use selfhosted S3 build cache

.github/actions/cache/action.yml

@@ -0,0 +1,138 @@
+
+name: Setup build cache
+description: Setup build cache using a self-hosted S3 instance if in our repo, or github cache if on a fork.
+inputs:
+  key:
+    description: The cache base key (job name by default). Gets suffixed by the current git ref
+    default: "${{github.job}}"
+  path:
+    description: The cache path.
+  s3-user:
+    description: Username of self-hosted s3
+  s3-password:
+    description: Password of self-hosted s3
+  s3-user-readonly:
+    description: Readonly username of self-hosted s3
+  s3-password-readonly:
+    description: Readonly password of self-hosted s3
+  s3-host:
+    description: Host of self-hosted s3
+  s3-bucket:
+    description: Bucket of self-hosted s3
+
+runs:
+  using: "composite"
+  steps:
+    # ==================================================================================================================
+    # S3 cache setup
+    # ==================================================================================================================
+    #
+    # Cache setup for our own repo
+    # Stores the cache on a self-hosted S3 instance based on cache name and suffixed by the git ref
+    # example: editor_debug_linux_x86_64-refs_heads_master
+
+    - name: Prepare GitHub ref
+      if: ${{ inputs.s3-host != '' }}
+      id: prepare_ref
+      uses: actions/[email protected]
+      with:
+        script: |
+          let branch = '';
+          // Check if it is a PR
+          if (process.env.GITHUB_HEAD_REF) {
+            branch = process.env.GITHUB_HEAD_REF.replace(/\//g, '_'); // Use head ref for PRs
+          } else {
+            branch = process.env.GITHUB_REF.split('/').slice(2).join('_'); // Standard ref
+          }
+          core.exportVariable('PREPARED_REF', branch);
+
+    - name: Setup s3 cache
+      id: cache_setup
+      if: ${{ inputs.s3-host != '' }}
+      uses: philiplehmann/[email protected]
+      with:
+        endpoint: ${{inputs.s3-host}} # optional, default s3.amazonaws.com
+        port: 443 # minio port
+        insecure: false # optional, use http instead of https. default false
+        accessKey: ${{inputs.s3-user}} # required
+        secretKey: ${{inputs.s3-password}} # required
+        bucket: ${{inputs.s3-bucket}} # required
+        key: ${{ inputs.key }}-${{ env.PREPARED_REF }}
+        path: ${{ inputs.path }}
+
+    - name: Use master cache if specific cache was missed
+      # if we got no cache hit, this means we do not yet have a cache for this git ref
+      # hence we restore the cache from master with readonly S3 credentials
+      # this basically allows us to reuse the cache from master, do our compilation and "copy" that to the git ref
+      # specific cache and because the credentials are read only, we do not override the master cache in the end
+      if: steps.cache_setup.outputs.cache-hit != 'true'
+      uses: philiplehmann/[email protected]
+      with:
+        endpoint: ${{inputs.s3-host}}
+        port: 443
+        insecure: false
+        accessKey: ${{inputs.s3-user-readonly}}
+        secretKey: ${{inputs.s3-password-readonly}}
+        bucket: ${{inputs.s3-bucket}}
+        key: ${{ inputs.key }}-refs_heads_master # use master cache readonly if no cache hit
+        path: ${{ inputs.path }}
+      continue-on-error: true
+
+    # ==================================================================================================================
+    # fork cache setup
+    # ==================================================================================================================
+    #
+    # Cache setup for forks or pull requests from forks
+    # Stores the cache on github hosted caches
+
+
+    # Upload cache on completion and check it out now
+    - name: Load .scons_cache directory
+      if: ${{ github.ref == 'refs/heads/master' && inputs.s3-host == '' }}
+      uses: actions/cache@v4
+      with:
+        path: ${{inputs.path}}
+        key: ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+
+        # We try to match an existing cache to restore from it. Each potential key is checked against
+        # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that
+        # starts with "linux-template-minimal", such as "linux-template-minimal-master-refs/heads/master-6588a4a29af1621086feac0117d5d4d37af957fd".
+        #
+        # We check these prefixes in this order:
+        #
+        #   1. The exact match, including the base branch, the commit reference, and the SHA hash of the commit.
+        #   2. A partial match for the same base branch and the same commit reference.
+        #   3. A partial match for the same base branch and the base branch commit reference.
+        #   4. A partial match for the same base branch only (not ideal, matches any PR with the same base branch).
+
+        restore-keys: |
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-refs/heads/${{env.GODOT_BASE_BRANCH}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}
+
+    # Check out cache but do not upload it at the end
+    - name: Load .scons_cache directory
+      if: ${{ github.ref != 'refs/heads/master' &&  inputs.s3-host == '' }}
+      uses: actions/cache/restore@v4 # only restores the cache but does not upload it
+      with:
+        path: ${{inputs.path}}
+        key: ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+
+        # We try to match an existing cache to restore from it. Each potential key is checked against
+        # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that
+        # starts with "linux-template-minimal", such as "linux-template-minimal-master-refs/heads/master-6588a4a29af1621086feac0117d5d4d37af957fd".
+        #
+        # We check these prefixes in this order:
+        #
+        #   1. The exact match, including the base branch, the commit reference, and the SHA hash of the commit.
+        #   2. A partial match for the same base branch and the same commit reference.
+        #   3. A partial match for the same base branch and the base branch commit reference.
+        #   4. A partial match for the same base branch only (not ideal, matches any PR with the same base branch).
+
+        restore-keys: |
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}-refs/heads/${{env.GODOT_BASE_BRANCH}}
+          ${{inputs.key}}-${{env.GODOT_BASE_BRANCH}}
+

.github/workflows/assemble_android.yml

@@ -0,0 +1,100 @@
+name: 🤖 Assemble Android
+on:
+  workflow_call:
+    inputs:
+      godot-version:
+        type: string
+      jvm-version:
+        type: string
+
+concurrency:
+  group: ci-${{github.actor}}-${{github.head_ref || github.run_number}}-${{github.ref}}-linux_assemble
+  cancel-in-progress: true
+
+jobs:
+  create-android-export-template:
+    runs-on: ubuntu-latest
+    name: ${{ matrix.name }}
+    steps:
+      - name: Maximize build space
+        uses: AdityaGarg8/[email protected]
+        with:
+          remove-dotnet: 'true'
+          remove-haskell: 'true'
+          remove-codeql: 'true'
+          remove-docker-images: 'true'
+          remove-swapfile: 'true'
+
+      - name: Clone Godot Engine
+        uses: actions/checkout@v4
+        with:
+          repository: godotengine/godot
+          ref: ${{ inputs.godot-version }}
+
+      - name: Clone Godot JVM module.
+        uses: actions/checkout@v4
+        with:
+          path: modules/kotlin_jvm
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: ${{ inputs.jvm-version }}
+
+      - name: Setup python and scons
+        uses: ./.github/actions/godot-deps
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v3
+        with:
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
+
+      - name: Download android libs release arm32
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_release_android_arm32
+          path: "platform/android/java/lib/libs/"
+
+      - name: Download android libs debug arm32
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_debug_android_arm32
+          path: "platform/android/java/lib/libs/"
+
+      - name: Download android libs release arm64
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_release_android_arm64
+          path: "platform/android/java/lib/libs/"
+
+      - name: Download android libs debug arm64
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_debug_android_arm64
+          path: "platform/android/java/lib/libs/"
+
+      - name: Download android libs release x86_64
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_release_android_x86_64
+          path: "platform/android/java/lib/libs/"
+
+      - name: Download android libs debug x86_64
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_debug_android_x86_64
+          path: "platform/android/java/lib/libs/"
+
+      - name: Generate Godot templates
+        run: |
+          ls -l platform/android/java/lib/libs/
+          cd platform/android/java
+          ./gradlew generateGodotTemplates
+          cd ../../..
+          ls -l bin/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: export_template_android_all
+          path: "bin/*"

.github/workflows/assemble_export_templates.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   create-export-templates-tpz:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: Create export template tpz
     steps:
       - name: Create templates folder

.github/workflows/assemble_ios.yml

@@ -24,19 +24,52 @@ jobs:
           repository: godotengine/godot
           ref: ${{ inputs.godot-version }}
 
+      - name: Clone Godot JVM module.
+        uses: actions/checkout@v4
+        with:
+          path: modules/kotlin_jvm
+
       # Python must be kept below 3.12
       #https://github.com/KhronosGroup/MoltenVK/blob/3b9e335fe8fc8a72cf6099acbd54128889445c0a/.github/workflows/CI.yml#L44
       - name: Setup python and scons
         uses: ./.github/actions/godot-deps
         with:
           python-version: 3.11
 
-      - name: Download ios export template binaries
+      - name: Download ios export template binaries debug
         uses: actions/download-artifact@v4
         with:
-          name: export_template_ios_all
+          name: export_template_debug_ios_arm64
           path: "./"
 
+      - name: Download ios export template binaries release
+        uses: actions/download-artifact@v4
+        with:
+          name: export_template_release_ios_arm64
+          path: "./"
+
+      - name: Setup MoltenVK build cache
+        uses: ./modules/kotlin_jvm/.github/actions/cache
+        with:
+          key: assemble_ios_moltenvk_cache
+          path: |
+            MoltenVk/Package
+            MoltenVk/External
+            MoltenVk/xcuserdata
+            MoltenVk/project.xcworkspace
+            MoltenVk/*.xccheckout
+            MoltenVk/*.xcscmblueprint
+            MoltenVk/build
+            MoltenVk/*.o
+            MoltenVk/*.a
+          s3-host: ${{ secrets.S3_HOST }}
+          s3-bucket: ${{ secrets.S3_BUCKET }}
+          s3-user: ${{ secrets.S3_USER }}
+          s3-password: ${{ secrets.S3_PASSWORD }}
+          s3-user-readonly: ${{ secrets.S3_USER_READONLY }}
+          s3-password-readonly: ${{ secrets.S3_PASSWORD_READONLY }}
+        continue-on-error: true
+
       - name: Clone MoltenVK
         uses: actions/checkout@v4
         with:

.github/workflows/assemble_linux.yml

@@ -15,7 +15,7 @@ concurrency:
 
 jobs:
   create-linux-editor-zip:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: ${{ matrix.name }}
     strategy:
       fail-fast: false