[RORDEV-1228] Move CI from Azure Pipelines to GitHub Actions (Ubicloud Linux + GitHub-hosted Windows)

.github/actionlint.yaml

@@ -0,0 +1,5 @@
+# Custom self-hosted runner labels used by CI (Ubicloud for Linux; Windows uses GitHub-hosted).
+# Registering them here silences actionlint's "unknown label" false positives.
+self-hosted-runner:
+  labels:
+    - ubicloud-standard-4       # 4 vCPU / 16 GB — the only Ubicloud size CI uses

azure-pipelines.yml

@@ -31,74 +31,13 @@ variables:
   # container jobs and never leak to the bare Windows jobs (which use a workspace-local GRADLE_USER_HOME).
   GRADLE_OPTS: '-Dorg.gradle.java.installations.auto-download=false'
 
-trigger:
-  # Coalesce rapid pushes into one run (Azure has no auto-cancel for branch runs).
-  batch: true
-  branches:
-    include:
-      - master
-      - develop
-      - 'epic/*'
-      - '*/epic/*'
-  tags:
-    exclude:
-      - '*'
-  paths:
-    # Azure path filters DO support wildcards, but the old patterns were too shallow to cover deep
-    # paths: `*.md` matches a single segment (root-level .md only), `*/*/*.md` matches exactly two
-    # dir levels — so `.claude/skills/<name>/SKILL.md` (three levels) fell through — and `.claude`
-    # had no exclusion at all, so markdown/skill-only changes still triggered CI. Exclude the whole
-    # docs/skills trees by prefix plus the root-level markdown docs. Code changes elsewhere still run.
-    # NOTE: the root-level *.md entries are an explicit list (a `*.md` glob would only match root level
-    # anyway in Azure path filters) — keep it in sync if you add a new root-level markdown doc.
-    exclude:
-      - docs
-      - .claude
-      - README.md
-      - CONTRIBUTING.md
-      - SECURITY.md
-      - LICENSE.md
-      - CLAUDE.md
-      - development.md
+# CI has moved to GitHub Actions (.github/workflows/ci.yml) on Ubicloud/GitHub-hosted runners.
+# Azure triggers are DISABLED; this pipeline remains manually runnable as a fallback until deletion.
+trigger: none
 
-pr:
-  # A new commit cancels the prior in-progress PR run instead of stacking another
-  # full run on the queue (GitHub default; set explicitly so it is version-controlled).
-  autoCancel: true
-  branches:
-    include:
-      - master
-      - develop
-      - 'epic/*'
-      - '*/epic/*'
-  paths:
-    # Azure path filters DO support wildcards, but the old patterns were too shallow to cover deep
-    # paths: `*.md` matches a single segment (root-level .md only), `*/*/*.md` matches exactly two
-    # dir levels — so `.claude/skills/<name>/SKILL.md` (three levels) fell through — and `.claude`
-    # had no exclusion at all, so markdown/skill-only changes still triggered CI. Exclude the whole
-    # docs/skills trees by prefix plus the root-level markdown docs. Code changes elsewhere still run.
-    # NOTE: the root-level *.md entries are an explicit list (a `*.md` glob would only match root level
-    # anyway in Azure path filters) — keep it in sync if you add a new root-level markdown doc.
-    exclude:
-      - docs
-      - .claude
-      - README.md
-      - CONTRIBUTING.md
-      - SECURITY.md
-      - LICENSE.md
-      - CLAUDE.md
-      - development.md
+pr: none
 
-# Scheduled rebuild of the CI toolchains image (BUILD_TOOLCHAINS_IMAGE only; every other stage is
-# gated on Build.Reason != 'Schedule'). Keeps the baked Gradle cache fresh without manual action:
-# a dep bumped during the week falls back to the network until the next rebuild closes the gap.
-schedules:
-  - cron: '0 3 * * 1'
-    displayName: 'Weekly CI toolchains image rebuild'
-    branches:
-      include:
-        - develop
-    always: true
+# (weekly toolchains-image rebuild now lives in the GitHub Actions workflow's schedule)
 
 parameters:
   - name: actionToPerform

build-base/src/main/java/tech/beshu/ror/buildbase/ParallelProcessRunner.java

@@ -90,6 +90,9 @@ public List<Integer> run() throws InterruptedException, ExecutionException, IOEx
                 .directory(workingDirectory)
                 .redirectErrorStream(true)
                 .redirectOutput(cmd.outputFile);
+        // Shard workers reuse the ROR plugin zip the parent's prebuild already assembled
+        // (see RorPluginGradleProject.assemble) instead of re-running a nested gradle build.
+        pb.environment().put("ROR_REUSE_ASSEMBLED", "1");
         Process process = pb.start();
         processes.add(process);
         futures.add(process.onExit());

build-base/src/main/java/tech/beshu/ror/buildbase/ShardedGradlewTest.java

@@ -51,14 +51,22 @@ public abstract class ShardedGradlewTest extends DefaultTask {
   public void runShards() {
     int shardCount = shardCountValue();
     File projectDir = getProject().getRootProject().getProjectDir();
-    File gradlew = new File(projectDir, "gradlew");
+    boolean isWindows = System.getProperty("os.name", "").toLowerCase().contains("win");
+    File gradlew = new File(projectDir, isWindows ? "gradlew.bat" : "gradlew");
     File logDir =
         new File(getProject().getLayout().getBuildDirectory().getAsFile().get(), "sharded-logs");
 
     ParallelProcessRunner runner = new ParallelProcessRunner(projectDir);
 
     for (int i = 0; i < shardCount; i++) {
       List<String> cmd = new ArrayList<>();
+      if (isWindows) {
+        // Windows CreateProcess cannot launch batch files directly (error 193); wrap in cmd.exe.
+        // ProcessHandle.descendants().destroyForcibly() in ParallelProcessRunner is cross-platform,
+        // so cancellation reaping works the same as on Linux.
+        cmd.add("cmd.exe");
+        cmd.add("/c");
+      }
       cmd.add(gradlew.getAbsolutePath());
       cmd.add("--no-daemon"); // mandatory for descendant-tree integrity (see class javadoc)
       cmd.add("integration-tests:test");
@@ -69,6 +77,16 @@ public void runShards() {
       }
       cmd.add("-PshardCount=" + shardCount);
       cmd.add("-PshardIndex=" + i);
+      // Child daemons only orchestrate one 512m test-worker JVM + docker containers; without this
+      // they inherit gradle.properties' -Xmx6144m, reserving ~6GB x K on a 16GB runner — the real
+      // memory ceiling behind the k=5/k=6 host-OOM deaths, not Elasticsearch itself.
+      cmd.add("-Dorg.gradle.jvmargs=-Xmx1024m -XX:MaxMetaspaceSize=512m");
+      // THIS task's dependsOn already ran prebuildEsImage once, before any shard spawned. The
+      // child's own prebuildEsImage dependency is NOT a cache hit: it launches a nested Tooling-API
+      // build ("Assembling ROR ...") and K concurrent nested builds in one workspace race each
+      // other to death. Exclude it — shards consume the image the parent prebuilt.
+      cmd.add("-x");
+      cmd.add("integration-tests:prebuildEsImage");
 
       File shardLog = new File(logDir, "shard-" + i + ".log");
       runner.addCommand(cmd, shardLog);

ci/ci-lib.sh

@@ -124,9 +124,9 @@ function tag {
   echo "Tagging as $GIT_TAG"
   git config --global push.default matching
   git config --global user.email "support@readonlyrest.com"
-  git config --global user.name "Azure Pipeline"
+  git config --global user.name "CI"
   # -f overwrites any stale local tag from a previous failed push attempt
-  git tag -fa "$GIT_TAG" -m "Generated tag from Azure Pipeline build $TRAVIS_BUILD_NUMBER"
+  git tag -fa "$GIT_TAG" -m "Generated tag from CI build $TRAVIS_BUILD_NUMBER"
   git push origin "$GIT_TAG"
   return 0
 }

ci/docker-hub-auth.sh

@@ -18,8 +18,13 @@
 if [ -n "${DOCKER_HUB_RO_TOKEN:-}" ] && [ "${DOCKER_HUB_RO_TOKEN}" != '$(DOCKER_HUB_RO_TOKEN)' ] \
    && [ -n "${DOCKER_HUB_USER:-}" ] && [ "${DOCKER_HUB_USER}" != '$(DOCKER_HUB_USER)' ]; then
   export DOCKER_AUTH_CONFIG="{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$(printf '%s:%s' "$DOCKER_HUB_USER" "$DOCKER_HUB_RO_TOKEN" | base64 -w0)\"}}}"
-  # Redact from logs despite the global system.debug:true (the value is base64(user:token)).
-  echo "##vso[task.setvariable variable=DOCKER_AUTH_CONFIG;isSecret=true]$DOCKER_AUTH_CONFIG"
+  # Redact from logs (the value is base64(user:token)). Each CI has its own log command; emitting
+  # the Azure ##vso line on GitHub Actions would PRINT the secret instead of masking it.
+  if [ -n "${GITHUB_ACTIONS:-}" ]; then
+    echo "::add-mask::$DOCKER_AUTH_CONFIG"
+  else
+    echo "##vso[task.setvariable variable=DOCKER_AUTH_CONFIG;isSecret=true]$DOCKER_AUTH_CONFIG"
+  fi
   echo "[TEST] Docker Hub authenticated pulls ENABLED (user '$DOCKER_HUB_USER')"
 else
   echo "[TEST] Docker Hub authenticated pulls DISABLED (anonymous, rate-limited) — DOCKER_HUB_USER/DOCKER_HUB_RO_TOKEN not both set"

ci/github/PORT.md

@@ -0,0 +1,130 @@
+# Azure Pipelines → GitHub Actions port
+
+`.github/workflows/ci.yml` ports `azure-pipelines.yml` to GitHub Actions.
+Linux jobs run on **Ubicloud** (`ubicloud-standard-4` = 4 vCPU / **16 GB**) inside the
+same `beshultd/ror-ci-toolchains` image; Windows jobs run on **GitHub-hosted
+`windows-2025`** (free for public repos — the same VM family Azure used).
+
+The **build logic is unchanged** — every Linux job still calls `ci/run-pipeline.sh`
+with a `ROR_TASK`, exactly as Azure did. `run-pipeline.sh` and its sourced libs
+contain no Azure-specific constructs (verified), so only the orchestration layer
+was rewritten. All 54 `ROR_TASK` values referenced by the workflow exist in the script.
+
+**Azure triggers are disabled** (`trigger: none`, `pr: none`, schedules removed) in the
+same PR; the Azure pipeline stays manually runnable as a fallback until it's deleted.
+
+## Stage → job mapping
+
+| Azure stage | GH job | Notes |
+|---|---|---|
+| SUPERSEDE_GUARD | — | replaced by `concurrency` (cancel-in-progress **for PRs only**; branch pushes queue, so an in-flight release is never cancelled — Azure `batch: true` semantics) |
+| DISK_PROBE | — | dropped (Azure host-recon only; N/A on Ubicloud) |
+| ES_S3_UP | `es_s3_up` | `newes/*` branches only; ordered **before** all check/test jobs, as on Azure |
+| BUILD_TOOLCHAINS_IMAGE | `build_toolchains_image` | schedule + manual |
+| TOOLCHAINS_VERIFY | `toolchains_verify` | |
+| OPTIONAL_CHECKS (CVE) | `cve_check` | `continue-on-error`; monthly cache key (`yyyyMM`), Azure `Cache@2` semantics |
+| REQUIRED_CHECKS | `required_checks` | 4-way matrix |
+| TEST (unit) | `unit_tests` | |
+| TEST (Linux IT ×3 jobs) | `it_linux` | **dynamic matrix** from `setup`: full 34-version set on develop/master/epic + manual, 10-version subset on PRs. Skipped legs never boot a VM. |
+| TEST (Windows IT ×3 jobs) | `it_windows` | dynamic matrix: 7 versions on develop/master/epic, 3 on PRs, full 33 on manual `run_all_tests_on_windows` |
+| TEST (Windows unit, manual) | `unit_tests_windows` | manual `run_all_tests_on_windows` only |
+| BUILD_ROR | `build_ror` | PR only; gates on Linux **and Windows** results (Azure `succeeded('TEST')` parity) |
+| DETERMINE_CI_TYPE | `determine_ci_type` | `!cancelled()` + explicit result checks — see below |
+| UPLOAD_PRE_ROR | `upload_pre_ror` | pre-release, auto only |
+| RELEASE_ROR (+ without-testing) | `release_ror` | `permissions: contents: write` for the tag push; `TRAVIS_BUILD_NUMBER=github.run_number` for the tag message |
+| PUBLISH_MVN_ARTIFACTS (+ without-testing) | `publish_mvn` | master release only |
+
+### The skipped-needs trap (why release jobs check `needs.<job>.result` explicitly)
+
+GitHub skips a job whose `needs` contains a skipped job (implicit `success()`).
+The manual `release_without_testing` path *intentionally* skips the test jobs, so
+`determine_ci_type` / `release_ror` / `publish_mvn` use `!cancelled()` plus explicit
+two-arm conditions, mirroring Azure:
+
+- **auto arm**: push to develop/master, `required_checks`/`unit_tests`/`it_linux`/`it_windows`
+  all `result == 'success'`
+- **manual arm**: `workflow_dispatch` + `actionToPerform == 'release_without_testing'` +
+  `toolchains_verify.result == 'success'`
+
+## Trigger-scenario walk-through (regression table)
+
+Which jobs run per trigger (✓ run / − skip), matching the Azure stage conditions:
+
+| Job | PR | push develop | push master | push newes/* PR | dispatch: linux tests | dispatch: win tests | dispatch: release w/o testing | schedule |
+|---|---|---|---|---|---|---|---|---|
+| setup | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ |
+| build_toolchains_image | − | − | − | − | − | − | − | ✓ |
+| toolchains_verify | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | − |
+| es_s3_up | − | − | − | ✓ | − | − | − | − |
+| cve_check | ✓ | ✓ | ✓ | ✓ | − | − | − | − |
+| required_checks | ✓ | ✓ | ✓ | ✓ | − | − | − | − |
+| unit_tests | ✓ | ✓ | ✓ | ✓ | ✓ | − | − | − |
+| it_linux | ✓ (10) | ✓ (34) | ✓ (34) | ✓ (10) | ✓ (34) | − | − | − |
+| it_windows | ✓ (3) | ✓ (7) | ✓ (7) | ✓ (3) | − | ✓ (33) | − | − |
+| unit_tests_windows | − | − | − | − | − | ✓ | − | − |
+| build_ror | ✓ | − | − | ✓ | − | − | − | − |
+| determine_ci_type | − | ✓ | ✓ | − | − | − | ✓ (on develop/master ref) | − |
+| upload_pre_ror | − | ✓ if `-pre` | ✓ if `-pre` | − | − | − | − | − |
+| release_ror | − | ✓ if release | ✓ if release | − | − | − | ✓ if release | − |
+| publish_mvn | − | − | ✓ if release | − | − | − | ✓ if release+master | − |
+
+## Azure feature translations
+
+| Azure | GitHub |
+|---|---|
+| `$(System.AccessToken)` | `secrets.GITHUB_TOKEN` |
+| `##vso[task.setvariable]` | `>> $GITHUB_OUTPUT` / `>> $GITHUB_ENV` |
+| `##vso[...;isSecret=true]` (docker-hub-auth.sh) | `::add-mask::` — the script is now CI-aware (emitting the ##vso line on GH would have **printed** the secret) |
+| `Cache@2` monthly CVE key | `actions/cache@v4` + `date +%Y%m` key |
+| `DownloadSecureFile@1` (secret.pgp) | base64 secret `PGP_SECRET_KEY_B64`, decoded in-step |
+| `PublishTestResults@2` | `actions/upload-artifact` of `**/TEST*.xml` on failure |
+| PR-vs-branch matrix subsets (3 jobs per OS) | one dynamic matrix per OS from the `setup` job |
+| `free-host-disk.sh` (`target: host`) | runs normally; `AGENT_ISSELFHOSTED=1` makes it a no-op (Ubicloud VMs have no Azure bloat to reclaim) |
+| git tag push (persistCredentials) | `permissions: contents: write` on `release_ror` — **no SSH deploy key needed** |
+
+## Deliberate behavior differences vs Azure
+
+- **`paths-ignore: '**/*.md'` is broader than Azure**, which could only exclude root-level
+  `.md` files plus the `docs`/`.claude` trees (Azure glob limitation — its own comment says
+  deeper exclusion was wanted). Here any md-only change anywhere skips CI. Markdown can't
+  affect build outputs, so this is the intent Azure couldn't express.
+
+## Deliberately dropped (and why)
+
+- **DISK_PROBE** (and the `run_disk_probe` manual action) — Azure host-disk recon; irrelevant
+  on Ubicloud VMs. `ci/probe-host-disk.sh` stays in the tree for the Azure manual fallback.
+- **SUPERSEDE_GUARD** + `ci/stale-azure-pipeline-runs-canceler.sh` — native `concurrency`.
+- **Docker pre-clean / always()-reap steps** from the Azure IT template — both Ubicloud and
+  GH-hosted runners are ephemeral (fresh VM per job); there are no leftovers to reap and no
+  sibling jobs sharing a daemon. `run-pipeline.sh`'s SIGTERM trap still reaps on cancel.
+## IT parallelism (not in Azure — added during the port)
+
+Azure ran each IT leg's suites serially in one JVM (~85–100 min/leg incl. queue). Here every
+leg runs **4 sharded test JVMs** on its VM (`integration-tests:shardedTest`, `IT_PARALLELISM`),
+cutting a leg to ~18–24 min. Windows runs 3 shards (native ES processes, per-shard port/dir/
+WireMock windows). Two safety mechanisms make this fit 16 GB:
+
+- **Duration-balanced packing** (`ROR_BALANCED_SHARDS`): suites are LPT-packed into shards by
+  measured wall time (`integration-tests/suite-timings.json`) instead of name-hash, so no shard
+  becomes the long pole. Timings are advisory — suites without an entry default to 60s and still run.
+- **`HeavySuiteGate`** (`ROR_HEAVY_SUITE_PERMITS`, currently 2): a machine-wide file-lock
+  semaphore capping how many multi-node-cluster suites boot containers concurrently across all
+  shard JVMs. Without it, level packing host-OOMs 16 GB runners. Crash-safe: an OOM-killed
+  worker's lock dies with its process.
+
+Per-shard console logs upload as the `sharded-logs-*` artifact and per-shard JUnit XML as
+`*-results` (both always, pass or fail). Measured limits, for whoever tunes this next: 5 workers or 3 permits exceed
+either 16 GB (at 512m ES heaps) or the 4-vCPU boot-time budget — both were tried on isolated
+probe runs and reverted.
+
+## Before first run — do these
+
+1. **Install the Ubicloud GitHub App** on the repo (Linux jobs). Windows needs nothing —
+   GitHub-hosted. Without Ubicloud, every Linux job queues forever.
+2. **Set secrets/variables**: `ci/github/set-secrets.sh` (17 secrets + 8 variables, see SECRETS.md).
+3. **Smoke-run**: `workflow_dispatch` → `run_all_tests_on_linux` on a branch validates
+   toolchains_verify + unit + full Linux IT without touching release paths. Watch the first IT
+   leg: testcontainers inside the `container:` job must reach the host Docker daemon
+   (Actions auto-mounts `/var/run/docker.sock`; Ubicloud provides host Docker).
+4. Azure is already trigger-disabled; it remains manually runnable from the Azure DevOps UI
+   as a fallback until you delete `azure-pipelines.yml` + `ci/azure-templates/`.