Merge pull request 'ci(release): enforce artifact-consistent versions' (#4608) from harden-release-artifact-version-consistency into staging

Reviewed-on: https://code.carverauto.dev/carverauto/serviceradar/pulls/4608

Author: mfreeman451

.forgejo/workflows/release.yml

@@ -21,8 +21,8 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag to publish (defaults to the event tag or v-prefixed VERSION)."
-        required: false
+        description: "Existing release tag to retry (for example, v1.4.22)."
+        required: true
         type: string
       dry_run:
         description: "Run publish_packages in --dry_run mode."
@@ -127,31 +127,34 @@ jobs:
 
           git fetch --no-tags origin +refs/heads/staging:refs/remotes/origin/staging
 
-          tag="${INPUT_TAG}"
-          if [[ -z "${tag}" && "${GITHUB_REF}" == refs/tags/* ]]; then
+          if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
+            tag="${INPUT_TAG}"
+            if [[ -z "${tag}" ]]; then
+              echo "Manual release dispatch requires an existing release tag." >&2
+              exit 1
+            fi
+            ./scripts/validate-release-tag.sh "${tag}"
+            git fetch --no-tags origin "refs/tags/${tag}:refs/tags/${tag}"
+          elif [[ "${GITHUB_REF}" == refs/tags/* ]]; then
             tag="${GITHUB_REF#refs/tags/}"
+            ./scripts/validate-release-tag.sh "${tag}"
+          else
+            echo "Release publication requires a tag-push event or an explicit manual tag." >&2
+            exit 1
           fi
-          if [[ -z "${tag}" ]]; then
-            version="$(tr -d '\r\n' < VERSION)"
-            tag="v${version}"
-          fi
-          ./scripts/validate-release-tag.sh "${tag}"
 
-          if git rev-parse -q --verify "refs/tags/${tag}^{commit}" >/dev/null; then
-            release_commit="$(git rev-list -n1 "refs/tags/${tag}^{commit}")"
-          else
-            if [[ "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
-              echo "Release tag ${tag} is not available in this checkout" >&2
-              exit 1
-            fi
-            release_commit="$(git rev-parse HEAD)"
+          if ! git rev-parse -q --verify "refs/tags/${tag}^{commit}" >/dev/null; then
+            echo "Release tag ${tag} is not available in this checkout." >&2
+            exit 1
           fi
+          release_commit="$(git rev-list -n1 "refs/tags/${tag}^{commit}")"
 
           if ! git merge-base --is-ancestor "${release_commit}" "${RELEASE_BASE_REF}"; then
             echo "Release commit ${release_commit} is not reachable from ${RELEASE_BASE_REF}" >&2
             exit 1
           fi
 
+          ./scripts/validate-release-metadata.sh "${tag}" "${release_commit}"
           printf 'tag=%s\ncommit=%s\n' "${tag}" "${release_commit}" >> "${GITHUB_OUTPUT}"
 
       - name: Cache Bazel artifacts
@@ -246,13 +249,8 @@ jobs:
             exit 1
           fi
 
+          ./scripts/validate-release-metadata.sh "${tag}" "${release_commit}" >/dev/null
           version="${tag#v}"
-          file_version="$(git show "${release_commit}:VERSION")"
-          file_version="${file_version%%[$'\r\n']*}"
-          if [[ -n "$file_version" && "$file_version" != "$version" ]]; then
-            echo "VERSION file ($file_version) does not match release tag ($version)" >&2
-            exit 1
-          fi
 
           # Auto-detect pre-release from version pattern
           is_prerelease="${INPUT_PRERELEASE}"
@@ -456,7 +454,11 @@ jobs:
 
       - name: Publish Helm chart to OCI registry
         env:
+          RELEASE_TAG: ${{ steps.release.outputs.tag }}
+          RELEASE_COMMIT: ${{ steps.release.outputs.commit }}
           VERSION: ${{ steps.release.outputs.version }}
+          CHART_OCI_USERNAME: ${{ secrets.HARBOR_CHART_ROBOT_USERNAME }}
+          CHART_OCI_TOKEN: ${{ secrets.HARBOR_CHART_ROBOT_SECRET }}
         run: |
           set -euo pipefail
           export HELM_STATE_DIR="${PWD}/.helm-home"
@@ -473,12 +475,15 @@ jobs:
             echo "${token}" | ./scripts/run-helm.sh registry login "${OCI_REGISTRY}" -u "${user}" --password-stdin
           }
 
-          if [[ -z "${OCI_USERNAME:-}" || -z "${OCI_TOKEN:-}" ]]; then
-            echo "Missing Harbor robot credentials. Set HARBOR_ROBOT_USERNAME and HARBOR_ROBOT_SECRET." >&2
+          if [[ -z "${CHART_OCI_USERNAME:-}" || -z "${CHART_OCI_TOKEN:-}" ]]; then
+            echo "Missing protected chart publisher credentials. Configure HARBOR_CHART_ROBOT_USERNAME and HARBOR_CHART_ROBOT_SECRET in the release environment." >&2
             exit 1
           fi
 
-          login_registry "${OCI_USERNAME}" "${OCI_TOKEN}"
+          login_registry "${CHART_OCI_USERNAME}" "${CHART_OCI_TOKEN}"
+
+          ./scripts/validate-release-metadata.sh "${RELEASE_TAG}" "${RELEASE_COMMIT}" >/dev/null
+          ./scripts/check-oci-chart-version-available.sh "${VERSION}"
 
           # Package and push chart
           ./scripts/run-helm.sh package helm/serviceradar

docs/docs/release-artifact-integrity.md

@@ -0,0 +1,89 @@
+---
+title: Release Artifact Integrity
+---
+
+# Release artifact integrity
+
+ServiceRadar product releases use one version identity across every published
+artifact. For product version `X.Y.Z`, all of the following must agree:
+
+- Git tag `vX.Y.Z`
+- `VERSION` value `X.Y.Z`
+- Helm `Chart.yaml` `version` and `appVersion` values `X.Y.Z`
+- OCI Helm chart tag `X.Y.Z`
+- OCI image semantic tags `vX.Y.Z`
+- Forgejo release tag `vX.Y.Z`
+
+The release workflow validates this agreement at the tagged commit. It refuses
+to publish from an arbitrary branch head, including during manual dispatch.
+
+## Cut a release
+
+Prepare the changelog entry, then use the supported release helper from a clean
+release branch:
+
+```bash
+scripts/cut-release.sh --version X.Y.Z --dry-run
+scripts/cut-release.sh --version X.Y.Z
+```
+
+The helper checks both `origin` for `vX.Y.Z` and the OCI chart repository for
+chart version `X.Y.Z` before changing release metadata. Both checks fail closed:
+if Git or the registry cannot be queried reliably, stop and restore access
+instead of bypassing the guard.
+
+Follow the merge and tag-push commands printed by the helper. The release tag
+must point to a commit reachable from `staging`. Do not push a release tag until
+the release change has merged.
+
+## Retry a failed publication
+
+Use the release workflow's manual dispatch and provide the existing release tag.
+The workflow fetches that exact tag, verifies that it is reachable from
+`origin/staging`, and revalidates the metadata at the tagged commit. A branch,
+commit SHA, missing tag, or value derived from the current `VERSION` file is not
+a valid manual release source.
+
+The retry remains subject to artifact immutability. It can complete missing
+artifacts, but it cannot replace an OCI chart version that already exists.
+
+## Recover from an occupied version
+
+Treat a published OCI chart version as an immutable audit record, even if no
+matching Git tag or Forgejo release exists. Deleting a Git tag does not make an
+OCI chart version reusable, and operators must not delete or overwrite the chart
+to force a release through.
+
+If the requested version is occupied:
+
+1. Leave the existing Git and OCI records intact.
+2. Select the next unoccupied product version.
+3. Add a changelog entry for that version.
+4. Run `scripts/cut-release.sh` again with the new version.
+
+Chart-only versions `1.4.16`, `1.4.17`, and `1.4.18` are known historical
+records and must remain reserved. The same rule applies to any future partial
+or interrupted release.
+
+## Protect chart publication credentials
+
+The protected Forgejo `release` environment must provide these chart-specific
+secrets:
+
+- `HARBOR_CHART_ROBOT_USERNAME`
+- `HARBOR_CHART_ROBOT_SECRET`
+
+The associated Harbor robot account must have only the permissions needed to
+pull and push `serviceradar/charts/serviceradar`. Do not grant delete authority.
+Protect the Forgejo environment with release approvals and the repository's
+release tag policy.
+
+The general `HARBOR_ROBOT_USERNAME` and `HARBOR_ROBOT_SECRET` credentials used
+by image builds and ordinary CI must not have push or delete authority for the
+product chart repository. Developer and validation paths may receive read-only
+access when they need to check whether a chart version exists.
+
+These Harbor and Forgejo access controls are deployment configuration. Verify
+them whenever credentials are rotated or a new runner is introduced; the
+workflow's separate secret names prevent accidental reuse but cannot replace
+registry-side authorization.

docs/sidebars.ts

@@ -132,6 +132,7 @@ const sidebars: SidebarsConfig = {
         {type: 'doc', id: 'cnpg-monitoring', label: 'CNPG Monitoring'},
         {type: 'doc', id: 'observability-rollup-recovery', label: 'Observability Rollup Recovery'},
         {type: 'doc', id: 'object-store-retention', label: 'Object Store Retention'},
+        {type: 'doc', id: 'release-artifact-integrity', label: 'Release Artifact Integrity'},
         {type: 'doc', id: 'troubleshooting-guide', label: 'Troubleshooting'},
       ],
     },

openspec/changes/harden-release-artifact-version-consistency/design.md

@@ -0,0 +1,87 @@
+## Context
+
+ServiceRadar currently uses the same numeric namespace for product Git tags
+(`vX.Y.Z`) and OCI Helm chart packages (`X.Y.Z`). The intended release path
+updates `VERSION` and `Chart.yaml` together, then publishes from a release
+tag. In practice, chart-only packages were published with new chart versions
+while `VERSION`, application images, and Git releases remained at `1.4.15`.
+Those immutable package versions now block a later product release from using
+the same number.
+
+## Goals / Non-Goals
+
+### Goals
+
+- Make a product version explainable and traceable to one immutable source.
+- Detect occupied Git and OCI chart versions before a release branch is
+  mutated or a publish job starts.
+- Prevent manual workflow dispatch from publishing product artifacts from an
+  untagged commit.
+- Preserve the ability to release Helm configuration fixes promptly, but make
+  those releases visible as product releases.
+
+### Non-Goals
+
+- Delete, retag, overwrite, or reinterpret existing OCI packages.
+- Rewrite Git history or reuse an already published version.
+- Automatically sync demo or production as part of this guard.
+- Create a second, independent public chart-versioning scheme in this change.
+
+## Decisions
+
+### A product release has one version identity
+
+For a product release `X.Y.Z`, the Git tag is `vX.Y.Z`, `VERSION` is `X.Y.Z`,
+and Helm `version` and `appVersion` are both `X.Y.Z`. The release workflow
+publishes the OCI chart `X.Y.Z` and image tags `vX.Y.Z` only after validating
+those inputs. A Helm-only correction therefore receives a formal product
+release and changelog entry instead of silently reserving a future product
+number.
+
+### The tag, not a dispatch input or HEAD, is the source of truth
+
+Push and manual-dispatch paths resolve an already existing tag, check that its
+commit is reachable from `origin/staging`, and check all source metadata at
+that commit. Manual dispatch is a retry/control surface for a real tag, not a
+way to manufacture a release identity from an arbitrary branch or commit.
+
+### Occupancy checks fail closed before mutation and publication
+
+The cut helper queries both the remote Git tag namespace and the OCI chart
+repository. An existing chart tag, an existing Git tag, or an inability to
+verify either source aborts the cut before any release metadata is written.
+The workflow repeats the artifact-occupancy check immediately before chart
+publication to close the time-of-check/time-of-use window.
+
+### Publish authority is isolated
+
+Only the protected signing workflow receives write credentials for the OCI
+chart repository. Normal developer and CI paths may perform read-only
+occupancy checks. The release runbook documents how an operator responds to a
+pre-existing version: select a new version, update the changelog, and preserve
+the existing artifact as an audit record.
+
+## Risks / Trade-offs
+
+- A configuration-only Helm correction now needs a formal product release.
+  This is intentional: users can see exactly which ServiceRadar version
+  introduced the configuration change.
+- OCI availability becomes a preflight dependency. The check fails closed,
+  which may delay a release during a registry outage but avoids an ambiguous
+  partial publication.
+- Existing orphaned package versions remain visible. Removing them would not
+  reliably invalidate cached/digest-pinned clients and would destroy useful
+  provenance.
+
+## Migration Plan
+
+1. Treat existing chart-only `1.4.16` through `1.4.18` packages as immutable
+   historical artifacts and do not reuse their versions.
+2. Record that product releases `v1.4.19` through `v1.4.22` were subsequently
+   published; do not reinterpret or reuse the earlier chart-only versions.
+3. Add source, metadata, and occupancy guards with regression tests before the
+   next product release.
+4. Restrict chart write credentials to the protected release environment and
+   document the operator recovery procedure.
+5. Verify a subsequent release and a manual retry both resolve the same tag
+   and cannot publish a second chart version.

openspec/changes/harden-release-artifact-version-consistency/proposal.md

@@ -0,0 +1,42 @@
+# Change: Make product release versions artifact-consistent
+
+## Why
+
+OCI Helm packages `1.4.16`, `1.4.17`, and `1.4.18` were published while the
+last corresponding ServiceRadar application release remained `v1.4.15`. Each
+package embeds application version and demo image tag `v1.4.15`; it is a
+chart-only publication, not a product release. OCI artifact versions are
+immutable release identities in practice, so deleting or reusing them would
+make caches, locks, provenance, and audit trails disagree. The result is an
+unexplained gap in user-visible product versions.
+
+## What Changes
+
+- Make a product release version one immutable identity across its Git tag,
+  `VERSION`, Helm `version` and `appVersion`, semantic image tags, OCI Helm
+  package, and Forgejo release.
+- Make `scripts/cut-release.sh` fail before modifying files when either the
+  remote Git tag or the OCI Helm chart version is already occupied or cannot
+  be verified.
+- Require every release-workflow dispatch to resolve an existing release tag
+  reachable from `staging`; remove the manual-dispatch fallback that treats an
+  arbitrary checked-out commit as a release source.
+- Require the release workflow to verify the tagged commit's version metadata
+  before it can publish a chart or other product artifacts.
+- Reserve chart publishing credentials for the protected release path. A
+  chart-only configuration change must be shipped in a documented product
+  release rather than directly consuming a future application version.
+- Add release-contract tests and an operator runbook for occupied-version
+  recovery. Existing chart-only packages remain immutable audit records and
+  are never deleted or reissued.
+
+## Impact
+
+- Affected specs: new `release-artifact-integrity` capability.
+- Affected code: `scripts/cut-release.sh`, release source validation helpers,
+  `.forgejo/workflows/release.yml`, release-contract tests, Helm release
+  publishing documentation, and protected registry credential configuration.
+- Operational impact: chart-only packages `1.4.16` through `1.4.18` remain
+  historical audit records. Product releases have since advanced through
+  `v1.4.22`; future releases cannot bypass the unified release path or reuse an
+  occupied Git/OCI version.