[RONDB-1083] Make startupProbe timeouts configurable for ndbmtd and M… #475

Workflow file for this run

.github/workflows/run_tests.yml at 928ef0e

	name: Integration tests

	on:
	pull_request:
	# Not running on "closed" - that is taken care of by "push" (if merged)
	types: [opened, synchronize, reopened]

	# This cancels any previous job from the same PR if the PR has been updated.
	# This cancel-in-progress only works per PR (thus, two different PRs wont be cancelled).
	# Concurrency is not an issue because the self-hosted worker will anyways only run one
	# job at a time from one repo.
	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	jobs:
	lint-test:
	name: Lint Helm chart
	runs-on: ubuntu-latest
	env:
	TEMPLATE_OUT_DIR: template_out
	steps:
	- name: Checkout main repo
	uses: actions/checkout@v4

	- run: \|
	echo "GH ref: ${{ github.ref }}"
	echo "GH ref_name: ${{ github.ref_name }}"

	- name: Check copyright header
	run: ./.github/update_copyright.sh

	- name: Set up Helm
	uses: azure/setup-helm@v4
	with:
	version: '3.13.3'

	- run: helm lint --strict .
	- run: helm lint --strict --values values/dummy_lint.yaml .

	- name: Helm template
	if: always()
	run: \|
	helm template . \
	--include-crds \
	--debug \
	--dry-run \
	--values values/dummy_lint.yaml \
	--output-dir $TEMPLATE_OUT_DIR

	- run: ls -l $TEMPLATE_OUT_DIR/**

	- uses: docker://ghcr.io/yannh/kubeconform:latest
	with:
	entrypoint: '/kubeconform'
	args: "-summary -strict -ignore-missing-schemas ${{ env.TEMPLATE_OUT_DIR }}"

	- name: Upload templates
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: templates
	path: ${{ env.TEMPLATE_OUT_DIR }}
	retention-days: 5

	benchmark-and-stability:
	needs: [lint-test]
	name: Benchmarking, scaling and stability
	if: github.repository == 'logicalclocks/rondb-helm'
	concurrency:
	group: self-hosted-runner
	cancel-in-progress: false
	runs-on: [self-hosted, ARM64]
	env:
	K8S_NAMESPACE: rondb-helm-${{ github.run_id }}-${{ github.run_attempt }}
	RONDB_CLUSTER_NAME: my-rondb
	steps:
	- name: Checkout main repo
	uses: actions/checkout@v4

	- name: Check kubectl (should be Minikube)
	run: \|
	kubectl version --client
	kubectl get nodes

	- name: Pre-cleanup stale resources
	run: \|
	# Delete any leftover rondb-helm namespaces from previous cancelled/failed runs
	for ns in $(kubectl get namespaces -o name \| grep 'namespace/rondb-helm-' \| cut -d/ -f2); do
	if [ "$ns" != "$K8S_NAMESPACE" ]; then
	echo "Cleaning up stale namespace: $ns"
	helm delete --namespace="$ns" my-rondb 2>/dev/null \|\| true
	kubectl delete namespace "$ns" --timeout=120s 2>/dev/null \|\| true
	fi
	done
	# Delete any leftover lifecycle test namespaces
	for ns in cluster-a cluster-b cluster-c cluster-d; do
	if kubectl get namespace "$ns" &>/dev/null; then
	echo "Cleaning up stale namespace: $ns"
	helm delete --namespace="$ns" "$ns" 2>/dev/null \|\| true
	kubectl delete namespace "$ns" --timeout=120s 2>/dev/null \|\| true
	fi
	# Each lifecycle cluster uses its own name as PriorityClass
	kubectl delete priorityclass "$ns" 2>/dev/null \|\| true
	done
	# Delete orphaned PriorityClass if it exists
	kubectl delete priorityclass rondb-high-priority 2>/dev/null \|\| true

	# Static CPU manager policy will only work if the FeatureGate is enabled
	- name: Create original RonDB cluster
	run: \|
	helm install $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--create-namespace \
	--timeout=15m \
	--values ./values/minikube/mini.yaml \
	--set benchmarking.enabled=true \
	--set staticCpuManagerPolicy=true \
	--set terminationGracePeriodSeconds=40 \
	--set meta.rdrs.statefulSet.endToEndTls.enabled=true \
	--set clusterSize.numNodeGroups=2 \
	--set clusterSize.minNumMySQLServers=1 \
	--set clusterSize.maxNumMySQLServers=2 \
	--set clusterSize.minNumRdrs=1 \
	--set clusterSize.maxNumRdrs=2 \
	--set rondbConfig.TotalMemoryConfig=4096M \
	.
	sleep 10

	- run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=generate-data

	- name: Waiting for benchmark job to complete
	run: bash .github/wait_job.sh $K8S_NAMESPACE benchs 240

	- name: Collect bench logs
	if: always()
	uses: ./.github/actions/collect_bench_logs
	with:
	namespace: ${{ env.K8S_NAMESPACE }}

	- name: Terminate RonDB data node
	run: kubectl -n $K8S_NAMESPACE delete pod node-group-0-0 --force

	- name: Test deploy stability
	shell: bash
	timeout-minutes: 6
	env:
	SLEEP_SECONDS: 10
	MIN_STABLE_MINUTES: 1
	run: bash .github/test_deploy_stability.sh
	- run: kubectl -n $K8S_NAMESPACE get pods

	# Check that data has not been lost
	- run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=verify-data
	- run: ./.github/delete_test_pods.sh $K8S_NAMESPACE

	# Avoid pro-longed stability check because of MySQLd HPA.
	# Can't scale down MySQLds because of config.ini.
	- name: Scale data node replicas
	run: \|
	helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--timeout=15m \
	--reuse-values \
	--set clusterSize.minNumMySQLServers=2 \
	--set clusterSize.activeDataReplicas=2 \
	.

	- name: Test deploy stability
	shell: bash
	timeout-minutes: 6
	env:
	SLEEP_SECONDS: 10
	MIN_STABLE_MINUTES: 1
	run: bash .github/test_deploy_stability.sh
	- run: kubectl -n $K8S_NAMESPACE get pods

	# Simple change in config.ini
	- name: Rolling restart of cluster
	run: \|
	helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--timeout=15m \
	--reuse-values \
	--set rondbConfig.EmptyApiSlots=5 \
	.

	- name: Test deploy stability
	shell: bash
	timeout-minutes: 12
	env:
	SLEEP_SECONDS: 10
	MIN_STABLE_MINUTES: 1
	run: bash .github/test_deploy_stability.sh
	- run: kubectl -n $K8S_NAMESPACE get pods

	# Check again that data is still there
	- run: ./.github/delete_test_pods.sh $K8S_NAMESPACE
	- run: helm test -n $K8S_NAMESPACE $RONDB_CLUSTER_NAME --logs --filter name=verify-data

	# RONDB-1053: numNodeGroups immutability. Rejection tests run a
	# real `helm upgrade` — helm 3.13+'s --dry-run=client returns
	# empty from `lookup` and trips the wrong error branch. Failed
	# renders and failed pre-upgrade hooks both abort before apply,
	# so rejections leave the cluster unchanged. The bypass test
	# uses --dry-run since the check is skipped and the upgrade
	# would otherwise mutate.
	- name: numNodeGroups immutability — reject increase
	run: \|
	set -e
	if out=$(helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--reuse-values \
	--set clusterSize.numNodeGroups=3 \
	. 2>&1); then
	echo "$out"
	echo "ERROR: helm upgrade should have failed (numNodeGroups 2→3)"
	exit 1
	fi
	echo "$out"
	echo "$out" \| grep -qF "deployed=2, requested=3"

	- name: numNodeGroups immutability — reject decrease
	run: \|
	set -e
	if out=$(helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--reuse-values \
	--set clusterSize.numNodeGroups=1 \
	. 2>&1); then
	echo "$out"
	echo "ERROR: helm upgrade should have failed (numNodeGroups 2→1)"
	exit 1
	fi
	echo "$out"
	echo "$out" \| grep -qF "deployed=2, requested=1"

	- name: numNodeGroups immutability — forceNodeGroupChange bypasses
	run: \|
	set -e
	out=$(helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--reuse-values --dry-run \
	--set clusterSize.numNodeGroups=3 \
	--set forceNodeGroupChange=true \
	. 2>&1) && rc=0 \|\| rc=$?
	echo "$out"
	if echo "$out" \| grep -qF "clusterSize.numNodeGroups is immutable"; then
	echo "ERROR: forceNodeGroupChange should have bypassed the immutability check"
	exit 1
	fi
	if [ $rc -ne 0 ]; then
	echo "ERROR: upgrade with forceNodeGroupChange=true failed (not for immutability reasons); exit=$rc"
	exit $rc
	fi

	- name: numNodeGroups immutability — Job branch (mode=upgrade) rejects increase
	run: \|
	set -e
	# Job's hook-delete-policy keeps failed runs around so
	# kubectl logs below can read its stderr.
	if out=$(helm upgrade -i $RONDB_CLUSTER_NAME \
	--namespace=$K8S_NAMESPACE \
	--reuse-values \
	--set mode=upgrade \
	--set clusterSize.numNodeGroups=3 \
	. 2>&1); then
	echo "$out"
	echo "ERROR: helm upgrade should have failed via Job hook (mode=upgrade, 2→3)"
	exit 1
	fi
	echo "$out"
	# Confirms the Job-branch fired (vs. inline render fail).
	echo "$out" \| grep -qF "pre-upgrade hooks failed"
	# Drift guard: Job error format must match the inline branch.
	kubectl -n $K8S_NAMESPACE logs \
	job/$RONDB_CLUSTER_NAME-rondb-topology-check \
	\| grep -qF "deployed=2, requested=3"
	# Failed hook Jobs aren't covered by the Job's delete policy
	# and aren't cleaned by `helm uninstall` (they belong to the
	# failed upgrade, not the last successful release). The
	# teardown step waits for an empty namespace, so drop it here.
	kubectl -n $K8S_NAMESPACE delete \
	job/$RONDB_CLUSTER_NAME-rondb-topology-check

	- name: Collect logs
	if: always()
	uses: ./.github/actions/collect_logs
	with:
	namespace: ${{ env.K8S_NAMESPACE }}
	folder_name: k8s_logs

	- name: Remove cluster
	if: always()
	uses: ./.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.K8S_NAMESPACE }}
	helm_chart: ${{ env.RONDB_CLUSTER_NAME }}

	test-lifecycle:
	needs: [benchmark-and-stability]
	name: Test Global Replication with backup/restore
	if: github.repository == 'logicalclocks/rondb-helm'
	concurrency:
	group: self-hosted-runner
	cancel-in-progress: false
	runs-on: [self-hosted, ARM64]
	env:
	CLUSTER_A_NAME: cluster-a
	CLUSTER_B_NAME: cluster-b
	CLUSTER_C_NAME: cluster-c
	CLUSTER_D_NAME: cluster-d
	steps:
	- name: Checkout main repo
	uses: actions/checkout@v4

	- name: Check kubectl (should be Minikube)
	run: \|
	kubectl version --client
	kubectl get nodes

	- name: Pre-cleanup stale resources
	run: \|
	for ns in $CLUSTER_A_NAME $CLUSTER_B_NAME $CLUSTER_C_NAME $CLUSTER_D_NAME; do
	if kubectl get namespace "$ns" &>/dev/null; then
	echo "Cleaning up stale namespace: $ns"
	helm delete --namespace="$ns" "$ns" 2>/dev/null \|\| true
	kubectl delete namespace "$ns" --timeout=120s 2>/dev/null \|\| true
	fi
	# Each cluster uses its own name as PriorityClass
	kubectl delete priorityclass "$ns" 2>/dev/null \|\| true
	done
	kubectl delete priorityclass rondb-high-priority 2>/dev/null \|\| true

	- name: Setup MinIO
	run: \|
	./test_scripts/setup_minio.sh

	# This script will also remove all clusters & namespaces
	- name: Run lifecycle test
	timeout-minutes: 30
	run: \|
	./test_scripts/lifecycle-test.sh \
	$CLUSTER_A_NAME \
	$CLUSTER_B_NAME \
	$CLUSTER_C_NAME \
	$CLUSTER_D_NAME

	- name: Collect logs
	if: failure()
	uses: ./.github/actions/collect_logs
	with:
	namespace: ${{ env.CLUSTER_A_NAME }}
	folder_name: ${{ env.CLUSTER_A_NAME }}

	- name: Remove cluster
	if: always()
	uses: ./.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.CLUSTER_A_NAME }}
	helm_chart: ${{ env.CLUSTER_A_NAME }}

	- name: Collect logs
	if: failure()
	uses: ./.github/actions/collect_logs
	with:
	namespace: ${{ env.CLUSTER_B_NAME }}
	folder_name: ${{ env.CLUSTER_B_NAME }}

	- name: Remove cluster
	if: always()
	uses: ./.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.CLUSTER_B_NAME }}
	helm_chart: ${{ env.CLUSTER_B_NAME }}

	- name: Collect logs
	if: failure()
	uses: ./.github/actions/collect_logs
	with:
	namespace: ${{ env.CLUSTER_C_NAME }}
	folder_name: ${{ env.CLUSTER_C_NAME }}

	- name: Remove cluster
	if: always()
	uses: ./.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.CLUSTER_C_NAME }}
	helm_chart: ${{ env.CLUSTER_C_NAME }}

	- name: Collect logs
	if: failure()
	uses: ./.github/actions/collect_logs
	with:
	namespace: ${{ env.CLUSTER_D_NAME }}
	folder_name: ${{ env.CLUSTER_D_NAME }}

	- name: Remove cluster
	if: always()
	uses: ./.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.CLUSTER_D_NAME }}
	helm_chart: ${{ env.CLUSTER_D_NAME }}

	- name: Delete MinIO tenant
	if: always()
	run: \|
	source ./test_scripts/minio.env
	helm delete --namespace $MINIO_TENANT_NAMESPACE tenant

	- name: Delete MinIO namespace
	if: always()
	run: \|
	source ./test_scripts/minio.env
	kubectl delete namespace $MINIO_TENANT_NAMESPACE --timeout=50s \|\| true

	# Cross-version chart upgrade test for the pre-upgrade MGMd hook
	# (templates/mgmd_pre_upgrade.yaml). Installs the chart at branch-0.7
	# (which pre-dates the hook) and upgrades to the PR head, then asserts
	# the upgrade did not trigger arbitration loss (NDB Error 2305) or a
	# whole-cluster system restart. Requested by maintainer in PR #170 review
	# comment 3161156784.
	upgrade-cross-version:
	needs: [test-lifecycle]
	name: Cross-version chart upgrade (no arbitration loss)
	if: github.repository == 'logicalclocks/rondb-helm'
	concurrency:
	group: self-hosted-runner
	cancel-in-progress: false
	runs-on: [self-hosted, ARM64]
	env:
	K8S_NAMESPACE: rondb-helm-upgrade-${{ github.run_id }}-${{ github.run_attempt }}
	RONDB_CLUSTER_NAME: my-rondb
	steps:
	- name: Checkout new chart (PR head)
	uses: actions/checkout@v4
	with:
	path: new-chart

	- name: Checkout old chart (branch-0.7, chart 0.7.29 / app 24.10.19)
	uses: actions/checkout@v4
	with:
	repository: logicalclocks/rondb-helm
	ref: branch-0.7
	path: old-chart

	- name: Check kubectl (should be Minikube)
	run: \|
	kubectl version --client
	kubectl get nodes

	- name: Pre-cleanup stale resources
	run: \|
	# Delete any leftover rondb-helm namespaces from previous cancelled/failed runs
	for ns in $(kubectl get namespaces -o name \| grep 'namespace/rondb-helm-' \| cut -d/ -f2); do
	if [ "$ns" != "$K8S_NAMESPACE" ]; then
	echo "Cleaning up stale namespace: $ns"
	helm delete --namespace="$ns" my-rondb 2>/dev/null \|\| true
	kubectl delete namespace "$ns" --timeout=120s 2>/dev/null \|\| true
	fi
	done
	kubectl delete priorityclass rondb-high-priority 2>/dev/null \|\| true

	# Install the OLD chart with small.yaml — needed for the
	# `numNodeGroups: 1, activeDataReplicas: 2` topology this test
	# exercises (mini.yaml ships `activeDataReplicas: 1` and smaller
	# memory tunings).
	#
	# --set overrides:
	# - clusterSize.{min,max}NumMySQLServers=1: one mysqld is
	# enough to exercise the upgrade and saves runner resources.
	# - TotalMemoryConfig=4096M: AutomaticMemoryConfig requires
	# >=3589 MB on this 1x2 topology with small.yaml's memory
	# reservations; 2048M fails with NDB error 2350.
	# - resources.limits.memory.ndbmtdsMiB=5120: small.yaml's 3300
	# MiB cap is below what TotalMemoryConfig=4096M needs at
	# startphase 0; without this the pod is cgroup-OOM-killed.
	# - backups.enabled=false: 0.7.29 defaults `backups.enabled:
	# true` with a hardcoded aws-credentials Secret that doesn't
	# exist in CI; the new chart defaults to null so this carries
	# through as a no-op.
	- name: Install old chart (0.7.29 / 24.10.19)
	run: \|
	helm install $RONDB_CLUSTER_NAME ./old-chart \
	--namespace=$K8S_NAMESPACE \
	--create-namespace \
	--wait --timeout=15m \
	--values ./old-chart/values/minikube/small.yaml \
	--set staticCpuManagerPolicy=true \
	--set clusterSize.minNumMySQLServers=1 \
	--set clusterSize.maxNumMySQLServers=1 \
	--set rondbConfig.TotalMemoryConfig=4096M \
	--set resources.limits.memory.ndbmtdsMiB=5120 \
	--set backups.enabled=false

	# Inline diagnostics on install failure. `helm --wait` is silent
	# until success or timeout, so without these a failure surfaces
	# in the runner log only as `context deadline exceeded`. Each
	# section below targets a specific failure-mode question
	# (OOMKilled vs node-OOM, --set typos, schema rejection, unbound
	# PVCs, cluster-wide RonDB events).
	- name: Diagnostics on install failure
	if: failure()
	run: \|
	echo "=== pods ==="
	kubectl -n $K8S_NAMESPACE get pods -o wide \|\| true
	echo "=== events (last 30, newest first) ==="
	kubectl -n $K8S_NAMESPACE get events --sort-by='.metadata.creationTimestamp' \
	\| tail -n 30 \|\| true
	echo "=== ndbmtd / mgmd / rdrs / mysqld container logs (last 50 lines each) ==="
	for pod in $(kubectl -n $K8S_NAMESPACE get pods -o jsonpath='{.items[*].metadata.name}'); do
	# Init containers first — a pod stuck in Init:N/M has its
	# failure narrative here, and `.spec.containers[*]` alone
	# silently skips them.
	for c in $(kubectl -n $K8S_NAMESPACE get pod "$pod" -o jsonpath='{.spec.initContainers[*].name}'); do
	echo "--- $pod / $c (init, current) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c "$c" --tail=50 2>&1 \|\| true
	echo "--- $pod / $c (init, previous) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c "$c" --previous --tail=50 2>&1 \|\| true
	done
	for c in $(kubectl -n $K8S_NAMESPACE get pod "$pod" -o jsonpath='{.spec.containers[*].name}'); do
	echo "--- $pod / $c (current) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c "$c" --tail=50 2>&1 \|\| true
	echo "--- $pod / $c (previous) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c "$c" --previous --tail=50 2>&1 \|\| true
	done
	done
	# `kubectl describe pod` surfaces lastState.terminated
	# (Reason=OOMKilled, ExitCode=137, Signal=...) plus per-container
	# Limits — the direct answer to "why did this container die".
	echo "=== pod descriptions (lastState.terminated: reason, exitCode, signal) ==="
	kubectl -n $K8S_NAMESPACE describe pods \|\| true
	# Verifies `--set resources.limits.memory.*` actually
	# propagated. A typo in the dotted path silently no-ops.
	echo "=== container resource limits as rendered by the chart ==="
	for sts in $(kubectl -n $K8S_NAMESPACE get statefulset \
	-o jsonpath='{.items[*].metadata.name}'); do
	echo "--- statefulset/$sts ---"
	kubectl -n $K8S_NAMESPACE get statefulset "$sts" \
	-o jsonpath='{range .spec.template.spec.containers[*]}{.name}: requests={.resources.requests} limits={.resources.limits}{"\n"}{end}' \
	\|\| true
	echo
	done
	# Distinguishes container-cgroup OOM from node-level memory
	# pressure (MemoryPressure=True + Allocated resources table).
	echo "=== node state (MemoryPressure, Allocatable vs Allocated) ==="
	kubectl describe nodes \|\| true
	echo "=== resource utilisation (only printed if metrics-server is up) ==="
	kubectl top nodes 2>&1 \|\| true
	kubectl -n $K8S_NAMESPACE top pods 2>&1 \|\| true
	# Resolved values (defaults + overlay + every --set). Catches
	# missing/typo'd overrides at a glance.
	echo "=== helm release: resolved values (post --set) ==="
	helm get values $RONDB_CLUSTER_NAME -n $K8S_NAMESPACE --all \|\| true
	# Release-level state for failures where pod-side diagnostics
	# are partial or empty (schema rejection, hook non-zero exit).
	echo "=== helm release: history + status ==="
	helm history $RONDB_CLUSTER_NAME -n $K8S_NAMESPACE \|\| true
	helm status $RONDB_CLUSTER_NAME -n $K8S_NAMESPACE \|\| true
	# Unbound PVCs surface only as Pending; not in pod logs.
	echo "=== persistent volume claims (binding state) ==="
	kubectl -n $K8S_NAMESPACE get pvc -o wide \|\| true
	for pvc in $(kubectl -n $K8S_NAMESPACE get pvc \
	-o jsonpath='{range .items[?(@.status.phase!="Bound")]}{.metadata.name} {end}'); do
	echo "--- pvc/$pvc (not Bound) ---"
	kubectl -n $K8S_NAMESPACE describe pvc "$pvc" \|\| true
	done
	# Cluster-wide RonDB events (data-node connect/disconnect,
	# arbitration decisions, GCP rounds) not visible in
	# per-container logs.
	echo "=== cluster.log tail (mgmd's authoritative cluster narrative) ==="
	kubectl -n $K8S_NAMESPACE exec mgmds-0 -c mgmd -- \
	tail -n 200 /srv/hops/mysql-cluster/log/cluster.log 2>&1 \|\| true

	# Snapshot the current cluster.log line count so the post-upgrade
	# assertion (A2) only inspects lines emitted during the upgrade
	# window — the install itself logs "Start phase 0 completed
	# (system restart)" as part of normal startup.
	- name: Snapshot cluster.log offset
	run: \|
	OFFSET=$(kubectl -n $K8S_NAMESPACE exec mgmds-0 -c mgmd -- \
	wc -l /srv/hops/mysql-cluster/log/cluster.log \| awk '{print $1}')
	echo "Captured cluster.log offset: $OFFSET"
	echo "$OFFSET" > /tmp/cluster-log-offset.txt

	# Re-supply install values on upgrade (no `--reuse-values`).
	# Per maintainer, Hopsworks customers always pass values files
	# on every upgrade. Practical effect: new-chart defaults are
	# merged in (filling schema-required fields the OLD chart
	# didn't ship, e.g. `meta.ddlMySQLd` from HWORKS-2753).
	- name: Upgrade to new chart (PR head)
	run: \|
	helm upgrade $RONDB_CLUSTER_NAME ./new-chart \
	--namespace=$K8S_NAMESPACE \
	--wait --timeout=15m \
	--values ./old-chart/values/minikube/small.yaml \
	--set staticCpuManagerPolicy=true \
	--set clusterSize.minNumMySQLServers=1 \
	--set clusterSize.maxNumMySQLServers=1 \
	--set rondbConfig.TotalMemoryConfig=4096M \
	--set resources.limits.memory.ndbmtdsMiB=5120 \
	--set backups.enabled=false

	# A1: NDB Error 2305 = "arbitrator decided to shutdown this node".
	# The exact failure mode the pre-upgrade hook prevents.
	- name: Assert no arbitration loss (Error 2305)
	run: \|
	fail=0
	# Refuse-to-pass on empty selector — guards against silent
	# pass if the chart ever renames the rondbService label.
	pods=$(kubectl -n $K8S_NAMESPACE get pods -l rondbService=ndbmtd \
	-o jsonpath='{.items[*].metadata.name}')
	if [ -z "$pods" ]; then
	echo "::error::No pods found with label rondbService=ndbmtd — chart may have renamed the label"
	exit 1
	fi
	for pod in $pods; do
	echo "Checking $pod for Error 2305"
	if kubectl -n $K8S_NAMESPACE exec "$pod" -c ndbmtd -- \
	bash -c 'grep -l "Error: 2305" /srv/hops/mysql-cluster/log/ndb_*_error.log 2>/dev/null'; then
	echo "::error::Pod $pod logged NDB Error 2305 (arbitration loss) during upgrade"
	kubectl -n $K8S_NAMESPACE exec "$pod" -c ndbmtd -- \
	bash -c 'grep -A2 -B2 "Error: 2305" /srv/hops/mysql-cluster/log/ndb_*_error.log' \|\| true
	fail=1
	fi
	done
	exit $fail

	# A2: a true system restart means every data node was down
	# simultaneously. The slice starts at the snapshotted offset so
	# we skip the install's own startup messages.
	#
	# Match `Start node: N using system restart` (MGMd's explicit
	# decision), NOT `Start phase 0 completed (system restart)` —
	# the latter is a placeholder logged on every ndbmtd boot
	# (including routine node-restarts) before the restart type is
	# decided at phase 2, so it false-positives on every successful
	# rolling upgrade.
	- name: Assert no system restart in upgrade window
	run: \|
	OFFSET=$(cat /tmp/cluster-log-offset.txt)
	slice=$(kubectl -n $K8S_NAMESPACE exec mgmds-0 -c mgmd -- \
	tail -n +$((OFFSET + 1)) /srv/hops/mysql-cluster/log/cluster.log)
	if echo "$slice" \| grep -E 'Start node: [0-9]+ using system restart'; then
	echo "::error::cluster.log shows a true system restart (lost quorum) during the upgrade window"
	# Full slice for inline diagnosis (parallel rolling vs.
	# hook-induced shutdown).
	echo "=== full cluster.log slice (lines after $OFFSET) ==="
	echo "$slice"
	exit 1
	fi
	echo "No true system-restart decisions in upgrade-window slice (lines after $OFFSET)."

	# On assertion failure, print pod-level evidence inline (restart
	# counts, lastState, ndbmtd-pod events, previous-pod logs, and
	# each StatefulSet's rendered podManagementPolicy +
	# updateStrategy) so diagnosis doesn't require an artifact pull.
	- name: Diagnostics on upgrade/assertion failure
	if: failure()
	run: \|
	echo "=== ndbmtd pod descriptions (restart count, lastState, terminated reason) ==="
	for pod in $(kubectl -n $K8S_NAMESPACE get pods -l rondbService=ndbmtd \
	-o jsonpath='{.items[*].metadata.name}'); do
	echo "--- $pod ---"
	kubectl -n $K8S_NAMESPACE describe pod "$pod" \| grep -E \
	'Restart Count\|Last State\|Reason\|Exit Code\|Started:\|Finished:\|^Events:\|^\s+(Normal\|Warning)' \|\| true
	done
	echo "=== events filtered to ndbmtd pods (creation order) ==="
	kubectl -n $K8S_NAMESPACE get events --sort-by='.metadata.creationTimestamp' \
	--field-selector involvedObject.kind=Pod 2>&1 \
	\| grep -E 'node-group\|ndbmtd' \|\| true
	echo "=== ndbmtd previous-pod logs (OLD-image pod's last 80 lines before kill) ==="
	for pod in $(kubectl -n $K8S_NAMESPACE get pods -l rondbService=ndbmtd \
	-o jsonpath='{.items[*].metadata.name}'); do
	echo "--- $pod / ndbmtd (previous) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c ndbmtd --previous --tail=80 2>&1 \|\| true
	done
	echo "=== ndbmtd current-pod logs (NEW-image pod's first 80 lines) ==="
	for pod in $(kubectl -n $K8S_NAMESPACE get pods -l rondbService=ndbmtd \
	-o jsonpath='{.items[*].metadata.name}'); do
	echo "--- $pod / ndbmtd (current) ---"
	kubectl -n $K8S_NAMESPACE logs "$pod" -c ndbmtd --tail=80 2>&1 \|\| true
	done
	echo "=== StatefulSet update spec (rendered podManagementPolicy + updateStrategy) ==="
	for sts in $(kubectl -n $K8S_NAMESPACE get statefulset \
	-o jsonpath='{.items[*].metadata.name}'); do
	echo "--- $sts ---"
	kubectl -n $K8S_NAMESPACE get statefulset "$sts" \
	-o jsonpath='podManagementPolicy={.spec.podManagementPolicy} updateStrategy={.spec.updateStrategy}{"\n"}' \
	2>&1 \|\| true
	done

	# A3: sanity check that the upgrade actually touched MGMd's image,
	# so A1/A2 don't pass for the wrong reason. The `:24.10.19` is
	# paired with the OLD-chart pin (`ref: branch-0.7`); update both
	# together if you ever change which OLD chart this test runs against.
	- name: Assert MGMd image was upgraded
	run: \|
	image=$(kubectl -n $K8S_NAMESPACE get sts mgmds \
	-o jsonpath='{.spec.template.spec.containers[?(@.name=="mgmd")].image}')
	echo "MGMd image after upgrade: $image"
	if [[ "$image" == *":24.10.19" ]]; then
	echo "::error::MGMd image still at 24.10.19 — upgrade did not roll the StatefulSet"
	exit 1
	fi

	- name: Collect logs
	if: always()
	uses: ./new-chart/.github/actions/collect_logs
	with:
	namespace: ${{ env.K8S_NAMESPACE }}
	folder_name: cross_version_logs

	- name: Remove cluster
	if: always()
	uses: ./new-chart/.github/actions/remove_cluster
	timeout-minutes: 4
	with:
	namespace: ${{ env.K8S_NAMESPACE }}
	helm_chart: ${{ env.RONDB_CLUSTER_NAME }}

Navigation Menu

Uh oh!

[RONDB-1083] Make startupProbe timeouts configurable for ndbmtd and M… #475

Workflow file

[RONDB-1083] Make startupProbe timeouts configurable for ndbmtd and M… #475

Uh oh!

Workflow file for this run