chore: reorganize KIND test environments and cleanup obsolete scripts

- Create kind-provers.env (real provers, matches next-scenario.env)
- Create kind-minimal.env (fast epochs + fake provers for quick iteration)
- Remove kind-test.env (replaced by above)
- Remove test_k8s.sh and test_prod_deployment.sh (obsolete)
- Remove obsolete bootstrap.sh test commands that used test_k8s.sh
- Update test_kind.sh to require env sourcing by caller
- Filter eth-beacon/eth-execution from stern logs

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: ludamad

ci.sh

@@ -181,7 +181,7 @@ case "$cmd" in
     export CI_DASHBOARD="network"
     export JOB_ID="x-network-tests-kind-${1:?test_command is required}"
     export AWS_SHUTDOWN_TIME=180 # 3 hours for KIND tests
-    export CPUS=64
+    export CPUS=192
     export INSTANCE_POSTFIX="n-kind"
     bootstrap_ec2 "./bootstrap.sh ci-network-tests-kind $*"
     ;;

spartan/.gitignore

@@ -24,5 +24,6 @@ environments/*
 !environments/testnet-canary.env
 !environments/testnet.env
 !environments/tps-scenario.env
-!environments/kind-test.env
+!environments/kind-minimal.env
+!environments/kind-provers.env
 *.tfvars

spartan/bootstrap.sh

@@ -282,97 +282,13 @@ case "$cmd" in
   test|test_cmds|gke|build|gcp_auth)
     $cmd
     ;;
-  "test-kind-smoke")
-    OVERRIDES="telemetry.enabled=false,bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh kind src/spartan/smoke.test.ts 1-validators.yaml smoke${NAME_POSTFIX:-}
-    ;;
-  "test-kind-4epochs")
-    # TODO(#12163) reenable bot once not conflicting with transfer
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh kind src/spartan/4epochs.test.ts ci.yaml four-epochs${NAME_POSTFIX:-}
-    ;;
-  "test-kind-4epochs-sepolia")
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false SEPOLIA_RUN=true \
-      ./scripts/test_k8s.sh kind src/spartan/4epochs.test.ts ci-sepolia.yaml four-epochs${NAME_POSTFIX:-}
-    ;;
-  "test-kind-proving")
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh kind src/spartan/proving.test.ts ci.yaml proving${NAME_POSTFIX:-}
-    ;;
-  "test-kind-transfer")
-    # TODO(#12163) reenable bot once not conflicting with transfer
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh kind src/spartan/transfer.test.ts ci.yaml transfer${NAME_POSTFIX:-}
-    ;;
-  "test-kind-1tps")
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false RESOURCES_FILE=gcloud-1tps-sim.yaml \
-      ./scripts/test_k8s.sh kind src/spartan/1tps.test.ts ci-1tps.yaml one-tps${NAME_POSTFIX:-}
-    ;;
-  "test-kind-10tps-10%-drop")
-    OVERRIDES="telemetry.enabled=false,bot.enabled=false,validator.p2p.dropTransactions=true,validator.p2p.dropTransactionsProbability=0.1" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-    ./scripts/test_k8s.sh kind src/spartan/n_tps.test.ts ci-1tps.yaml ten-tps${NAME_POSTFIX:-}
-  ;;
-  "test-kind-10tps-30%-drop")
-    OVERRIDES="telemetry.enabled=false,bot.enabled=false,validator.p2p.dropTransactions=true,validator.p2p.dropTransactionsProbability=0.3" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-    ./scripts/test_k8s.sh kind src/spartan/n_tps.test.ts ci-tx-drop.yaml ten-tps${NAME_POSTFIX:-}
-  ;;
-  "test-kind-10tps-50%-drop")
-    OVERRIDES="telemetry.enabled=false,bot.enabled=false,validator.p2p.dropTransactions=true,validator.p2p.dropTransactionsProbability=0.5" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-    ./scripts/test_k8s.sh kind src/spartan/n_tps.test.ts ci-tx-drop.yaml ten-tps${NAME_POSTFIX:-}
-  ;;
   "test-kind-upgrade-rollup")
+    source scripts/source_network_env.sh
+    source_network_env kind-provers
     OVERRIDES="${OVERRIDES:-bot.enabled=false}" \
     FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
       ./scripts/test_kind.sh src/spartan/upgrade_via_cli.test.ts upgrade-rollup-version${NAME_POSTFIX:-}
     ;;
-  "test-prod-deployment")
-    FRESH_INSTALL=false INSTALL_METRICS=false ./scripts/test_prod_deployment.sh
-    ;;
-  "test-cli-upgrade")
-    OVERRIDES="telemetry.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh kind src/spartan/upgrade_via_cli.test.ts 1-validators.yaml upgrade-via-cli${NAME_POSTFIX:-}
-    ;;
-  "test-gke-transfer")
-    execution_client="$1"
-    # TODO(#12163) reenable bot once not conflicting with transfer
-    OVERRIDES="bot.enabled=false"
-    if [ -n "$execution_client" ]; then
-      OVERRIDES="$OVERRIDES,ethereum.execution.client=$execution_client"
-    fi
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false RESOURCES_FILE=gcloud-1tps-sim.yaml  \
-      ./scripts/test_k8s.sh gke src/spartan/transfer.test.ts ci-fast-epoch.yaml ${NAMESPACE:-"transfer${NAME_POSTFIX:-}"}
-    ;;
-  "test-gke-1tps")
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false RESOURCES_FILE=gcloud-1tps-sim.yaml \
-      ./scripts/test_k8s.sh gke src/spartan/1tps.test.ts ci-1tps.yaml ${NAMESPACE:-"one-tps${NAME_POSTFIX:-}"}
-    ;;
-  "test-gke-4epochs")
-    # TODO(#12163) reenable bot once not conflicting with transfer
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh gke src/spartan/4epochs.test.ts ci-1tps.yaml ${NAMESPACE:-"four-epochs${NAME_POSTFIX:-}"}
-    ;;
-  "test-gke-upgrade-rollup-version")
-    OVERRIDES="bot.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh gke src/spartan/upgrade_rollup_version.test.ts ci.yaml ${NAMESPACE:-"upgrade-rollup-version${NAME_POSTFIX:-}"}
-    ;;
-  "test-gke-cli-upgrade")
-    OVERRIDES="telemetry.enabled=false" \
-    FRESH_INSTALL=${FRESH_INSTALL:-true} INSTALL_METRICS=false \
-      ./scripts/test_k8s.sh gke src/spartan/upgrade_via_cli.test.ts 1-validators.yaml ${NAMESPACE:-"upgrade-via-cli${NAME_POSTFIX:-}"}
-    ;;
   "network_teardown")
     env_file="$1"
     # Sets up basic env vars like CLUSTER for gcp auth

spartan/environments/kind-test.env spartan/environments/kind-minimal.envspartan/environments/kind-test.env renamed to spartan/environments/kind-minimal.env

@@ -1,5 +1,5 @@
 # KIND environment for local Kubernetes testing
-# Uses minimal resource settings suitable for local KIND clusters
+# Minimal setup with fast epochs for quick iteration
 
 NAMESPACE=${NAMESPACE:-kind}
 CLUSTER=kind
@@ -9,12 +9,12 @@ CREATE_AZTEC_INFRA=true
 LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
 L1_ACCOUNT_MNEMONIC="test test test test test test test test test test test junk"
 FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
-REAL_VERIFIER=true
+REAL_VERIFIER=false
 SENTINEL_ENABLED=false
 
-# Epoch and slot timing
-AZTEC_EPOCH_DURATION=4
+# Fast epoch timing for quick iteration
 AZTEC_SLOT_DURATION=24
+AZTEC_EPOCH_DURATION=4
 AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=2
 AZTEC_SLASHING_EXECUTION_DELAY_IN_ROUNDS=1
 AZTEC_SLASHING_OFFSET_IN_ROUNDS=1
@@ -54,6 +54,3 @@ PUBLISHERS_PER_PROVER=1
 
 # RPC
 RPC_REPLICAS=1
-
-# DO NOT auto-run tests - we want to run manually
-RUN_TESTS=false

spartan/environments/kind-provers.env

@@ -0,0 +1,54 @@
+# KIND environment for local Kubernetes testing with real provers
+# Uses settings from next-scenario.env, adapted for KIND
+
+NAMESPACE=${NAMESPACE:-kind}
+CLUSTER=kind
+CREATE_ETH_DEVNET=true
+CREATE_ROLLUP_CONTRACTS=true
+CREATE_AZTEC_INFRA=true
+LABS_INFRA_MNEMONIC="test test test test test test test test test test test junk"
+L1_ACCOUNT_MNEMONIC="test test test test test test test test test test test junk"
+FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
+SENTINEL_ENABLED=false
+
+# Epoch and slot timing (from next-scenario.env)
+AZTEC_EPOCH_DURATION=32
+AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2
+AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2
+
+# Slashing settings (from next-scenario.env)
+AZTEC_SLASHING_ROUND_SIZE_IN_EPOCHS=1
+AZTEC_SLASHING_QUORUM=17
+AZTEC_SLASHING_OFFSET_IN_ROUNDS=2
+AZTEC_LOCAL_EJECTION_THRESHOLD=90000000000000000000
+
+# Governance settings (from next-scenario.env)
+AZTEC_GOVERNANCE_PROPOSER_QUORUM=11
+AZTEC_GOVERNANCE_PROPOSER_ROUND_SIZE=20
+AZTEC_GOVERNANCE_VOTING_DURATION=300
+
+R2_ACCESS_KEY_ID=""
+R2_SECRET_ACCESS_KEY=""
+
+OTEL_COLLECTOR_ENDPOINT="http://metrics-opentelemetry-collector.metrics:4318"
+
+# Docker image - use AZTEC_DOCKER_IMAGE if already set (e.g., by CI), otherwise default
+AZTEC_DOCKER_IMAGE=${AZTEC_DOCKER_IMAGE:-aztecprotocol/aztec:latest}
+
+# Validators (from next-scenario.env)
+VALIDATOR_REPLICAS=4
+VALIDATORS_PER_NODE=12
+PUBLISHERS_PER_VALIDATOR_KEY=1
+VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000
+
+# Provers (from next-scenario.env)
+PROVER_REPLICAS=8
+PROVER_AGENTS_PER_PROVER=1
+PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000
+PUBLISHERS_PER_PROVER=1
+
+# RPC
+RPC_REPLICAS=2
+
+# DO NOT auto-run tests - we want to run manually
+RUN_TESTS=false

spartan/scripts/test_kind.sh

@@ -1,7 +1,12 @@
 #!/usr/bin/env bash
-# Usage: ./test_kind.sh <test_file> [namespace]
+# Usage: source environments/kind-provers.env && ./test_kind.sh <test_file> [namespace]
 # Deploys a network to KIND and runs the specified test.
 #
+# Prerequisites:
+#   Source the appropriate env file before running:
+#   - kind-minimal.env: Fast testing with fake provers
+#   - kind-provers.env: Real provers (slower, matches next-scenario.env)
+#
 # Environment variables:
 #   FRESH_INSTALL (default: "true") - Delete namespace before deployment
 #   OVERRIDES (default: "") - Helm value overrides
@@ -18,10 +23,6 @@ source "${REPO_ROOT}/ci3/source"
 
 cd "$SPARTAN_DIR"
 
-# Source the KIND test environment configuration
-source scripts/source_network_env.sh
-source_network_env kind-test
-
 test_file="${1:?test_file is required}"
 namespace="${2:-upgrade-test}"
 
@@ -32,7 +33,7 @@ overrides="${OVERRIDES:-}"
 # Ensure KIND cluster is running
 ./bootstrap.sh kind
 
-# Set up namespace (override NAMESPACE from kind-test.env with specific test namespace)
+# Set up namespace (override NAMESPACE from env file with specific test namespace)
 export NAMESPACE="$namespace"
 export CLUSTER="kind"
 
@@ -46,16 +47,29 @@ if [ "$install_metrics" = "true" ]; then
   ./bootstrap.sh metrics-kind
 fi
 
-# Capture logs
-mkdir -p scripts/logs
+# Capture pod logs via stern, streaming directly to cache_log
 stern_pid=""
+
+function start_stern {
+  if command -v stern &>/dev/null; then
+    echo "Starting stern log capture for namespace $namespace (streaming to cache_log)..."
+    # Pipe stern directly to cache_log for live streaming to Redis
+    # Exclude noisy eth-beacon and eth-execution pods
+    stern ".*" -n "$namespace" --exclude-pod="eth-beacon.*" --exclude-pod="eth-execution.*" --since=1s 2>&1 | cache_log "kind-$namespace-pods" &
+    stern_pid=$!
+    echo "Stern streaming started with PID $stern_pid"
+  else
+    echo "Warning: stern not installed, pod logs will not be captured"
+  fi
+}
+
 function cleanup {
   set +e
   if [ -n "$stern_pid" ]; then
+    echo "Stopping stern (PID $stern_pid)..."
     kill "$stern_pid" 2>/dev/null || true
+    wait "$stern_pid" 2>/dev/null || true
   fi
-  # Upload logs
-  (cat "scripts/logs/kind-$namespace.log" 2>/dev/null || true) | cache_log "kind test $test_file" || true
 
   if [ "$fresh_install" = "true" ]; then
     # Use kind_teardown.sh for comprehensive cleanup (namespace + terraform state)
@@ -64,12 +78,6 @@ function cleanup {
 }
 trap cleanup EXIT INT TERM
 
-# Start stern to capture logs
-if command -v stern &>/dev/null; then
-  stern ".*" -n "$namespace" > "scripts/logs/kind-$namespace.log" 2>&1 &
-  stern_pid=$!
-fi
-
 # Deploy the network
 echo "Deploying network to KIND namespace: $namespace"
 
@@ -81,6 +89,9 @@ fi
 # Run the deployment
 ./scripts/deploy_network.sh
 
+# Start stern AFTER namespace exists to capture pod logs
+start_stern
+
 # Wait for validator pods to be ready
 echo "Waiting for validator pods to be ready..."
 kubectl wait pod -l app.kubernetes.io/component=sequencer-node --for=condition=Ready -n "$namespace" --timeout=15m || true