From 6ae088f7e08c4b3577db150224ec6ed31a8fa13a Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Mon, 20 Oct 2025 21:54:16 -0400 Subject: [PATCH 1/3] use stern for capturing logs --- test/e2e-common.sh | 49 ++++++++++++++++++++++----- test/e2e-external-domain-tls-tests.sh | 3 -- test/e2e-tests.sh | 4 --- test/e2e-upgrade-tests.sh | 3 -- test/presubmit-tests.sh | 3 -- 5 files changed, 41 insertions(+), 21 deletions(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 369a663fe170..21cde0cfeace 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -49,6 +49,7 @@ export YTT_FILES=() export TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX)}" readonly E2E_YAML_DIR=${E2E_YAML_DIR:-"${TMP_DIR}/e2e-yaml"} +readonly STERN_PID_FILE=${TMP_DIR}/stern.pid # This the namespace used to install Knative Serving. Use generated UUID as namespace. export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}" @@ -422,21 +423,49 @@ function knative_teardown() { # Create test resources and images function test_setup() { + echo ">> Uploading test images..." + ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1 + echo ">> Setting up logging..." # Install kail if needed. - if ! which kail > /dev/null; then - go install github.com/boz/kail/cmd/kail@v0.17.4 + if ! which stern > /dev/null; then + go install github.com/stern/stern@v1.33.0 fi + mkdir -p $(log_dir) + # Capture all logs. - kail > "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt" & - local kail_pid=$! - # Clean up kail so it doesn't interfere with job shutting down - add_trap "kill $kail_pid || true" EXIT + # We use stdbuf so that when we pipe logs to awk it flushes to the file + stdbuf -oL stern . --all-namespaces --timestamps --max-log-requests 2000 2> "$(log_dir)/stern.stderr.log" | \ + stdbuf -oL awk -v logdir="$(log_dir)" ' + { + ns=$1; + pod=$2; + gsub("/", "_", ns); + gsub("/", "_", pod); + system("mkdir -p " logdir "/" ns); + print $0 >> (logdir "/" ns "/" pod ".log"); + fflush(logdir "/" ns "/" pod ".log"); + }' & + + echo $! > ${STERN_PID_FILE} + + # Clean up stern so it doesn't interfere with job shutting down + add_trap "stop_logging" EXIT SIGTERM +} - echo ">> Uploading test images..." - ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1 +function stop_logging() { + kill $(cat ${STERN_PID_FILE}) + + pushd "$(dirname $(log_dir))" + tar -czf "$(basename $(log_dir)).tar.gz" "$(basename $(log_dir))" + rm -rf "$(basename $(log_dir))" + popd +} + +function log_dir() { + echo "${ARTIFACTS}/log-$(basename "${E2E_SCRIPT}" .sh)" } # Dump more information when test fails. @@ -635,3 +664,7 @@ function run_ytt() { function run_kapp() { go_run github.com/vmware-tanzu/carvel-kapp/cmd/kapp@v0.60.0 "$@" } + +function on_success() { + rm "${ARTIFACTS}"/log-* +} diff --git a/test/e2e-external-domain-tls-tests.sh b/test/e2e-external-domain-tls-tests.sh index b83c8b71b9dd..bd03a3923667 100755 --- a/test/e2e-external-domain-tls-tests.sh +++ b/test/e2e-external-domain-tls-tests.sh @@ -211,7 +211,4 @@ fi subheader "Cleanup external domain tls" cleanup_external_domain_tls_common -# Remove the kail log file if the test flow passes. -# This is for preventing too many large log files to be uploaded to GCS in CI. -rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt" success diff --git a/test/e2e-tests.sh b/test/e2e-tests.sh index e04724ebe121..372a12ed3479 100755 --- a/test/e2e-tests.sh +++ b/test/e2e-tests.sh @@ -170,8 +170,4 @@ fi (( failed )) && fail_test -# Remove the kail log file if the test flow passes. -# This is for preventing too many large log files to be uploaded to GCS in CI. -rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt" - success diff --git a/test/e2e-upgrade-tests.sh b/test/e2e-upgrade-tests.sh index 505c2f9ae5fb..1f853eada227 100755 --- a/test/e2e-upgrade-tests.sh +++ b/test/e2e-upgrade-tests.sh @@ -54,7 +54,4 @@ go_test_e2e -tags=upgrade -timeout=${TIMEOUT} \ --disable-logstream \ --resolvabledomain=$(use_resolvable_domain) || fail_test -# Remove the kail log file if the test flow passes. -# This is for preventing too many large log files to be uploaded to GCS in CI. -rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt" success diff --git a/test/presubmit-tests.sh b/test/presubmit-tests.sh index e47a58cce099..dd76d167fe32 100755 --- a/test/presubmit-tests.sh +++ b/test/presubmit-tests.sh @@ -21,9 +21,6 @@ # Use the flags --build-tests, --unit-tests and --integration-tests # to run a specific set of tests. -export GO111MODULE=on -export GOFLAGS=-mod=vendor - source $(dirname $0)/../vendor/knative.dev/hack/presubmit-tests.sh # We use the default build, unit and integration test runners. From 927c496b3c117347ddcbbe9e96cf9fcd61f5c1d4 Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Tue, 21 Oct 2025 17:56:23 -0400 Subject: [PATCH 2/3] Update test/e2e-common.sh Co-authored-by: Nader Ziada --- test/e2e-common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 21cde0cfeace..71dc4ab81435 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -428,7 +428,7 @@ function test_setup() { echo ">> Setting up logging..." - # Install kail if needed. + # Install stern if needed. if ! which stern > /dev/null; then go install github.com/stern/stern@v1.33.0 fi From aceee4eea34d2c5bdb143d9d0a053e0617c54b3f Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Tue, 21 Oct 2025 18:14:55 -0400 Subject: [PATCH 3/3] don't mkdir -p on every logline --- test/e2e-common.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/e2e-common.sh b/test/e2e-common.sh index 71dc4ab81435..c311e15b800f 100644 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -442,9 +442,19 @@ function test_setup() { { ns=$1; pod=$2; + gsub("/", "_", ns); gsub("/", "_", pod); - system("mkdir -p " logdir "/" ns); + + dir = logdir "/" ns + + if (!(dir in seen_dirs)) { + if (system("[ -d \"" dir "\" ] || mkdir -p \"" dir "\"") != 0) { + print "Failed to create dir:", dir > "/dev/stderr" + } + seen_dirs[dir] = 1 + } + print $0 >> (logdir "/" ns "/" pod ".log"); fflush(logdir "/" ns "/" pod ".log"); }' &