switch to stern for log collection. Also tar.gz the file (#16167)

* use stern for capturing logs

* Update test/e2e-common.sh

Co-authored-by: Nader Ziada <[email protected]>

* don't mkdir -p on every logline

---------

Co-authored-by: Nader Ziada <[email protected]>

Author: dprotaso

test/e2e-common.sh

@@ -49,6 +49,7 @@ export YTT_FILES=()
 export TMP_DIR="${TMP_DIR:-$(mktemp -d -t ci-$(date +%Y-%m-%d-%H-%M-%S)-XXXXXXXXXX)}"
 
 readonly E2E_YAML_DIR=${E2E_YAML_DIR:-"${TMP_DIR}/e2e-yaml"}
+readonly STERN_PID_FILE=${TMP_DIR}/stern.pid
 
 # This the namespace used to install Knative Serving. Use generated UUID as namespace.
 export SYSTEM_NAMESPACE="${SYSTEM_NAMESPACE:-$(uuidgen | tr 'A-Z' 'a-z')}"
@@ -422,21 +423,59 @@ function knative_teardown() {
 
 # Create test resources and images
 function test_setup() {
+  echo ">> Uploading test images..."
+  ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1
+
   echo ">> Setting up logging..."
 
-  # Install kail if needed.
-  if ! which kail > /dev/null; then
-    go install github.com/boz/kail/cmd/[email protected]
+  # Install stern if needed.
+  if ! which stern > /dev/null; then
+    go install github.com/stern/[email protected]
   fi
 
+  mkdir -p $(log_dir)
+
   # Capture all logs.
-  kail > "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt" &
-  local kail_pid=$!
-  # Clean up kail so it doesn't interfere with job shutting down
-  add_trap "kill $kail_pid || true" EXIT
+  # We use stdbuf so that when we pipe logs to awk it flushes to the file
+  stdbuf -oL stern . --all-namespaces --timestamps --max-log-requests 2000 2> "$(log_dir)/stern.stderr.log" | \
+  stdbuf -oL awk -v logdir="$(log_dir)" '
+  {
+    ns=$1;
+    pod=$2;
 
-  echo ">> Uploading test images..."
-  ${REPO_ROOT_DIR}/test/upload-test-images.sh || return 1
+    gsub("/", "_", ns);
+    gsub("/", "_", pod);
+
+    dir = logdir "/" ns
+
+    if (!(dir in seen_dirs)) {
+        if (system("[ -d \"" dir "\" ] || mkdir -p \"" dir "\"") != 0) {
+            print "Failed to create dir:", dir > "/dev/stderr"
+        }
+        seen_dirs[dir] = 1
+    }
+
+    print $0 >> (logdir "/" ns "/" pod ".log");
+    fflush(logdir "/" ns "/" pod ".log");
+  }' &
+
+  echo $! > ${STERN_PID_FILE}
+
+  # Clean up stern so it doesn't interfere with job shutting down
+  add_trap "stop_logging" EXIT SIGTERM
+}
+
+function stop_logging() {
+  kill $(cat ${STERN_PID_FILE})
+
+  pushd "$(dirname $(log_dir))"
+    tar -czf "$(basename $(log_dir)).tar.gz" "$(basename $(log_dir))"
+    rm -rf "$(basename $(log_dir))"
+  popd
+}
+
+function log_dir() {
+  echo "${ARTIFACTS}/log-$(basename "${E2E_SCRIPT}" .sh)"
 }
 
 # Dump more information when test fails.
@@ -635,3 +674,7 @@ function run_ytt() {
 function run_kapp() {
   go_run github.com/vmware-tanzu/carvel-kapp/cmd/[email protected] "$@"
 }
+
+function on_success() {
+  rm "${ARTIFACTS}"/log-*
+}

test/e2e-external-domain-tls-tests.sh

@@ -211,7 +211,4 @@ fi
 subheader "Cleanup external domain tls"
 cleanup_external_domain_tls_common
 
-# Remove the kail log file if the test flow passes.
-# This is for preventing too many large log files to be uploaded to GCS in CI.
-rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt"
 success

test/e2e-tests.sh

@@ -170,8 +170,4 @@ fi
 
 (( failed )) && fail_test
 
-# Remove the kail log file if the test flow passes.
-# This is for preventing too many large log files to be uploaded to GCS in CI.
-rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt"
-
 success

test/e2e-upgrade-tests.sh

@@ -54,7 +54,4 @@ go_test_e2e -tags=upgrade -timeout=${TIMEOUT} \
   --disable-logstream \
   --resolvabledomain=$(use_resolvable_domain) || fail_test
 
-# Remove the kail log file if the test flow passes.
-# This is for preventing too many large log files to be uploaded to GCS in CI.
-rm "${ARTIFACTS}/k8s.log-$(basename "${E2E_SCRIPT}").txt"
 success

test/presubmit-tests.sh

@@ -21,9 +21,6 @@
 # Use the flags --build-tests, --unit-tests and --integration-tests
 # to run a specific set of tests.
 
-export GO111MODULE=on
-export GOFLAGS=-mod=vendor
-
 source $(dirname $0)/../vendor/knative.dev/hack/presubmit-tests.sh
 
 # We use the default build, unit and integration test runners.