Skip to content

OLS-2316 - Adding section about reporting periodic tests #592

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JoaoFula wants to merge 1 commit into
base: main
Choose a base branch
from
+747 −0
Open

OLS-2316 - Adding section about reporting periodic tests #592

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
366 changes: 366 additions & 0 deletions modules/testing/examples/reporting.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,366 @@
---
apiVersion: tekton.dev/v1beta1
kind: Pipeline
metadata:
name: ols-console-reporting # <1>
spec:
description: |
This pipeline is run weekly to gather the results of the periodic console test runs and email them to stakeholders.
params:
- name: namespace # <2>
description: 'Namespace where the pipelineRuns are run'
default: 'crt-nshift-lightspeed-tenant'
- name: application # <3>
description: 'Name of application where the tests run'
default: 'ols'
- name: tests-names # <4>
type: array
description: 'List of test names to check (e.g., pf5, pf6)'
default:
- 'e2e-console-pf5'
- 'e2e-console-pf6'
- name: tests-times # <5>
type: array
description: 'List of scheduled run times for the tests (corresponding to tests-names)'
default:
- '3:00 AM'
- '0:00 AM'
- name: stakeholder-emails # <6>
description: 'Comma-separated list of email addresses to send the report to'
default: 'openshift-lightspeed@redhat.com'
- name: subject # <7>
description: 'Subject of the email'
type: string
default: 'Weekly konflux test reporting'
- name: KONFLUX_UI_URL # <8>
description: 'URL for your konflux UI instance'
default: 'https://konflux-ui.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com'
- name: SMTP_SERVER # <9>
description: 'SMTP server to be used for sending the email'
default: 'smtp.gmail.com:587'
tasks:
- name: extract-tests-info-and-send-email
description: Check PipelineRuns for configured tests and gather results
params:
- name: SNAPSHOT
value: $(params.SNAPSHOT)
- name: NAMESPACE
value: $(params.namespace)
- name: APPLICATION
value: $(params.application)
- name: TESTS_NAMES
value: $(params.tests-names[*])
- name: TESTS_TIMES
value: $(params.tests-times[*])
- name: STAKEHOLDER-EMAILS
value: $(params.stakeholder-emails)
- name: SUBJECT
value: $(params.subject)
- name: KONFLUX_UI_URL
value: $(params.KONFLUX_UI_URL)
- name: SMTP_SERVER
value: $(params.SMTP_SERVER)
taskSpec:
params:
- name: SNAPSHOT
- name: NAMESPACE
- name: APPLICATION
- name: TESTS_NAMES
type: array
- name: TESTS_TIMES
type: array
- name: STAKEHOLDER-EMAILS
- name: SUBJECT
- name: KONFLUX_UI_URL
- name: SMTP_SERVER
steps:
- name: get-tests-info-and-send-email
image: registry.redhat.io/ubi9/ubi:latest
env:
- name: SNAPSHOT
value: $(params.SNAPSHOT)
- name: NAMESPACE
value: $(params.NAMESPACE)
- name: APPLICATION
value: $(params.APPLICATION)
- name: KONFLUX_TOKEN
valueFrom:
secretKeyRef:
name: release-bot-konflux-token # <10>
key: konflux-token
- name: STAKEHOLDER_EMAILS
value: $(params.STAKEHOLDER-EMAILS)
- name: INITIAL_SUBJECT
value: $(params.SUBJECT)
- name: KONFLUX_UI_URL
value: $(params.KONFLUX_UI_URL)
- name: SMTP_SERVER
value: $(params.SMTP_SERVER)
- name: USERNAME
valueFrom:
secretKeyRef:
name: gwsa-email-account # <11>
key: username
- name: PASSWORD
valueFrom:
secretKeyRef:
name: gwsa-email-account
key: password
args:
- $(params.TESTS_NAMES[*])
- $(params.TESTS_TIMES[*])
script: |
#!/bin/bash
set -euo pipefail
echo "Checking PipelineRuns in namespace: $NAMESPACE" >&2

# Read arguments - first half are test names, second half are test times
ALL_ARGS=("$@")
NUM_TESTS=$((${#ALL_ARGS[@]} / 2))

# Split args into names and times arrays
NAMES_ARRAY=("${ALL_ARGS[@]:0:$NUM_TESTS}")
TIMES_ARRAY=("${ALL_ARGS[@]:$NUM_TESTS:$NUM_TESTS}")

echo "Tests to check:" >&2
for i in "${!NAMES_ARRAY[@]}"; do
echo " - ${NAMES_ARRAY[$i]} (scheduled at ${TIMES_ARRAY[$i]})"
done
echo "---" >&2
dnf -y install jq wget s-nail
wget --no-verbose -O oc.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux.tar.gz \
&& tar -xvzf oc.tar.gz \
&& chmod +x kubectl oc \
&& mv oc kubectl /usr/local/bin/
curl -LO https://github.com/kubearchive/kubearchive/releases/latest/download/kubectl-ka-linux-amd64 \
&& chmod +x kubectl-ka-* \
&& mv kubectl-ka-linux-amd64 /usr/local/bin/kubectl-ka

echo "Logging into cluster with sa token and configuring host for kubearchive" >&2
oc login --token="${KONFLUX_TOKEN}" --server=https://api.stone-prd-rh01.pg1f.p1.openshiftapps.com:6443
kubectl ka config set host https://kubearchive-api-server-product-kubearchive.apps.stone-prd-rh01.pg1f.p1.openshiftapps.com

# Function to check PipelineRuns for a given prefix
check_pipelineruns() {
local prefix=$1
local scheduled_time=$2
local test_name=$3

echo "Checking PipelineRuns from kubearchive for prefix: $prefix" >&2

# Parse the scheduled time (e.g., "3:00 AM" -> 03:00 in 24h format)
local scheduled_hour=$(date -d "$scheduled_time" +%H 2>/dev/null || echo "00")
local scheduled_minute=$(date -d "$scheduled_time" +%M 2>/dev/null || echo "00")
local scheduled_minutes_total=$((10#$scheduled_hour * 60 + 10#$scheduled_minute))

echo "Looking for runs at approximately $scheduled_time (${scheduled_hour}:${scheduled_minute} UTC, +/- 2 minutes)" >&2

# Get the most recent PipelineRuns matching the prefix (last 7 days) from kubearchive
# Query each day separately to avoid the 100 result limit
local all_pipelineruns=""

# Query pipelineruns for each of the last 7 days at 6AM
for day_offset in {0..6}; do
# Calculate the timestamp for 6AM on this day
local before_timestamp=$(date -u -d "${day_offset} days ago 06:00:00" +"%Y-%m-%dT06:00:00.000000000Z")

echo "Querying pipelineruns before: $before_timestamp (day -$day_offset)" >&2

local day_pipelineruns=$(kubectl ka get pipelinerun -n "$NAMESPACE" \
--before "$before_timestamp" \
--archived \
-o json | jq -c --arg prefix "$prefix" \
--argjson target_minutes "$scheduled_minutes_total" \
'limit(1; .items[] | select(.metadata.name | startswith($prefix)) |
. as $item |
($item.metadata.creationTimestamp[11:16] | split(":") | (.[0] | tonumber) * 60 + (.[1] | tonumber)) as $run_minutes |
(if ($run_minutes - $target_minutes) < 0 then ($target_minutes - $run_minutes) else ($run_minutes - $target_minutes) end) as $diff |
select($diff <= 2) |
{name: .metadata.name,
status: (.status.conditions[0].status // "Unknown"),
reason: (.status.conditions[0].reason // "Unknown"),
creationTime: .metadata.creationTimestamp})')

# Append to all_pipelineruns if not empty
if [ -n "$day_pipelineruns" ]; then
all_pipelineruns="${all_pipelineruns}${day_pipelineruns}"$'\n'
fi
done

echo "$all_pipelineruns" >&2
if [ -z "$all_pipelineruns" ]; then
echo "No recent PipelineRuns found for $prefix" >&2
echo "{\"test\": \"$test_name\", \"scheduled_time\": \"$scheduled_time\", \"status\": \"NOT_FOUND\", \"runs\": []}"
return
fi

# Process all PipelineRuns in a single jq command (more efficient than looping)
local results=$(echo "$all_pipelineruns" | jq -s --arg namespace "$NAMESPACE" --arg app "$APPLICATION" '
[.[] | select(length > 0) |
{
name: .name,
status: (if (.status == "True" and .reason == "Succeeded") then "SUCCEEDED"
elif (.status == "False" and .reason == "Failed") then "FAILED"
elif (.reason == "Running" or .reason == "Started") then "RUNNING"
else "UNKNOWN" end),
created: .creationTime,
log_link: "${KONFLUX_UI_URL}/ns/\($namespace)/applications/\($app)/pipelineruns/\(.name)"
}]')

# Output debug info to stderr
echo "$results" | jq -r '.[] | " - \(.name): \(.status) (Created: \(.created))\n Logs: \(.log_link)"' >&2

# Return summary
echo "$results" | jq --arg test "$test_name" \
--arg time "$scheduled_time" \
'{test: $test, scheduled_time: $time, runs: .}'
echo "---" >&2
}

# Check all test runs and collect results
TEST_RESULTS="[]"
for i in "${!NAMES_ARRAY[@]}"; do
TEST_NAME="${NAMES_ARRAY[$i]}"
TEST_TIME="${TIMES_ARRAY[$i]}"
TEST_LABEL="Test-$((i+1))"

echo "=== $TEST_LABEL Results ($TEST_NAME) ===" >&2
RESULT=$(check_pipelineruns "$TEST_NAME" "$TEST_TIME" "$TEST_NAME")

# Add result to array
TEST_RESULTS=$(echo "$TEST_RESULTS" | jq --argjson result "$RESULT" '. += [$result]')
done

# Write combined results to task result
echo "$TEST_RESULTS" | jq -c '.'

echo "Report generation complete" >&2

echo "========================================================" >&2

echo "=== Starting Email Report Generation ===" >&2
Comment thread
JoaoFula marked this conversation as resolved.
echo "Stakeholders: $STAKEHOLDER_EMAILS" >&2

echo "Preparing email report for stakeholders: $STAKEHOLDER_EMAILS" >&2

# Parse results
echo "Parsing test results..." >&2
echo "Test Results Data (length: ${#TEST_RESULTS} chars):" >&2
echo "$TEST_RESULTS" | jq '.' >&2 || echo "Failed to parse test results JSON" >&2

# Get the number of test results
NUM_TESTS=$(echo "$TEST_RESULTS" | jq 'length // 0')
echo "Number of test results: $NUM_TESTS" >&2

# Function to format test results as HTML
format_results_html() {
local data="$1"
local test_name=$(echo "$data" | jq -r '.test // "Unknown"')
local scheduled_time=$(echo "$data" | jq -r '.scheduled_time // "Unknown"')
local runs_count=$(echo "$data" | jq '.runs | length // 0')

echo "<h2>$test_name Tests (Scheduled at $scheduled_time)</h2>"
if [ "${runs_count:-0}" -eq 0 ]; then
echo "<p style='color: orange;'><strong>No recent test runs found</strong></p>"
return
fi

echo "<table border='1' cellpadding='8' cellspacing='0' style='border-collapse: collapse; width: 100%;'>"
echo "<thead style='background-color: #f0f0f0;'>"
echo "<tr><th>Pipeline Run</th><th>Status</th><th>Created</th><th>Logs</th></tr>"
echo "</thead>"
echo "<tbody>"

echo "$data" | jq -c '.runs[]' | while read -r run; do
local name=$(echo "$run" | jq -r '.name')
local status=$(echo "$run" | jq -r '.status')
local created=$(echo "$run" | jq -r '.created')
local log_link=$(echo "$run" | jq -r '.log_link')

# Color code status
local status_color="black"
case "$status" in
SUCCEEDED) status_color="green" ;;
FAILED) status_color="red" ;;
RUNNING) status_color="blue" ;;
*) status_color="gray" ;;
esac

echo "<tr>"
echo "<td>$name</td>"
echo "<td style='color: $status_color; font-weight: bold;'>$status</td>"
echo "<td>$created</td>"
echo "<td><a href='$log_link'>View Logs</a></td>"
echo "</tr>"
done

echo "</tbody>"
echo "</table>"
echo "<br/>"
}

# Generate email subject
SUBJECT="${INITIAL_SUBJECT} - $(date '+%Y-%m-%d')"
echo "Email subject: $SUBJECT" >&2

# Generate HTML email body
echo "Generating HTML email body..." >&2

# Build HTML content for all test results
HTML_TEST_RESULTS=""
for i in $(seq 0 $((NUM_TESTS - 1))); do
TEST_DATA=$(echo "$TEST_RESULTS" | jq -c ".[$i]")
HTML_TEST_RESULTS+=$(format_results_html "$TEST_DATA")
done

cat > /tmp/email_body.html <<'EMAILEOF'
<!DOCTYPE html>
<html>
<head>
<style>
body { font-family: Arial, sans-serif; }
h1 { color: #333; }
table { margin-top: 10px; margin-bottom: 20px; }
</style>
</head>
<body>
<h1>OpenShift Lightspeed Console - Weekly Test Report</h1>
<p>This report summarizes the results of the periodic console tests.</p>
<hr/>
$HTML_TEST_RESULTS
<hr/>
<p style='color: #666; font-size: 12px;'>
Generated automatically by the ols-console-reporting pipeline on $(date)
</p>
</body>
</html>
EMAILEOF

echo "HTML email body generated" >&2

# Send email
echo "Sending email to: $STAKEHOLDER_EMAILS" >&2
echo "Using sender: noreply@redhat.com" >&2
echo "Using SMTP server: $SMTP_SERVER" >&2

# Use s-nail to send HTML email
# Create full MIME message with headers
echo "Executing s-nail command..." >&2
if s-nail \
-S mta="smtp://${SMTP_SERVER}" \
-S smtp-use-starttls \
-S smtp-auth=login \
-S smtp-auth-user="${USERNAME}" \
-S smtp-auth-password="${PASSWORD}" \
-S from="noreply@redhat.com" \
-s "$SUBJECT" \
-M "text/html" \
"$STAKEHOLDER_EMAILS" < /tmp/email_body.html; then
echo "Email sent successfully!" >&2
else
echo "ERROR: Failed to send email (exit code: $?)" >&2
echo "SMTP server was: $SMTP_SERVER" >&2
fi

echo "=== Email Report Generation Complete ===" >&2

Loading