Skip to content

Commit 02770c1

Browse files
john-david3john-david3
authored
Push agent v2 logs to QE grafana (#1185)
1 parent d702f8f commit 02770c1

File tree

14 files changed

+388
-58
lines changed

14 files changed

+388
-58
lines changed

‎.github/actions/start-promtail/action.yml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Start Promtail
2+
description: Start promtail in a Docker container to send test results to loki
3+
inputs:
4+
loki-dashboard-url:
5+
description: "URL of the Loki dashboard to send the logs"
6+
required: true
7+
runs:
8+
using: "composite"
9+
steps:
10+
- name: Start Promtail container
11+
shell: bash
12+
run: |
13+
docker run -d \
14+
--name=promtail \
15+
-v "${{ github.workspace }}/test/dashboard/prep/promtail.yaml:/etc/promtail/config.yaml" \
16+
-v "${{ github.workspace }}/test/dashboard/logs:/var/log" \
17+
-e TEST_OUTDIR=test/dashboard/logs \
18+
-e GITHUB_RUN_ID=${{ github.run_id }} \
19+
-e GITHUB_JOB=${{ github.job }} \
20+
-e GITHUB_WORKFLOW=${{ github.workflow }} \
21+
-e GITHUB_EVENT_NAME=${{ github.event_name }} \
22+
-e GITHUB_SERVER_URL=${{ github.server_url }} \
23+
-e GITHUB_REPOSITORY=${{ github.repository }} \
24+
-e GITHUB_HEAD_REF=${{ github.head_ref }} \
25+
-e GITHUB_SHA=${{ github.sha }} \
26+
-e GITHUB_ACTOR=${{ github.actor }} \
27+
-e LOKI_DASHBOARD_URL=${{ inputs.loki-dashboard-url }} \
28+
grafana/promtail:3.4.4 \
29+
-config.file=/etc/promtail/config.yaml \
30+
-config.expand-env=true

‎.github/workflows/ci.yml‎

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,19 @@ jobs:
5757
- uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
5858
with:
5959
go-version-file: 'go.mod'
60+
- name: Set Start Time
61+
run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
62+
- name: Create Directory
63+
run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/
64+
- name: Start Promtail
65+
uses: ./.github/actions/start-promtail
66+
with:
67+
loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
6068
- name: Run Unit Tests
61-
run: make unit-test
69+
run: make unit-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/raw_logs.log && exit "${PIPESTATUS[0]}"
70+
- name: Generate Test Results
71+
if: always()
72+
run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}} ${{github.workspace}}
6273
- name: Upload Test Coverage
6374
uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # v4.3.0
6475
with:
@@ -161,18 +172,29 @@ jobs:
161172
with:
162173
name: nginx-agent-unsigned-snapshots
163174
path: build
175+
- name: Set Start Time
176+
run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
177+
- name: Create Directory
178+
run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
179+
- name: Start Promtail
180+
uses: ./.github/actions/start-promtail
181+
with:
182+
loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
164183
- name: Run Integration Tests
165184
run: |
166185
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
167186
OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
168-
make integration-test
187+
make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
188+
- name: Generate Test Results
189+
if: always()
190+
run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
169191
- name: Container Output Logs
170192
if: failure()
171193
run: |
172194
docker ps -a
173195
dockerid=$(docker ps -a --format "{{.ID}}")
174196
docker logs "$dockerid"
175-
197+
176198
- name: Archive integration test logs
177199
if: success() || failure()
178200
uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
@@ -214,11 +236,22 @@ jobs:
214236
with:
215237
name: nginx-agent-unsigned-snapshots
216238
path: build
239+
- name: Set Start Time
240+
run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
241+
- name: Create Directory
242+
run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
243+
- name: Start Promtail
244+
uses: ./.github/actions/start-promtail
245+
with:
246+
loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
217247
- name: Run Integration Tests
218248
run: |
219249
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
220250
CONTAINER_NGINX_IMAGE_REGISTRY="${{ env.NGINX_OSS_REGISTRY }}" TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" OS_RELEASE="${{ matrix.container.release }}"\
221-
make official-image-integration-test
251+
make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
252+
- name: Generate Test Results
253+
if: always()
254+
run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
222255
- name: Container Output Logs
223256
if: failure()
224257
run: |
@@ -281,12 +314,23 @@ jobs:
281314
registry: ${{ secrets.REGISTRY_URL }}
282315
username: ${{ secrets.REGISTRY_USERNAME }}
283316
password: ${{ secrets.REGISTRY_PASSWORD }}
317+
- name: Set Start Time
318+
run: echo "START_TIME=$(date +"%Y-%m-%dT%H:%M:%S.%NZ")" >> ${GITHUB_ENV}
319+
- name: Create Directory
320+
run: mkdir -p ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/
321+
- name: Start Promtail
322+
uses: ./.github/actions/start-promtail
323+
with:
324+
loki-dashboard-url: ${{ secrets.LOKI_DASHBOARD_URL }}
284325
- name: Run Integration Tests
285326
run: |
286327
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
287328
CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
288329
OS_RELEASE="${{ matrix.container.release }}" IMAGE_PATH="${{ matrix.container.path }}" \
289-
make official-image-integration-test
330+
make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}${{matrix.container.version}}/raw_logs.log && exit "${PIPESTATUS[0]}"
331+
- name: Generate Test Results
332+
if: always()
333+
run: bash ./scripts/workflow/generate_results.sh ${{job.status}} ${{env.START_TIME}} ${{github.job}}/${{matrix.container.image}}${{matrix.container.version}} ${{github.workspace}}
290334
- name: Container Output Logs
291335
if: failure()
292336
run: |

‎Makefile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -193,16 +193,16 @@ unit-test: $(TEST_BUILD_DIR) test-core test-plugins test-sdk test-extensions ##
193193
@printf "\nTotal code coverage: " && $(GOTOOL) cover -func=$(TEST_BUILD_DIR)/coverage.out | grep 'total:' | awk '{print $$3}'
194194

195195
test-core: $(TEST_BUILD_DIR) ## Run core unit tests
196-
GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
196+
GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/core_coverage.out -covermode count ./src/core/...
197197

198198
test-plugins: $(TEST_BUILD_DIR) ## Run plugins unit tests
199-
GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
199+
GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/plugins_coverage.out -covermode count ./src/plugins/...
200200

201201
test-extensions: $(TEST_BUILD_DIR) ## Run extensions unit tests
202-
GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
202+
GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=$(TEST_BUILD_DIR)/extensions_coverage.out -covermode count ./src/extensions/...
203203

204204
test-sdk: $(TEST_BUILD_DIR) ## Run sdk unit tests from root directory
205-
cd sdk && GOWORK=off CGO_ENABLED=0 go test -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
205+
cd sdk && GOWORK=off CGO_ENABLED=0 go test -v -count=1 -coverprofile=../$(TEST_BUILD_DIR)/sdk_coverage.out -covermode count ./...
206206

207207
# Component tests
208208
component-test: test-component-build test-component-run ## Run component tests

‎docs/proto/proto.md‎

Lines changed: 31 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,6 @@
1818
- [AgentConnectStatus.StatusCode](#f5-nginx-agent-sdk-AgentConnectStatus-StatusCode)
1919
- [AgentLogging.Level](#f5-nginx-agent-sdk-AgentLogging-Level)
2020

21-
- [command_svc.proto](#command_svc-proto)
22-
- [Commander](#f5-nginx-agent-sdk-Commander)
23-
2421
- [command.proto](#command-proto)
2522
- [AgentActivityStatus](#f5-nginx-agent-sdk-AgentActivityStatus)
2623
- [ChunkedResourceChunk](#f5-nginx-agent-sdk-ChunkedResourceChunk)
@@ -42,6 +39,9 @@
4239
- [NginxConfigStatus.Status](#f5-nginx-agent-sdk-NginxConfigStatus-Status)
4340
- [UploadStatus.TransferStatus](#f5-nginx-agent-sdk-UploadStatus-TransferStatus)
4441

42+
- [command_svc.proto](#command_svc-proto)
43+
- [Commander](#f5-nginx-agent-sdk-Commander)
44+
4545
- [common.proto](#common-proto)
4646
- [CertificateDates](#f5-nginx-agent-sdk-CertificateDates)
4747
- [CertificateName](#f5-nginx-agent-sdk-CertificateName)
@@ -341,34 +341,6 @@ Log level enum
341341

342342

343343

344-
<a name="command_svc-proto"></a>
345-
<p align="right"><a href="#top">Top</a></p>
346-
347-
## command_svc.proto
348-
349-
350-
351-
352-
353-
354-
355-
356-
357-
<a name="f5-nginx-agent-sdk-Commander"></a>
358-
359-
### Commander
360-
Represents a service used to sent command messages between the management server and the agent.
361-
362-
| Method Name | Request Type | Response Type | Description |
363-
| ----------- | ------------ | ------------- | ------------|
364-
| CommandChannel | [Command](#f5-nginx-agent-sdk-Command) stream | [Command](#f5-nginx-agent-sdk-Command) stream | A Bidirectional streaming RPC established by the agent and is kept open |
365-
| Download | [DownloadRequest](#f5-nginx-agent-sdk-DownloadRequest) | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | A streaming RPC established by the agent and is used to download resources associated with commands The download stream will be kept open for the duration of the data transfer and will be closed when its done. The transfer is a stream of chunks as follows: header -&gt; data chunk 1 -&gt; data chunk N. Each data chunk is of a size smaller than the maximum gRPC payload |
366-
| Upload | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | [UploadStatus](#f5-nginx-agent-sdk-UploadStatus) | A streaming RPC established by the agent and is used to upload resources associated with commands |
367-
368-
369-
370-
371-
372344
<a name="command-proto"></a>
373345
<p align="right"><a href="#top">Top</a></p>
374346

@@ -680,6 +652,34 @@ Transfer status enum
680652

681653

682654

655+
<a name="command_svc-proto"></a>
656+
<p align="right"><a href="#top">Top</a></p>
657+
658+
## command_svc.proto
659+
660+
661+
662+
663+
664+
665+
666+
667+
668+
<a name="f5-nginx-agent-sdk-Commander"></a>
669+
670+
### Commander
671+
Represents a service used to sent command messages between the management server and the agent.
672+
673+
| Method Name | Request Type | Response Type | Description |
674+
| ----------- | ------------ | ------------- | ------------|
675+
| CommandChannel | [Command](#f5-nginx-agent-sdk-Command) stream | [Command](#f5-nginx-agent-sdk-Command) stream | A Bidirectional streaming RPC established by the agent and is kept open |
676+
| Download | [DownloadRequest](#f5-nginx-agent-sdk-DownloadRequest) | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | A streaming RPC established by the agent and is used to download resources associated with commands The download stream will be kept open for the duration of the data transfer and will be closed when its done. The transfer is a stream of chunks as follows: header -&gt; data chunk 1 -&gt; data chunk N. Each data chunk is of a size smaller than the maximum gRPC payload |
677+
| Upload | [DataChunk](#f5-nginx-agent-sdk-DataChunk) stream | [UploadStatus](#f5-nginx-agent-sdk-UploadStatus) | A streaming RPC established by the agent and is used to upload resources associated with commands |
678+
679+
680+
681+
682+
683683
<a name="common-proto"></a>
684684
<p align="right"><a href="#top">Top</a></p>
685685

‎scripts/workflow/generate_results.sh‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
#!/bin/bash
2+
3+
# Script to process test logs and generate formatted JSON result files
4+
# Usage: ./generate_results.sh <job_result> <start_time> <test_type> <workspace>
5+
6+
set -euo pipefail
7+
8+
# Check if the correct number of arguments are provided
9+
if [ "$#" -ne 4 ]; then
10+
echo "Usage: $0 <job_result> <start_time> <test_type> <workspace>"
11+
exit 1
12+
fi
13+
14+
JOB_RESULT="$1"
15+
START_TIME="$2"
16+
TEST_TYPE="$3"
17+
WORKSPACE="$4"
18+
19+
INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
20+
OUTPUT_PATH="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
21+
22+
# Validate input file exists
23+
if [ ! -f "$INPUT_FILE" ]; then
24+
echo "Error: Input file not found: $INPUT_FILE"
25+
exit 1
26+
fi
27+
28+
END_TIME="`date "+%Y-%m-%dT%H:%M:%S.%NZ"`"
29+
START_SECONDS=$(date -d "$START_TIME" +%s.%N)
30+
END_SECONDS=$(date -d "$END_TIME" +%s.%N)
31+
DURATION=$(echo "$END_SECONDS - $START_SECONDS" | bc)
32+
33+
MSG="" # individual test msg
34+
FAIL_MSG="" # msg for entire job run
35+
RESULT=""
36+
HAS_FAILED=false
37+
IS_RUNNING=false
38+
39+
load_job_status(){
40+
if [ "$JOB_RESULT" == "success" ]; then
41+
RESULT="pass"
42+
elif [ "$JOB_RESULT" == "failure" ]; then
43+
RESULT="fail"
44+
else
45+
RESULT="skip"
46+
fi
47+
}
48+
49+
format_logs_to_json(){
50+
line="$1"
51+
json="{"
52+
53+
while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
54+
key="${BASH_REMATCH[1]}"
55+
value="${BASH_REMATCH[2]}"
56+
line="${line#*"${key}=${value}"}"
57+
58+
if [[ "$value" == \"*\" ]]; then
59+
value="${value:1:${#value}-2}"
60+
value="${value//\"/\\\"}"
61+
fi
62+
json+="\"$key\":\"$value\","
63+
done
64+
65+
json="${json%,}}"
66+
echo "$json"
67+
}
68+
69+
format_results(){
70+
while IFS= read -r line; do
71+
72+
if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
73+
TEST_NAME="${BASH_REMATCH[1]}"
74+
IS_RUNNING=true
75+
MSG=""
76+
TEST_START=""
77+
TEST_END=""
78+
mkdir -p "$OUTPUT_PATH/$TEST_NAME/"
79+
RESULT_FILE="$OUTPUT_PATH/$TEST_NAME/result.json"
80+
LOG_FILE="$OUTPUT_PATH/$TEST_NAME/test.log"
81+
elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+testing ]]; then
82+
TEST_START="${BASH_REMATCH[1]}"
83+
elif [[ "$line" =~ ([0-9T:\.\-Z]+)[[:space:]]+finished[[:space:]]testing ]]; then
84+
TEST_END="${BASH_REMATCH[1]}"
85+
elif [[ "$line" == "FAIL" ]]; then
86+
HAS_FAILED=false
87+
MSG="$MSG_STR"
88+
FAIL_MSG+="$MSG"
89+
HAS_FAILED=false
90+
echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
91+
elif [[ "$line" == "--- PASS"* ]]; then
92+
TEST_RES="pass"
93+
IS_RUNNING=false
94+
echo "{\"start_at\": \"$START_TIME\", \"end_at\": \"$END_TIME\", \"duration_seconds\": \"$DURATION\", \"result\": \"$TEST_RES\", \"msg\": \"$MSG\"}" > $RESULT_FILE
95+
elif [[ "$line" == "--- FAIL"* ]]; then
96+
TEST_RES="fail"
97+
HAS_FAILED=true
98+
IS_RUNNING=false
99+
elif [[ "$line" == time=* && "$line" == *level=* ]]; then
100+
LOG_LINE=$(format_logs_to_json "$line")
101+
echo "$LOG_LINE" >> "$LOG_FILE"
102+
fi
103+
104+
if [ $HAS_FAILED == true ]; then
105+
MSG_STR+="$line"
106+
fi
107+
108+
done < "$INPUT_FILE"
109+
}
110+
111+
# Main body of the script
112+
{
113+
load_job_status
114+
format_results
115+
}

0 commit comments

Comments
 (0)