Skip to content

Commit efa3883

Browse files
authored
Push v3 results to QE grafana dashboard (#1216)
* update script for v3 results * fix paths * fix syntax errors * fix paths * fix test type parameters * update results script * fix auxiliary test suite name * add extra logs for start time * Update results script * Fix formatting * fix linter issue * fix timestamp issue * fix timestamp * fix timestamp * fix date format * fix promtail * Update loki url reference * Update results script * add review improvements
1 parent 836d987 commit efa3883

File tree

12 files changed

+493
-4
lines changed

12 files changed

+493
-4
lines changed
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: Start Promtail
2+
description: Start promtail in a docker container to ship test results to Grafana Loki, then stop the container
3+
inputs:
4+
loki_url:
5+
description: URL endpoint of the Grafana Loki instance
6+
required: true
7+
runs:
8+
using: 'composite'
9+
steps:
10+
- name: Start promtail container
11+
shell: bash
12+
run: |
13+
docker run -d \
14+
--name=promtail \
15+
-v ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml:/etc/promtail/promtail-config.yaml \
16+
-v ${{ github.workspace }}/test/dashboard/logs/:/var/log \
17+
-e TEST_OUTDIR=test/dashboard/logs \
18+
-e LOKI_URL=${{ inputs.loki_url }} \
19+
-e GITHUB_RUN_ID="${{ github.run_id }}" \
20+
-e GITHUB_WORKFLOW="${{ github.workflow }}" \
21+
-e GITHUB_EVENT_NAME="${{ github.event_name }}" \
22+
-e GITHUB_REPOSITORY="${{ github.repository }}" \
23+
-e GITHUB_SERVER_URL="${{ github.server_url }}" \
24+
-e GITHUB_JOB="${{ github.job }}" \
25+
-e GITHUB_HEAD_REF="${{ github.head_ref }}" \
26+
-e GITHUB_SHA="${{ github.sha }}" \
27+
-e GITHUB_ACTOR="${{ github.actor }}" \
28+
grafana/promtail:3.4.4 \
29+
-config.file=/etc/promtail/promtail-config.yaml \
30+
-config.expand-env=true

.github/workflows/ci.yml

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,25 @@ jobs:
136136
with:
137137
name: nginx-agent-unsigned-snapshots
138138
path: build
139+
140+
- name: Create Results Directory
141+
run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
142+
143+
- name: Start Promtail
144+
uses: ./.github/actions/start-promtail
145+
with:
146+
loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
147+
139148
- name: Run Integration Tests
140149
run: |
141150
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
142151
OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
143-
make integration-test
152+
make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
153+
exit "${PIPESTATUS[0]}"
154+
155+
- name: Format Results
156+
if: always()
157+
run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
144158

145159
official-oss-image-integration-tests:
146160
name: Integration Tests - Official OSS Images
@@ -173,13 +187,27 @@ jobs:
173187
with:
174188
name: nginx-agent-unsigned-snapshots
175189
path: build
190+
191+
- name: Create Results Directory
192+
run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
193+
194+
- name: Start Promtail
195+
uses: ./.github/actions/start-promtail
196+
with:
197+
loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
198+
176199
- name: Run Integration Tests
177200
run: |
178201
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
179202
CONTAINER_NGINX_IMAGE_REGISTRY="docker-registry.nginx.com" \
180203
TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" \
181204
OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" \
182-
make official-image-integration-test
205+
make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
206+
exit "${PIPESTATUS[0]}"
207+
208+
- name: Format Results
209+
if: always()
210+
run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
183211

184212
official-plus-image-integration-tests:
185213
name: Integration Tests - Official Plus Images
@@ -226,13 +254,27 @@ jobs:
226254
registry: ${{ secrets.REGISTRY_URL }}
227255
username: ${{ secrets.REGISTRY_USERNAME }}
228256
password: ${{ secrets.REGISTRY_PASSWORD }}
257+
258+
- name: Create Results Directory
259+
run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
260+
261+
- name: Start Promtail
262+
uses: ./.github/actions/start-promtail
263+
with:
264+
loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
265+
229266
- name: Run Integration Tests
230267
run: |
231268
go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
232269
CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" \
233270
TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
234271
OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" IMAGE_PATH="${{ matrix.container.path }}" \
235-
make official-image-integration-test
272+
make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
273+
exit "${PIPESTATUS[0]}"
274+
275+
- name: Format Results
276+
if: always()
277+
run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
236278

237279
performance-tests:
238280
name: Performance Tests

Makefile.tools

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ NFPM = github.com/goreleaser/nfpm/v2/cmd/[email protected]
88
GOTESTCOVERAGE = github.com/vladopajic/go-test-coverage/[email protected]
99
BENCHSTAT = golang.org/x/perf/cmd/[email protected]
1010
BUF = github.com/bufbuild/buf/cmd/[email protected]
11+
PROMTAIL = github.com/prometheus/promtail/cmd/[email protected]
1112

1213
install-tools: ## Install tool dependencies
1314
@echo "Installing Tools"
@@ -22,4 +23,5 @@ install-tools: ## Install tool dependencies
2223
@$(GOINST) $(GOTESTCOVERAGE)
2324
@$(GOINST) $(BENCHSTAT)
2425
@$(GOINST) $(BUF)
26+
@$(GOINST) $(PROMTAIL)
2527
@$(GORUN) $(LEFTHOOK) install

scripts/dashboard/format_results.sh

Lines changed: 224 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,224 @@
1+
#!/bin/bash
2+
# Script to process test logs and generate formatted result files
3+
# Usage: ./format_results.sh <job_result> <test_type> <workspace>
4+
5+
set -euo pipefail
6+
7+
# Check if required arguments are provided
8+
if [ "$#" -ne 3 ]; then
9+
echo "Usage: $0 <job_result> <test_type> <workspace>"
10+
exit 1
11+
fi
12+
13+
# Parameters
14+
RESULT="$1"
15+
TEST_TYPE="$2"
16+
WORKSPACE="$3"
17+
18+
# File paths
19+
INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
20+
OUTPUT_DIR="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
21+
22+
# Validate input file exists
23+
if [ ! -f "$INPUT_FILE" ]; then
24+
echo "Error: Input file not found: $INPUT_FILE"
25+
exit 1
26+
fi
27+
28+
format_log() {
29+
line="$1"
30+
json="{"
31+
32+
while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
33+
key="${BASH_REMATCH[1]}"
34+
value="${BASH_REMATCH[2]}"
35+
line="${line#*"${key}=${value}"}"
36+
37+
if [[ "$value" == \"*\" ]]; then
38+
value="${value:1:${#value}-2}"
39+
value="${value//\"/\\\"}"
40+
fi
41+
json+="\"$key\":\"$value\","
42+
done
43+
44+
json="${json%,}}"
45+
echo "$json"
46+
}
47+
48+
write_result() {
49+
start_at="$1"
50+
end_at="$2"
51+
result="$3"
52+
msg="$4"
53+
output_dir="$5"
54+
duration_seconds=0
55+
56+
if [[ $end_at == "end_at" ]]; then
57+
end_at=$(date +"%Y/%m/%d %H:%M:%S")
58+
fi
59+
60+
# Format timestamps
61+
if [[ "$start_at" =~ ^[0-9]{4}/[0-9]{2}/[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}$ && \
62+
"$end_at" =~ ^[0-9]{4}/[0-9]{2}/[0-9]{2}\ [0-9]{2}:[0-9]{2}:[0-9]{2}$ ]]; then
63+
duration_seconds=$(( $(date -d "$end_at" +%s) - $(date -d "$start_at" +%s) ))
64+
start_iso=""
65+
end_iso=""
66+
start_iso=$(date -d "$start_at" +"%Y-%m-%dT%H:%M:%S.%NZ")
67+
end_iso=$(date -d "$end_at" +"%Y-%m-%dT%H:%M:%S.%NZ")
68+
else
69+
duration_seconds=0
70+
fi
71+
72+
if [[ ${msg} == "msg" ]]; then
73+
msg=""
74+
fi
75+
76+
mkdir -p "$output_dir"
77+
result_file="$output_dir/result.json"
78+
79+
echo "{\"start_at\":\"$start_iso\", \"end_at\":\"$end_iso\", \"duration_seconds\":$duration_seconds, \"result\":\"$result\", \"msg\":\"$msg\"}" > "$result_file"
80+
}
81+
82+
format_results() {
83+
test_group=("name" "start_at" "end_at" "result" "msg")
84+
current_test=("name" "start_at" "end_at" "result" "msg")
85+
test_queue=()
86+
is_running=false
87+
has_failed=false
88+
error_trace=""
89+
90+
while IFS= read -r line; do
91+
# Detect if the line is a test start
92+
if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
93+
test_name="${BASH_REMATCH[1]}"
94+
has_failed=false
95+
96+
if [[ "${test_group[0]}" == "name" && "$is_running" == false ]]; then
97+
is_running=true
98+
test_group[0]="$test_name"
99+
elif [[ "${test_group[0]}" != "name" && "$is_running" == true ]]; then
100+
is_running=true
101+
if [[ "${current_test[0]}" != "${test_group[0]}" ]]; then
102+
test_queue+=("${current_test[@]}")
103+
fi
104+
fi
105+
106+
current_test=("$test_name" "start_at" "end_at" "result" "msg")
107+
continue
108+
fi
109+
110+
# Get start time
111+
if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space:]]+starting.*test* ]]; then
112+
test_start="${BASH_REMATCH[1]}"
113+
current_test[1]="$test_start"
114+
if [[ "${current_test[0]}" == "${test_group[0]}" ]]; then
115+
test_group[1]="$test_start"
116+
fi
117+
continue
118+
fi
119+
120+
# Get end time
121+
if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space:]]+finished.*test* ]]; then
122+
test_end="${BASH_REMATCH[1]}"
123+
if [[ "${current_test[2]}" == "end_at" ]]; then
124+
current_test[2]="$test_end"
125+
if [[ "${current_test[0]}" == "${test_group[0]}" ]]; then
126+
test_group[2]="$test_end"
127+
fi
128+
elif [[ "${current_test[2]}" != "end_at" ]]; then
129+
test_group[2]="$test_end"
130+
fi
131+
continue
132+
fi
133+
134+
# Capture error messages
135+
if [[ "$line" == *"Error Trace"* || "$line" == *"runtime error"* ]]; then
136+
has_failed=true
137+
error_trace+="${line}"$'\n'
138+
continue
139+
fi
140+
141+
# Detect result
142+
if [[ "$line" == *"--- PASS"* || "$line" == *"--- FAIL"* ]]; then
143+
[[ "$line" == *"--- PASS"* ]] && result_val="pass"
144+
[[ "$line" == *"--- FAIL"* ]] && result_val="fail"
145+
146+
has_failed=false
147+
148+
# Clear current_test field
149+
if [[ "${current_test[0]}" != "name" ]]; then
150+
if [[ "${current_test[0]}" == "${test_group[0]}" ]]; then
151+
current_test=("name" "start_at" "end_at" "result" "msg")
152+
else
153+
test_queue+=("${current_test[@]}")
154+
current_test=("name" "start_at" "end_at" "result" "msg")
155+
fi
156+
fi
157+
158+
# Write results for the test group
159+
if [[ "${test_group[0]}" != "name" && "${#test_queue[@]}" -eq 0 ]] ||
160+
[[ "${test_group[0]}" != "name" && "${#test_queue[@]}" -gt 0 ]]; then
161+
if [[ "$line" != *"${test_group[0]}"* ]]; then
162+
echo "Error: Test name did not match. Expected '${test_group[0]}', in line: '$line'."
163+
exit 1
164+
fi
165+
test_group[3]="$result_val"
166+
if [[ "$result_val" == "fail" ]]; then
167+
if [[ ${test_group[4]} == "msg" ]]; then
168+
test_group[4]=""
169+
fi
170+
test_group[4]+="$error_trace"
171+
fi
172+
write_result "${test_group[1]}" "${test_group[2]}" "${test_group[3]}" "${test_group[4]}" "$OUTPUT_DIR/${test_group[0]}"
173+
test_group=("name" "start_at" "end_at" "result" "msg")
174+
is_running=false
175+
continue
176+
fi
177+
178+
# Write results for individual tests in the queue
179+
if [[ "${test_group[0]}" == "name" && "${#test_queue[@]}" -gt 0 ]]; then
180+
test_match=("${test_queue[0]}" "${test_queue[1]}" "${test_queue[2]}" "${test_queue[3]}" "${test_queue[4]}")
181+
test_match[3]="$result_val"
182+
if [[ "$result_val" == "fail" ]]; then
183+
if [[ ${test_match[4]} == "msg" ]]; then
184+
test_match[4]=""
185+
fi
186+
test_match[4]+="$error_trace"
187+
fi
188+
write_result "${test_match[1]}" "${test_match[2]}" "${test_match[3]}" "${test_match[4]}" "$OUTPUT_DIR/${test_match[0]}"
189+
190+
for i in {0..4}; do
191+
unset 'test_queue[$i]'
192+
done
193+
test_queue=("${test_queue[@]:5}")
194+
fi
195+
196+
# No tests to analyze
197+
if [[ "${test_group[0]}" == "name" && "${#test_queue[@]}" -eq 0 ]]; then
198+
error_trace=""
199+
continue
200+
fi
201+
fi
202+
203+
# Capture error messages
204+
if [[ $has_failed == true ]]; then
205+
error_trace+="${line}"$'\n'
206+
fi
207+
208+
# Capture logs
209+
if [[ "$line" =~ time=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{3}Z)[[:space:]]+level= ]]; then
210+
LOG_LINE=$(format_log "$line")
211+
LOG_FILE_OUT_DIR="$OUTPUT_DIR/${test_group[0]}"
212+
LOG_FILE=${LOG_FILE_OUT_DIR}/test.log
213+
if [[ ! -d "$LOG_FILE_OUT_DIR" ]]; then
214+
mkdir -p "$LOG_FILE_OUT_DIR"
215+
fi
216+
echo "$LOG_LINE" >> "$LOG_FILE"
217+
continue
218+
fi
219+
done < "$INPUT_FILE"
220+
}
221+
222+
{
223+
format_results
224+
}

0 commit comments

Comments
 (0)