[DRAFT] Push v3 results to QE grafana dashboard

.github/actions/start-promtail/action.yml

@@ -0,0 +1,36 @@
+name: Start Promtail
+description: Start promtail in a docker container to ship test results to Grafana Loki, then stop the container
+inputs:
+  promtail_config:
+    description: Path to the promtail configuration file
+    required: true
+  log_dir:
+    description: Directory to store logs
+    required: true
+  loki_url:
+    description: URL endpoint of the Grafana Loki instance
+    required: true
+runs:
+  using: 'composite'
+  steps:
+    - name: Start promtail container
+      shell: bash
+      run: |
+        docker run -d \
+          --name=promtail \
+          -v ${{ inputs.promtail_config }}:/etc/promtail/promtail-config.yaml \
+          -v ${{ inputs.log_dir }}:/var/log \
+          -e TEST_OUTDIR=test/dashboard/logs \
+          -e LOKI_URL=${{ inputs.loki_url }} \
+          -e GITHUB_RUN_ID="${{ github.run_id }}" \
+          -e GITHUB_WORKFLOW="${{ github.workflow }}" \
+          -e GITHUB_EVENT_NAME="${{ github.event_name }}" \
+          -e GITHUB_REPOSITORY="${{ github.repository }}" \
+          -e GITHUB_SERVER_URL="${{ github.server_url }}" \
+          -e GITHUB_JOB="${{ github.job }}" \
+          -e GITHUB_HEAD_REF="${{ github.head_ref }}" \
+          -e GITHUB_SHA="${{ github.sha }}" \
+          -e GITHUB_ACTOR="${{ github.actor }}" \
+          grafana/promtail:3.4.4 \
+          -config.file=/etc/promtail/promtail-config.yaml \
+          -config.expand-env=true

.github/workflows/ci.yml

@@ -136,11 +136,27 @@ jobs:
         with:
           name: nginx-agent-unsigned-snapshots
           path: build
+
+      - name: Create Results Directory
+        run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
+
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+          log_dir: ${{ github.workspace }}/test/dashboard/logs/
+          promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml
+          loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
+
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \
-            make integration-test
+          make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
+          exit "${PIPESTATUS[0]}"
+
+      - name: Format Results
+        if: always()
+        run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
 
   official-oss-image-integration-tests:
     name: Integration Tests - Official OSS Images
@@ -173,13 +189,29 @@ jobs:
         with:
           name: nginx-agent-unsigned-snapshots
           path: build
+
+      - name: Create Results Directory
+        run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
+
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+          log_dir: ${{ github.workspace }}/test/dashboard/logs/
+          promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml
+          loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
+
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="docker-registry.nginx.com" \
           TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" \
           OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" \
-            make official-image-integration-test
+          make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
+          exit "${PIPESTATUS[0]}"
+
+      - name: Format Results
+        if: always()
+        run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
 
   official-plus-image-integration-tests:
     name: Integration Tests - Official Plus Images
@@ -226,13 +258,29 @@ jobs:
           registry: ${{ secrets.REGISTRY_URL }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
+
+      - name: Create Results Directory
+        run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}}
+
+      - name: Start Promtail
+        uses: ./.github/actions/start-promtail
+        with:
+          log_dir: ${{ github.workspace }}/test/dashboard/logs/
+          promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml
+          loki_url: ${{ secrets.LOKI_DASHBOARD_URL }}
+
       - name: Run Integration Tests
         run: |
           go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }}
           CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" \
           TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \
           OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" IMAGE_PATH="${{ matrix.container.path }}" \
-            make official-image-integration-test
+            make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log
+          exit "${PIPESTATUS[0]}"
+
+      - name: Format Results
+        if: always()
+        run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}}
 
   performance-tests:
     name: Performance Tests

Makefile.tools

@@ -8,6 +8,7 @@ NFPM            = github.com/goreleaser/nfpm/v2/cmd/[email protected]
 GOTESTCOVERAGE  = github.com/vladopajic/go-test-coverage/[email protected]
 BENCHSTAT       = golang.org/x/perf/cmd/[email protected]
 BUF             = github.com/bufbuild/buf/cmd/[email protected]
+PROMTAIL	    = github.com/prometheus/promtail/cmd/[email protected]
 
 install-tools: ## Install tool dependencies
 	@echo "Installing Tools"
@@ -22,4 +23,5 @@ install-tools: ## Install tool dependencies
 	@$(GOINST) $(GOTESTCOVERAGE)
 	@$(GOINST) $(BENCHSTAT)
 	@$(GOINST) $(BUF)
+	@$(GOINST) $(PROMTAIL)
 	@$(GORUN) $(LEFTHOOK) install

scripts/dashboard/format_results.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# parameters
+RESULT="$1"
+TEST_TYPE="$2"
+WORKSPACE="$3"
+
+# file paths
+INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log"
+OUTPUT_DIR="$WORKSPACE/test/dashboard/logs/$TEST_TYPE"
+
+# Validate input file exists
+if [ ! -f "$INPUT_FILE" ]; then
+  echo "Error: Input file $INPUT_FILE does not exist."
+  exit 1
+fi
+
+load_job_status(){
+    if [ "$RESULT" == "success" ]; then
+        JOB_RESULT="pass"
+    elif [ "$RESULT" == "failure" ]; then
+        JOB_RESULT="fail"
+    else
+        JOB_RESULT="skip"
+    fi
+}
+
+format_log() {
+    local line="$1"
+    json="{"
+
+    while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do
+        key="${BASH_REMATCH[1]}"
+        value="${BASH_REMATCH[2]}"
+        line="${line#*"${key}=${value}"}"
+
+        if [[ "$value" == \"*\" ]]; then
+            value="${value:1:${#value}-2}"
+            value="${value//\"/\\\"}"
+        fi
+        json+="\"$key\":\"$value\","
+    done
+
+    json="${json%,}}"    
+    echo "$json"
+}
+
+write_result() {
+    local test_name="$1"
+    local start_at="$2"
+    local end_at="$3"
+    local result="$4"
+    local msg="$5"
+    local duration_seconds=0
+    [[ -n "$start_at" && -n "$end_at" ]] && duration_seconds=$(( $(date -d "$end_at" +%s) - $(date -d "$start_at" +%s) ))
+
+    local start_iso=""
+    local end_iso=""
+    [[ -n "$start_at" ]] && start_iso=$(date -d "$start_at" +"%Y-%m-%dT%H:%M:%S.%NZ")
+    [[ -n "$end_at" ]] && end_iso=$(date -d "$end_at" +"%Y-%m-%dT%H:%M:%S.%NZ")
+
+    output_dir="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/$current_test/"
+    mkdir -p "$output_dir"
+    result_file="$output_dir/result.json"
+
+    echo "{\"start_at\":\"$start_iso\", \"end_at\":\"$end_iso\", \"duration_seconds\":$duration_seconds, \"result\":\"$result\", \"msg\":\"$msg\"}" > "$result_file"
+}
+
+format_results() {
+    test_stack=()
+    current_test=""
+    start_at=""
+    end_at=""
+    result=""
+    msg=""
+    isRunning=false
+
+    while IFS= read -r line; do
+        # Detect if the line is a test start
+        if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then
+            if [[ -n "$current_test" ]]; then
+                test_stack+=("$current_test|$start_at|$end_at|$result|$msg")
+            fi
+
+            current_test="${BASH_REMATCH[1]}"
+            start_at=$start_at
+            end_at=""
+            result="pass"
+            msg=""
+            continue
+        fi
+
+        # Get start time
+        if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space]]+starting.*tests ]]; then
+            test_start="${BASH_REMATCH[1]}"
+            continue
+        fi
+
+        # Get end time
+        if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space]]+finished.*tests ]]; then
+            test_end="${BASH_REMATCH[1]}"
+            continue
+        fi
+
+        # Detect result
+        if [[ "$line" == "--- PASS"* || "$line" == "--- FAIL"* || "$line" == "FAIL" ]]; then
+            [[ "$line" == "--- PASS"* ]] && result="pass"
+            [[ "$line" == "--- FAIL"* || "$line" == "FAIL" ]] && result="fail"
+
+            write_result "$current_test" "$start_at" "$end_at" "$result" "$msg" 
+            continue
+        fi
+
+        if [[ "$line" == time=* && "$line" == *level=* ]]; then
+            LOG_LINE=$(format_log "$line")
+            echo "$LOG_LINE" >> "$LOG_FILE"
+            continue
+        fi
+        if [[ "$result" == "fail" ]]; then
+            msg+="$line"
+        fi
+
+    done < "$INPUT_FILE"
+}
+
+{
+    load_job_status
+    format_results
+}