diff --git a/.github/actions/start-promtail/action.yml b/.github/actions/start-promtail/action.yml new file mode 100644 index 000000000..7e8d516e3 --- /dev/null +++ b/.github/actions/start-promtail/action.yml @@ -0,0 +1,36 @@ +name: Start Promtail +description: Start promtail in a docker container to ship test results to Grafana Loki, then stop the container +inputs: + promtail_config: + description: Path to the promtail configuration file + required: true + log_dir: + description: Directory to store logs + required: true + loki_url: + description: URL endpoint of the Grafana Loki instance + required: true +runs: + using: 'composite' + steps: + - name: Start promtail container + shell: bash + run: | + docker run -d \ + --name=promtail \ + -v ${{ inputs.promtail_config }}:/etc/promtail/promtail-config.yaml \ + -v ${{ inputs.log_dir }}:/var/log \ + -e TEST_OUTDIR=test/dashboard/logs \ + -e LOKI_URL=${{ inputs.loki_url }} \ + -e GITHUB_RUN_ID="${{ github.run_id }}" \ + -e GITHUB_WORKFLOW="${{ github.workflow }}" \ + -e GITHUB_EVENT_NAME="${{ github.event_name }}" \ + -e GITHUB_REPOSITORY="${{ github.repository }}" \ + -e GITHUB_SERVER_URL="${{ github.server_url }}" \ + -e GITHUB_JOB="${{ github.job }}" \ + -e GITHUB_HEAD_REF="${{ github.head_ref }}" \ + -e GITHUB_SHA="${{ github.sha }}" \ + -e GITHUB_ACTOR="${{ github.actor }}" \ + grafana/promtail:3.4.4 \ + -config.file=/etc/promtail/promtail-config.yaml \ + -config.expand-env=true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eaee9f553..4d66edb18 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,11 +136,27 @@ jobs: with: name: nginx-agent-unsigned-snapshots path: build + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + log_dir: ${{ github.workspace }}/test/dashboard/logs/ + promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + - name: Run Integration Tests run: | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} OS_RELEASE="${{ matrix.container.image }}" OS_VERSION="${{ matrix.container.version }}" \ - make integration-test + make integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} official-oss-image-integration-tests: name: Integration Tests - Official OSS Images @@ -173,13 +189,29 @@ jobs: with: name: nginx-agent-unsigned-snapshots path: build + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + log_dir: ${{ github.workspace }}/test/dashboard/logs/ + promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + - name: Run Integration Tests run: | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} CONTAINER_NGINX_IMAGE_REGISTRY="docker-registry.nginx.com" \ TAG="${{ matrix.container.version }}-${{ matrix.container.image }}" \ OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" \ - make official-image-integration-test + make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} official-plus-image-integration-tests: name: Integration Tests - Official Plus Images @@ -226,13 +258,29 @@ jobs: registry: ${{ secrets.REGISTRY_URL }} username: ${{ secrets.REGISTRY_USERNAME }} password: ${{ secrets.REGISTRY_PASSWORD }} + + - name: Create Results Directory + run: mkdir -p ${{ github.workspace }}/test/dashboard/logs/${{ github.job }}/${{matrix.container.image}}-${{matrix.container.version}} + + - name: Start Promtail + uses: ./.github/actions/start-promtail + with: + log_dir: ${{ github.workspace }}/test/dashboard/logs/ + promtail_config: ${{ github.workspace }}/test/dashboard/promtail/promtail-config.yaml + loki_url: ${{ secrets.LOKI_DASHBOARD_URL }} + - name: Run Integration Tests run: | go install github.com/goreleaser/nfpm/v2/cmd/nfpm@${{ env.NFPM_VERSION }} CONTAINER_NGINX_IMAGE_REGISTRY="${{ secrets.REGISTRY_URL }}" \ TAG="${{ matrix.container.plus }}-${{ matrix.container.image }}-${{ matrix.container.version }}" \ OS_RELEASE="${{ matrix.container.release }}" OS_VERSION="${{ matrix.container.version }}" IMAGE_PATH="${{ matrix.container.path }}" \ - make official-image-integration-test + make official-image-integration-test | tee ${{github.workspace}}/test/dashboard/logs/${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}}/raw_logs.log + exit "${PIPESTATUS[0]}" + + - name: Format Results + if: always() + run: bash ./scripts/dashboard/format_results.sh ${{job.status}} ${{github.job}}/${{matrix.container.image}}-${{matrix.container.version}} ${{github.workspace}} performance-tests: name: Performance Tests diff --git a/Makefile.tools b/Makefile.tools index 512a729a2..8e7bcb492 100644 --- a/Makefile.tools +++ b/Makefile.tools @@ -8,6 +8,7 @@ NFPM = github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.36.1 GOTESTCOVERAGE = github.com/vladopajic/go-test-coverage/v2@v2.10.1 BENCHSTAT = golang.org/x/perf/cmd/benchstat@v0.0.0-20240404204407-f3e401e020e4 BUF = github.com/bufbuild/buf/cmd/buf@v1.30.1 +PROMTAIL = github.com/prometheus/promtail/cmd/promtail@v2.10.0 install-tools: ## Install tool dependencies @echo "Installing Tools" @@ -22,4 +23,5 @@ install-tools: ## Install tool dependencies @$(GOINST) $(GOTESTCOVERAGE) @$(GOINST) $(BENCHSTAT) @$(GOINST) $(BUF) + @$(GOINST) $(PROMTAIL) @$(GORUN) $(LEFTHOOK) install diff --git a/scripts/dashboard/format_results.sh b/scripts/dashboard/format_results.sh new file mode 100755 index 000000000..f4f4bbb77 --- /dev/null +++ b/scripts/dashboard/format_results.sh @@ -0,0 +1,131 @@ +#!/bin/bash + +set -euo pipefail + +# parameters +RESULT="$1" +TEST_TYPE="$2" +WORKSPACE="$3" + +# file paths +INPUT_FILE="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/raw_logs.log" +OUTPUT_DIR="$WORKSPACE/test/dashboard/logs/$TEST_TYPE" + +# Validate input file exists +if [ ! -f "$INPUT_FILE" ]; then + echo "Error: Input file $INPUT_FILE does not exist." + exit 1 +fi + +load_job_status(){ + if [ "$RESULT" == "success" ]; then + JOB_RESULT="pass" + elif [ "$RESULT" == "failure" ]; then + JOB_RESULT="fail" + else + JOB_RESULT="skip" + fi +} + +format_log() { + local line="$1" + json="{" + + while [[ "$line" =~ ([a-zA-Z0-9_]+)=((\"([^\"\\]|\\.)*\")|[^[:space:]]+) ]]; do + key="${BASH_REMATCH[1]}" + value="${BASH_REMATCH[2]}" + line="${line#*"${key}=${value}"}" + + if [[ "$value" == \"*\" ]]; then + value="${value:1:${#value}-2}" + value="${value//\"/\\\"}" + fi + json+="\"$key\":\"$value\"," + done + + json="${json%,}}" + echo "$json" +} + +write_result() { + local test_name="$1" + local start_at="$2" + local end_at="$3" + local result="$4" + local msg="$5" + local duration_seconds=0 + [[ -n "$start_at" && -n "$end_at" ]] && duration_seconds=$(( $(date -d "$end_at" +%s) - $(date -d "$start_at" +%s) )) + + local start_iso="" + local end_iso="" + [[ -n "$start_at" ]] && start_iso=$(date -d "$start_at" +"%Y-%m-%dT%H:%M:%S.%NZ") + [[ -n "$end_at" ]] && end_iso=$(date -d "$end_at" +"%Y-%m-%dT%H:%M:%S.%NZ") + + output_dir="$WORKSPACE/test/dashboard/logs/$TEST_TYPE/$current_test/" + mkdir -p "$output_dir" + result_file="$output_dir/result.json" + + echo "{\"start_at\":\"$start_iso\", \"end_at\":\"$end_iso\", \"duration_seconds\":$duration_seconds, \"result\":\"$result\", \"msg\":\"$msg\"}" > "$result_file" +} + +format_results() { + test_stack=() + current_test="" + start_at="" + end_at="" + result="" + msg="" + isRunning=false + + while IFS= read -r line; do + # Detect if the line is a test start + if [[ "$line" =~ ^===\ RUN[[:space:]]+(.+) ]]; then + if [[ -n "$current_test" ]]; then + test_stack+=("$current_test|$start_at|$end_at|$result|$msg") + fi + + current_test="${BASH_REMATCH[1]}" + start_at=$start_at + end_at="" + result="pass" + msg="" + continue + fi + + # Get start time + if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space]]+starting.*tests ]]; then + test_start="${BASH_REMATCH[1]}" + continue + fi + + # Get end time + if [[ "$line" =~ ^([0-9]{4}/[0-9]{2}/[0-9]{2}[[:space:]][0-9]{2}:[0-9]{2}:[0-9]{2}).*INFO[[:space]]+finished.*tests ]]; then + test_end="${BASH_REMATCH[1]}" + continue + fi + + # Detect result + if [[ "$line" == "--- PASS"* || "$line" == "--- FAIL"* || "$line" == "FAIL" ]]; then + [[ "$line" == "--- PASS"* ]] && result="pass" + [[ "$line" == "--- FAIL"* || "$line" == "FAIL" ]] && result="fail" + + write_result "$current_test" "$start_at" "$end_at" "$result" "$msg" + continue + fi + + if [[ "$line" == time=* && "$line" == *level=* ]]; then + LOG_LINE=$(format_log "$line") + echo "$LOG_LINE" >> "$LOG_FILE" + continue + fi + if [[ "$result" == "fail" ]]; then + msg+="$line" + fi + + done < "$INPUT_FILE" +} + +{ + load_job_status + format_results +} diff --git a/test/dashboard/promtail/promtail-config.yaml b/test/dashboard/promtail/promtail-config.yaml new file mode 100644 index 000000000..87fb9c1d0 --- /dev/null +++ b/test/dashboard/promtail/promtail-config.yaml @@ -0,0 +1,129 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + sync_period: "10s" + +clients: + - url: "${{ secrets.LOKI_DASHBOARD_URL }}" + external_labels: + systest_project: agent_v3 + systest_type: "${GITHUB_JOB}" + +scrape_configs: + - job_name: test-logs + static_configs: + - targets: + - localhost + labels: + systest_job: test-logs + __path__: /var/log/**/test.log + + pipeline_stages: + - json: + expressions: + time: + - timestamp: + source: time + format: RFC3339Nano + - template: + source: ci_pipeline_id + template: "${GITHUB_RUN_ID}" + - template: + source: ci_pipeline_url + template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + - template: + source: ci_pipeline_name + template: "${WORKFLOW}" + - template: + source: ci_pipeline_source + template: "${GITHUB_EVENT_NAME}" + - template: + source: ci_job_name + template: "${GITHUB_JOB}" + - template: + source: ci_commit_ref + template: "${GITHUB_HEAD_REF}" + - template: + source: ci_commit_sha + template: "${GITHUB_SHA}" + - template: + source: ci_commit_author + template: "${GITHUB_ACTOR}" + - template: + source: systest_path + template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}' + + - structured_metadata: + ci_pipeline_id: + ci_pipeline_url: + ci_pipeline_name: + ci_pipeline_source: + ci_job_name: + ci_commit_ref: + ci_commit_sha: + ci_commit_author: + filename: + systest_path: + + - labeldrop: + - filename + + - job_name: test-results + static_configs: + - targets: + - localhost + labels: + systest_job: test-results + __path__: /var/log/**/result.json + pipeline_stages: + - json: + expressions: + start_at: + - timestamp: + source: time + format: RFC3339Nano + - template: + source: ci_pipeline_id + template: "${GITHUB_RUN_ID}" + - template: + source: ci_pipeline_url + template: "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + - template: + source: ci_pipeline_name + template: "${WORKFLOW}" + - template: + source: ci_pipeline_source + template: "${GITHUB_EVENT_NAME}" + - template: + source: ci_job_name + template: "${GITHUB_JOB}" + - template: + source: ci_commit_ref + template: "${GITHUB_HEAD_REF}" + - template: + source: ci_commit_sha + template: "${GITHUB_SHA}" + - template: + source: ci_commit_author + template: "${GITHUB_ACTOR}" + - template: + source: systest_path + template: '{{ trimPrefix "${TEST_OUTDIR}/" .filename | dir | replace "." "/" }}' + + - structured_metadata: + ci_pipeline_id: + ci_pipeline_url: + ci_pipeline_name: + ci_pipeline_source: + ci_job_name: + ci_commit_ref: + ci_commit_sha: + ci_commit_author: + filename: + systest_path: + + - labeldrop: + - filename diff --git a/test/integration/auxiliarycommandserver/connection_test.go b/test/integration/auxiliarycommandserver/connection_test.go index 4b2204daa..a3f46f9b5 100644 --- a/test/integration/auxiliarycommandserver/connection_test.go +++ b/test/integration/auxiliarycommandserver/connection_test.go @@ -8,6 +8,7 @@ package auxiliarycommandserver import ( "context" "fmt" + "log/slog" "net" "net/http" "os" @@ -32,19 +33,22 @@ func (s *AuxiliaryTestSuite) SetupSuite() { t := s.T() // Expect errors in logs should be false for recconnection tests // For now for these test we will skip checking the logs for errors + slog.Info("starting auxiliary command server tests") s.teardownTest = utils.SetupConnectionTest(t, false, false, true, "../../config/agent/nginx-agent-with-auxiliary-command.conf") } func (s *AuxiliaryTestSuite) TearDownSuite() { + slog.Info("finished auxiliary command server tests") s.teardownTest(s.T()) } -func TestSuite(t *testing.T) { +func TestAuxiliaryTestSuite(t *testing.T) { suite.Run(t, new(AuxiliaryTestSuite)) } func (s *AuxiliaryTestSuite) TestAuxiliary_Test1_Connection() { + slog.Info("starting auxiliary command server connection tests") s.instanceID = utils.VerifyConnection(s.T(), 2, utils.MockManagementPlaneAPIAddress) s.False(s.T().Failed()) utils.VerifyUpdateDataPlaneHealth(s.T(), utils.MockManagementPlaneAPIAddress) diff --git a/test/integration/installuninstall/install_uninstall_test.go b/test/integration/installuninstall/install_uninstall_test.go index 14b60a0f1..a6d207491 100644 --- a/test/integration/installuninstall/install_uninstall_test.go +++ b/test/integration/installuninstall/install_uninstall_test.go @@ -9,6 +9,7 @@ import ( "context" "fmt" "io" + "log/slog" "os" "path" "path/filepath" @@ -51,6 +52,8 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai LogMessage: "nginx_pid", } + slog.Info("starting install uninstall tests") + // start container without agent installed testContainer := helpers.StartAgentlessContainer( ctx, @@ -68,6 +71,7 @@ func installUninstallSetup(tb testing.TB, expectNoErrorsInLogs bool) (testcontai expectNoErrorsInLogs, nil, ) + slog.Info("finished install uninstall tests") } } diff --git a/test/integration/managementplane/config_apply_test.go b/test/integration/managementplane/config_apply_test.go index 39297e716..188b5ef6d 100644 --- a/test/integration/managementplane/config_apply_test.go +++ b/test/integration/managementplane/config_apply_test.go @@ -8,6 +8,7 @@ package managementplane import ( "context" "fmt" + "log/slog" "os" "sort" "testing" @@ -38,6 +39,7 @@ type ConfigApplyChunkingTestSuite struct { } func (s *ConfigApplyTestSuite) SetupSuite() { + slog.Info("starting config apply tests") s.ctx = context.Background() s.teardownTest = utils.SetupConnectionTest(s.T(), false, false, false, "../../config/agent/nginx-config-with-grpc-client.conf") @@ -48,6 +50,7 @@ func (s *ConfigApplyTestSuite) SetupSuite() { } func (s *ConfigApplyTestSuite) TearDownSuite() { + slog.Info("finished config apply tests") s.teardownTest(s.T()) } @@ -131,6 +134,7 @@ func (s *ConfigApplyTestSuite) TestConfigApply_Test4_TestFileNotInAllowedDirecto } func (s *ConfigApplyChunkingTestSuite) SetupSuite() { + slog.Info("starting config apply chunking tests") s.ctx = context.Background() s.teardownTest = utils.SetupConnectionTest(s.T(), false, false, false, "../../config/agent/nginx-config-with-max-file-size.conf") @@ -141,6 +145,7 @@ func (s *ConfigApplyChunkingTestSuite) SetupSuite() { } func (s *ConfigApplyChunkingTestSuite) TearDownSuite() { + slog.Info("finished config apply chunking tests") s.teardownTest(s.T()) } diff --git a/test/integration/managementplane/config_upload_test.go b/test/integration/managementplane/config_upload_test.go index 0f888d3c4..b5f85918c 100644 --- a/test/integration/managementplane/config_upload_test.go +++ b/test/integration/managementplane/config_upload_test.go @@ -8,6 +8,7 @@ package managementplane import ( "context" "fmt" + "log/slog" "net/http" "testing" @@ -26,6 +27,7 @@ type MPITestSuite struct { } func (s *MPITestSuite) TearDownSuite() { + slog.Info("finished MPI tests") s.teardownTest(s.T()) } @@ -34,6 +36,7 @@ func (s *MPITestSuite) TearDownTest() { } func (s *MPITestSuite) SetupSuite() { + slog.Info("starting MPI tests") s.ctx = context.Background() s.teardownTest = utils.SetupConnectionTest(s.T(), true, false, false, "../../config/agent/nginx-config-with-grpc-client.conf") diff --git a/test/integration/nginxless/nginx_less_mpi_connection_test.go b/test/integration/nginxless/nginx_less_mpi_connection_test.go index c979fadca..c5f58a50f 100644 --- a/test/integration/nginxless/nginx_less_mpi_connection_test.go +++ b/test/integration/nginxless/nginx_less_mpi_connection_test.go @@ -6,6 +6,7 @@ package nginxless import ( + "log/slog" "testing" "github.com/nginx/agent/v3/test/integration/utils" @@ -15,10 +16,13 @@ import ( // Verify that the agent sends a connection request to Management Plane even when Nginx is not present func TestNginxLessGrpc_Connection(t *testing.T) { + slog.Info("starting nginxless connection test") teardownTest := utils.SetupConnectionTest(t, true, true, false, "../../config/agent/nginx-config-with-grpc-client.conf") defer teardownTest(t) utils.VerifyConnection(t, 1, utils.MockManagementPlaneAPIAddress) assert.False(t, t.Failed()) + + slog.Info("Finished nginxless connection test") }