Test prom metrics

.github/workflows/datadog-ci.yml

@@ -0,0 +1,89 @@
+name: Datadog CI Integration
+
+on:
+  workflow_call:
+    inputs:
+      datadog-api-key:
+        required: true
+        type: string
+        description: 'Datadog API key for CI metrics and logs'
+      datadog-site:
+        required: false
+        type: string
+        default: 'datadoghq.com'
+        description: 'Datadog site (datadoghq.com, datadoghq.eu, etc.)'
+      service-name:
+        required: true
+        type: string
+        description: 'Service name for Datadog metrics'
+      environment:
+        required: false
+        type: string
+        default: 'ci'
+        description: 'Environment name'
+      tags:
+        required: false
+        type: string
+        default: 'ci:true'
+        description: 'Additional tags for metrics (comma-separated)'
+
+jobs:
+  datadog-setup:
+    runs-on: ubuntu-latest
+    outputs:
+      dd-trace-id: ${{ steps.datadog-setup.outputs.trace-id }}
+      dd-span-id: ${{ steps.datadog-setup.outputs.span-id }}
+    steps:
+      - name: Setup Datadog CI
+        id: datadog-setup
+        run: |
+          echo "trace-id=$(date +%s)" >> $GITHUB_OUTPUT
+          echo "span-id=$(date +%s)" >> $GITHUB_OUTPUT
+
+  datadog-monitor:
+    runs-on: ubuntu-latest
+    needs: datadog-setup
+    if: always()
+    steps:
+      - name: Send CI metrics to Datadog
+        run: |
+          curl -X POST "https://api.${{ inputs.datadog-site }}/api/v1/series" \
+            -H "Content-Type: application/json" \
+            -H "DD-API-KEY: ${{ inputs.datadog-api-key }}" \
+            -d '{
+              "series": [{
+                "metric": "github.ci.pipeline.duration",
+                "points": [[$(date +%s), ${{ github.run_duration }}]],
+                "tags": ["service:${{ inputs.service-name }}", "env:${{ inputs.environment }}", "workflow:${{ github.workflow }}", "job:${{ github.job }}"],
+                "type": "gauge"
+              }, {
+                "metric": "github.ci.pipeline.status",
+                "points": [[$(date +%s), 1]],
+                "tags": ["service:${{ inputs.service-name }}", "env:${{ inputs.environment }}", "workflow:${{ github.workflow }}", "job:${{ github.job }}", "status:${{ job.status }}"],
+                "type": "gauge"
+              }, {
+                "metric": "github.ci.repository.workflow_runs",
+                "points": [[$(date +%s), 1]],
+                "tags": ["service:${{ inputs.service-name }}", "env:${{ inputs.environment }}", "repo:${{ github.repository }}", "branch:${{ github.ref_name }}"],
+                "type": "gauge"
+              }, {
+                "metric": "github.ci.job.duration",
+                "points": [[$(date +%s), ${{ github.run_duration }}]],
+                "tags": ["service:${{ inputs.service-name }}", "env:${{ inputs.environment }}", "job:${{ github.job }}", "runner:${{ runner.os }}"],
+                "type": "gauge"
+              }]
+            }'
+
+      - name: Send CI events to Datadog
+        if: always()
+        run: |
+          curl -X POST "https://api.${{ inputs.datadog-site }}/api/v1/events" \
+            -H "Content-Type: application/json" \
+            -H "DD-API-KEY: ${{ inputs.datadog-api-key }}" \
+            -d '{
+              "title": "Lace CI Pipeline: ${{ github.workflow }}",
+              "text": "Workflow ${{ github.workflow }} completed with status: ${{ job.status }}\nRepository: ${{ github.repository }}\nBranch: ${{ github.ref_name }}\nCommit: ${{ github.sha }}",
+              "tags": ["service:${{ inputs.service-name }}", "env:${{ inputs.environment }}", "workflow:${{ github.workflow }}", "status:${{ job.status }}", "repo:${{ github.repository }}"],
+              "alert_type": "${{ job.status == 'success' && 'info' || 'error' }}",
+              "source_type_name": "github"
+            }' 

.github/workflows/e2e-tests-linux-split.yml

@@ -193,6 +193,9 @@ jobs:
     # when cancelling job always() will prevent step from being cancelled and we don't want process results in this case
     if: ${{ success() || failure() }}
     steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
       - name: Download all artifacts
         uses: actions/download-artifact@v4
         with:
@@ -212,6 +215,11 @@ jobs:
           tags=${{ needs.setup.outputs.tags }}
           " > environment.properties
 
+      - name: Convert metrics JSON to Datadog format
+        working-directory: './packages/e2e-tests/tools/'
+        run: |
+          bash convert_metrics_to_datadog.sh "${{ github.workflow }}" "${{ github.run_id }}" ../../../artifacts/metrics "${{ secrets.DATADOG_API_KEY }}"
+
       - name: Publish allure report to S3
         uses: andrcuns/[email protected]
         env:
@@ -260,3 +268,77 @@ jobs:
         with:
           name: performance-metrics
           path: ./artifacts/metrics
+
+      - name: Send E2E metrics to Datadog
+        run: |
+          echo "📊 E2E metrics have been sent to Datadog via the conversion script"
+          echo "📁 Datadog payload saved to: ./artifacts/metrics/datadog_payload.json"
+          echo "🔍 Check the previous step for detailed metrics information"
+
+        # ------------------------------------------------------------------------
+        # 3) Write Alloy (River) config — scrape localhost:9101 and remote_write
+        # ------------------------------------------------------------------------
+
+
+        # ------------------------------------------------------------------------
+        # 4) Launch Alloy for ~45 s to scrape + push the data, then shut it down.
+        # ------------------------------------------------------------------------
+
+
+      #      - name: Create Prometheus config
+      #        env:
+      #          PUSH_URL: ${{ secrets.GRAFANA_PUSH_URL }}    # e.g., https://prometheus-us-central1.grafana.net/api/prom/push
+      #          PUSH_USER: ${{ secrets.GRAFANA_USERNAME }}   # e.g., 787878
+      #          PUSH_PASS: ${{ secrets.GRAFANA_PASSWORD }}   # e.g., eyJrIjoxxxxxxxxxxxxxxyMX0=
+      #        run: |
+      #          cat >prometheus.yml <<EOF
+      #          global:
+      #            scrape_interval: 5s
+      #          scrape_configs:
+      #            - job_name: 'e2e-perf-metrics'
+      #              static_configs:
+      #                - targets: ['pushgateway:9091']
+      #          remote_write:
+      #            - url: "$PUSH_URL"
+      #              basic_auth:
+      #                username: "$PUSH_USER"
+      #                password: "$PUSH_PASS"
+      #          EOF
+      #          pwd
+      #          ls -al ./artifacts/metrics/prometheus.txt
+      #          head ./artifacts/metrics/prometheus.txt
+      #
+      #      - name: Start pushgateway server
+      #        run: |
+      #          docker run -d --name pushgateway -p 9091:9091 prom/pushgateway
+      #          cat ./artifacts/metrics/prometheus.txt  | curl --data-binary @- http://localhost:9091/metrics/job/e2e-perf
+      #          #docker run -d --name http-server -v $(pwd):/app -w /app -p 8000:8000 python:3.11-slim python -m http.server 8000 > http.log 2>&1
+      #          printf '\n*****************\n'
+      #          echo 'This is my current directory:'
+      #          pwd
+      #          printf '\n*****************\n'
+      #          echo 'This is ls -al in that directory'
+      #          ls -al
+      #          printf '\n*****************\n'
+      #
+      #      - name: Run Prometheus
+      #        run: |
+      #          # Enable logs for debugging
+      #          #docker run --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 --log.level=debug
+      #          docker run -d --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 > prom.log 2>&1
+      #          sleep 60
+      #          # Uncomment for troubleshooting.
+      #          printf '\n*****************\n'
+      #          printf '\n curl e2e_cpu_seconds_total \n'
+      #          curl http://localhost:9090/api/v1/query?query=e2e_cpu_seconds_total_v1
+      #          printf '\n*****************\n'
+      #          printf '\n cat prom.log \n'
+      #          cat prom.log
+      #          #printf '\n*****************\n'
+      #          #printf '\n send_failures_total \n'
+      #          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_queue_send_failures_total'
+      #          #printf '\n*****************\n'
+      #          #printf '\n storage_retries_total \n'
+      #          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_retries_total'
+
+