chore: set up grafana alloy

Author: bernokl

.github/workflows/e2e-tests-linux-split.yml

@@ -269,60 +269,122 @@ jobs:
           name: performance-metrics
           path: ./artifacts/metrics
 
-      - name: Create Prometheus config
-        env:
-          PUSH_URL: ${{ secrets.GRAFANA_PUSH_URL }}    # e.g., https://prometheus-us-central1.grafana.net/api/prom/push
-          PUSH_USER: ${{ secrets.GRAFANA_USERNAME }}   # e.g., 787878
-          PUSH_PASS: ${{ secrets.GRAFANA_PASSWORD }}   # e.g., eyJrIjoxxxxxxxxxxxxxxyMX0=
+      - name: Serve metrics file locally
         run: |
-          cat >prometheus.yml <<EOF
-          global:
-            scrape_interval: 5s
-          scrape_configs:
-            - job_name: 'e2e-perf-metrics'
-              static_configs:
-                - targets: ['pushgateway:9091']
-          remote_write:
-            - url: "$PUSH_URL"
-              basic_auth:
-                username: "$PUSH_USER"
-                password: "$PUSH_PASS"
-          EOF
-          pwd
-          ls -al ./artifacts/metrics/prometheus.txt
-          head ./artifacts/metrics/prometheus.txt
-
-      - name: Start pushgateway server
+          cat <<'PY' > serve_metrics.py
+          import http.server, socketserver, pathlib, sys
+          FILE = pathlib.Path('./artifacts/metrics/prometheus.txt')
+          class H(http.server.SimpleHTTPRequestHandler):
+              def do_GET(self):
+                  if self.path == './artifacts/metrics':
+                      content = FILE.read_bytes() if FILE.exists() else b'# metrics not ready\n'
+                      self.send_response(200)
+                      self.end_headers()
+                      self.wfile.write(content)
+                  else:
+                      self.send_error(404)
+          with socketserver.TCPServer(('0.0.0.0', 9101), H) as s:
+              print('Serving metrics on :9101/metrics', file=sys.stderr)
+              s.serve_forever()
+          PY
+          python serve_metrics.py &
+
+        # ------------------------------------------------------------------------
+        # 3) Write Alloy (River) config — scrape localhost:9101 and remote_write
+        # ------------------------------------------------------------------------
+      - name: Generate alloy.river
+        env:
+          GRAFANA_CLOUD_USERNAME: ${{ secrets.GRAFANA_ALLOY_USERNAME }}
+          GRAFANA_CLOUD_API_KEY:  ${{ secrets.GRAFANA_ALLOY_API_KEY }}
         run: |
-          docker run -d --name pushgateway -p 9091:9091 prom/pushgateway
-          cat ./artifacts/metrics/prometheus.txt  | curl --data-binary @- http://localhost:9091/metrics/job/e2e-perf
-          #docker run -d --name http-server -v $(pwd):/app -w /app -p 8000:8000 python:3.11-slim python -m http.server 8000 > http.log 2>&1
-          printf '\n*****************\n'
-          echo 'This is my current directory:'
-          pwd
-          printf '\n*****************\n'
-          echo 'This is ls -al in that directory'
-          ls -al
-          printf '\n*****************\n'
-
-      - name: Run Prometheus
+          mkdir -p alloy
+          cat > alloy/alloy.river <<RIVER
+          prometheus.remote_write "gc" {
+            endpoint {
+              url = "https://prometheus-us-central1.grafana.net/api/prom/push"
+              basic_auth {
+                username = env("GRAFANA_CLOUD_USERNAME")
+                password = env("GRAFANA_CLOUD_API_KEY")
+              }
+            }
+          }
+
+          prometheus.scrape "e2e" {
+            targets = [{
+              __address__ = "127.0.0.1:9101",
+            }]
+            forward_to = [prometheus.remote_write.gc.receiver]
+          }
+          RIVER
+
+        # ------------------------------------------------------------------------
+        # 4) Launch Alloy for ~45 s to scrape + push the data, then shut it down.
+        # ------------------------------------------------------------------------
+      - name: Run Alloy and push metrics
         run: |
-          # Enable logs for debugging
-          #docker run --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 --log.level=debug
-          docker run -d --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 > prom.log 2>&1
-          sleep 60
-          # Uncomment for troubleshooting.
-          printf '\n*****************\n'
-          printf '\n curl e2e_cpu_seconds_total \n'
-          curl http://localhost:9090/api/v1/query?query=e2e_cpu_seconds_total_v1
-          printf '\n*****************\n'
-          printf '\n cat prom.log \n'
-          cat prom.log
-          #printf '\n*****************\n'
-          #printf '\n send_failures_total \n'
-          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_queue_send_failures_total'
-          #printf '\n*****************\n'
-          #printf '\n storage_retries_total \n'
-          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_retries_total'
+          docker run --rm \
+            -e ALLOY_CONFIG=/etc/alloy.river \
+            -v "$PWD/alloy/alloy.river:/etc/alloy.river:ro" \
+            grafana/alloy:latest run /etc/alloy.river &
+          ALLOY_PID=$!
+          # wait for two scrape cycles + remote write
+          sleep 45
+          kill "$ALLOY_PID" || true
+
+      #      - name: Create Prometheus config
+      #        env:
+      #          PUSH_URL: ${{ secrets.GRAFANA_PUSH_URL }}    # e.g., https://prometheus-us-central1.grafana.net/api/prom/push
+      #          PUSH_USER: ${{ secrets.GRAFANA_USERNAME }}   # e.g., 787878
+      #          PUSH_PASS: ${{ secrets.GRAFANA_PASSWORD }}   # e.g., eyJrIjoxxxxxxxxxxxxxxyMX0=
+      #        run: |
+      #          cat >prometheus.yml <<EOF
+      #          global:
+      #            scrape_interval: 5s
+      #          scrape_configs:
+      #            - job_name: 'e2e-perf-metrics'
+      #              static_configs:
+      #                - targets: ['pushgateway:9091']
+      #          remote_write:
+      #            - url: "$PUSH_URL"
+      #              basic_auth:
+      #                username: "$PUSH_USER"
+      #                password: "$PUSH_PASS"
+      #          EOF
+      #          pwd
+      #          ls -al ./artifacts/metrics/prometheus.txt
+      #          head ./artifacts/metrics/prometheus.txt
+      #
+      #      - name: Start pushgateway server
+      #        run: |
+      #          docker run -d --name pushgateway -p 9091:9091 prom/pushgateway
+      #          cat ./artifacts/metrics/prometheus.txt  | curl --data-binary @- http://localhost:9091/metrics/job/e2e-perf
+      #          #docker run -d --name http-server -v $(pwd):/app -w /app -p 8000:8000 python:3.11-slim python -m http.server 8000 > http.log 2>&1
+      #          printf '\n*****************\n'
+      #          echo 'This is my current directory:'
+      #          pwd
+      #          printf '\n*****************\n'
+      #          echo 'This is ls -al in that directory'
+      #          ls -al
+      #          printf '\n*****************\n'
+      #
+      #      - name: Run Prometheus
+      #        run: |
+      #          # Enable logs for debugging
+      #          #docker run --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 --log.level=debug
+      #          docker run -d --name prometheus -v $(pwd):/etc/prometheus -p 9090:9090 --link pushgateway:pushgateway prom/prometheus:latest --config.file=/etc/prometheus/prometheus.yml --web.listen-address=:9090 > prom.log 2>&1
+      #          sleep 60
+      #          # Uncomment for troubleshooting.
+      #          printf '\n*****************\n'
+      #          printf '\n curl e2e_cpu_seconds_total \n'
+      #          curl http://localhost:9090/api/v1/query?query=e2e_cpu_seconds_total_v1
+      #          printf '\n*****************\n'
+      #          printf '\n cat prom.log \n'
+      #          cat prom.log
+      #          #printf '\n*****************\n'
+      #          #printf '\n send_failures_total \n'
+      #          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_queue_send_failures_total'
+      #          #printf '\n*****************\n'
+      #          #printf '\n storage_retries_total \n'
+      #          #curl 'http://localhost:9090/api/v1/query?query=prometheus_remote_storage_retries_total'