Dedup

.github/workflows/docker.yaml

@@ -0,0 +1,60 @@
+name: Build and push Docker image
+on:
+  push:
+    branches:
+      - main
+    tags:
+      - "*"
+  pull_request:
+    branches:
+      - main
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-push-image:
+    name: Build and push Docker image
+    runs-on: [matterlabs-ci-runner]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set git SHA
+        id: git_sha
+        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
+
+      - name: Set Docker tag
+        id: docker_tag
+        run: |
+          if [[ "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "tag=${{ steps.git_sha.outputs.sha_short }}" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.ref }}" == refs/tags/* ]]; then
+            echo "tag=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_OUTPUT
+          elif [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "tag=none" >> $GITHUB_OUTPUT
+          else
+            echo "Unsupported event ${GITHUB_EVENT_NAME} or ref ${GITHUB_REF}, only refs/heads/, refs/tags/ and pull_request are supported."
+            exit 1
+          fi
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push Docker image
+        id: docker_build
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile.ml
+          push: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/')) }}
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/stackdriver-exporter:${{ steps.docker_tag.outputs.tag }}
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/stackdriver-exporter:latest
+
+      - name: Print image digest to summary
+        run: |
+          echo "Image tag: ${{ steps.docker_tag.outputs.tag }}" >> $GITHUB_STEP_SUMMARY

Dockerfile.ml

@@ -0,0 +1,24 @@
+FROM golang:1.24-alpine AS builder
+
+WORKDIR /app
+
+# Install necessary build tools
+RUN apk add --no-cache curl git make
+
+# Cache dependencies
+COPY go.mod go.sum ./
+RUN go mod download
+
+# Copy source
+COPY . .
+
+# Build
+RUN make build
+
+FROM quay.io/prometheus/busybox:latest
+
+COPY --from=builder /app/stackdriver_exporter /bin/
+
+USER       nobody
+ENTRYPOINT ["/bin/stackdriver_exporter"]
+EXPOSE     9255

collectors/deduplicator.go

@@ -0,0 +1,75 @@
+package collectors
+
+import (
+	"sort"
+	"strconv"
+	"time"
+
+	"github.com/prometheus-community/stackdriver_exporter/hash"
+)
+
+// MetricDeduplicator helps prevent sending duplicate metrics within a single scrape cycle.
+// It tracks metrics based on a signature derived from FQName, labels, and timestamp.
+type MetricDeduplicator struct {
+	sentSignatures map[uint64]struct{}
+}
+
+// NewMetricDeduplicator creates a new deduplicator instance.
+func NewMetricDeduplicator() *MetricDeduplicator {
+	return &MetricDeduplicator{
+		sentSignatures: make(map[uint64]struct{}),
+	}
+}
+
+// CheckAndMark checks if a metric signature has already been seen for this deduplicator instance.
+// If seen, it returns true (indicating a duplicate).
+// If not seen, it marks the signature as seen and returns false.
+func (d *MetricDeduplicator) CheckAndMark(fqName string, labelKeys, labelValues []string, ts time.Time) bool {
+	signature := d.hashLabelsTimestamp(fqName, labelKeys, labelValues, ts)
+	if _, exists := d.sentSignatures[signature]; exists {
+		return true // Duplicate detected
+	}
+	d.sentSignatures[signature] = struct{}{} // Mark as seen
+	return false // Not a duplicate
+}
+
+// hashLabelsTimestamp calculates a hash based on FQName, sorted labels, and timestamp.
+func (d *MetricDeduplicator) hashLabelsTimestamp(fqName string, labelKeys, labelValues []string, ts time.Time) uint64 {
+	dh := hash.New()
+	dh = hash.Add(dh, fqName)
+	dh = hash.AddByte(dh, hash.SeparatorByte)
+
+	// Create label pairs for stable sorting
+	pairs := make([]struct {
+		Key   string
+		Value string
+	}, len(labelKeys))
+	for i, key := range labelKeys {
+		// Ensure we don't go out of bounds if labelValues is shorter (shouldn't happen in normal flow)
+		val := ""
+		if i < len(labelValues) {
+			val = labelValues[i]
+		}
+		pairs[i] = struct {
+			Key   string
+			Value string
+		}{Key: key, Value: val}
+	}
+
+	// Sort pairs by key
+	sort.Slice(pairs, func(i, j int) bool {
+		return pairs[i].Key < pairs[j].Key
+	})
+
+	// Add sorted key-value pairs to hash
+	for _, pair := range pairs {
+		dh = hash.Add(dh, pair.Key)
+		dh = hash.AddByte(dh, hash.SeparatorByte)
+		dh = hash.Add(dh, pair.Value)
+		dh = hash.AddByte(dh, hash.SeparatorByte)
+	}
+
+	// Add timestamp (converted to string)
+	dh = hash.Add(dh, strconv.FormatInt(ts.UnixNano(), 10))
+	return dh
+}