promtorture: embed old version

Author: ringerc

testcases/promtorture/promkill/.dockerignore

@@ -0,0 +1,3 @@
+promkill
+config/k8s
+README.md

testcases/promtorture/promkill/.gitignore

@@ -0,0 +1 @@
+promkill

testcases/promtorture/promkill/Dockerfile

@@ -0,0 +1,23 @@
+# syntax=docker/dockerfile:1
+
+# Usage:
+#     docker buildx build -t promkill:latest .
+#     docker run --rm --publish 127.0.0.1:8080:8080 --name promkill promkill:latest
+#     curl localhost:8080/metrics
+#
+#  or
+#     kind load docker-image promkill:latest
+#     kubectl apply --context kind-kind -f config/k8s/deployment.yaml
+
+
+FROM golang:1.21-alpine AS build
+WORKDIR /app
+COPY go.mod go.sum ./
+RUN go mod download
+COPY *.go ./
+RUN CGO_ENABLED=0 GOOS=linux go build -o /usr/local/bin/promkill
+
+FROM scratch
+COPY --from=build /usr/local/bin/promkill /usr/local/bin/promkill
+EXPOSE 8080
+CMD ["/usr/local/bin/promkill"]

testcases/promtorture/promkill/README.md

@@ -0,0 +1,118 @@
+# Prometheus cardinality torture-testing
+
+(This is an earlier version of promtorture, retained here for reference)
+
+
+Should use https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md instead for 
+setup
+
+```bash
+kind create cluster
+
+# should be releasever instead
+kustomize build https://github.com/prometheus-operator/kube-prometheus > prom.yaml
+kfilt --kind=CustomResourceDefinition < prom.yaml | kubectl create --context kind-kind -f -
+kfilt --exclude-kind=CustomResourceDefinition < prom.yaml | kubectl create --context kind-kind -f -
+rm prom.yaml
+
+# should be a kustomization
+cat > resources-patch.json  <<'__END__'
+[
+  {
+    "op": "add",
+    "path": "/spec/resources",
+    "value": {
+      "requests": {
+        "memory": "1Gi",
+	"cpu": "4"
+      },
+      "limits": {
+        "memory": "1Gi",
+	"cpu": "15"
+      }
+    }
+  },
+  {
+    "op": "replace",
+    "path": "/spec/scrapeInterval",
+    "value": "1s"
+  },
+  {
+    "op": "replace",
+    "path": "/spec/replicas",
+    "value": 1
+  }
+]
+__END__
+
+kubectl patch -n monitoring prometheus/k8s --type json  --patch-file resources-patch.json
+
+docker buildx build -t promkill:latest .
+kind load docker-image promkill:latest
+kubectl apply --context kind-kind -f config/k8s/deployment.yaml
+kubectl apply --context kind-kind -f config/k8s/podmonitor.yaml
+```
+
+check exported metrics
+
+```bash
+kubectl port-forward $(kubectl get pod -n default -l app=promkill -o name | head -1) 8080:8080
+curl -sSLf localhost:8080/metrics
+curl -sSLf http://localhost:8080/metrics | wc -l
+```
+
+check prom
+
+```bash
+kubectl -n monitoring get pods -l app.kubernetes.io/name=prometheus
+```
+
+forward prom api
+
+```bash
+kubectl -n monitoring port-forward services/prometheus-k8s 8081:8080 9091:9090
+```
+
+then visit http://localhost:9091/ for webui or API, e.g. enumerate labels
+
+Make sure it's being scraped
+
+```bash
+curl -X GET -sSLf http://localhost:9091/api/v1/targets
+```
+
+check metrics dims
+
+```bash
+curl -X GET -sSLf http://localhost:9091/api/v1/labels
+```
+
+watch mem
+
+```
+kube-capacity --pod-labels app.kubernetes.io/name=prometheus -u -c
+```
+
+
+### hacking prom
+
+```
+kubectl get -n monitoring prometheus/k8s -oyaml
+```
+
+also useful: `GET /api/v1/metadata`, `GET /api/v1/status/runtimeinfo`, `GET /api/v1/status/tsdb`
+
+* `curl -X GET -sSLf http://localhost:9091/api/v1/status/tsdb | jq '.'`
+* `curl -X GET -sSLf http://localhost:9091/api/v1/status/runtimeinfo | jq '.'`
+* `curl -X GET -sSLf http://localhost:9091/api/v1/status/config | jq -r '.data.yaml'`
+* `curl -X GET -sSLf http://localhost:9091/api/v1/status/flags`
+* `curl -X GET -sSLf http://localhost:9091/api/v1/metadata| jq '.'`
+* `curl -X GET -sSLf http://localhost:9091/flags | jq -r '.data.yaml'`
+
+Watch prom's own metrics
+
+## Poke at prom
+
+```bash
+kubectl get -n monitoring prometheus/k8s -oyaml > promcfg.yaml
+```

testcases/promtorture/promkill/config/k8s/deployment.yaml

@@ -0,0 +1,24 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: promkill
+  namespace: default
+  labels:
+    app: promkill
+spec:
+  replicas: 3
+  selector:
+    matchLabels:
+      app: promkill
+  template:
+    metadata:
+      labels:
+        app: promkill
+    spec:
+      containers:
+      - name: promkill
+        image: promkill:latest
+        ports:
+          - name: metrics
+            containerPort: 8080
+        imagePullPolicy: Never

testcases/promtorture/promkill/config/k8s/podmonitor.yaml

@@ -0,0 +1,10 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: promkill
+spec:
+  selector:
+    matchLabels:
+      app: promkill
+  podMetricsEndpoints:
+  - port: metrics

testcases/promtorture/promkill/go.mod

@@ -0,0 +1,14 @@
+module promkill
+
+go 1.20
+
+require (
+	github.com/VictoriaMetrics/metrics v1.24.0
+	github.com/google/uuid v1.3.1
+)
+
+require (
+	github.com/valyala/fastrand v1.1.0 // indirect
+	github.com/valyala/histogram v1.2.0 // indirect
+	golang.org/x/sys v0.7.0 // indirect
+)

testcases/promtorture/promkill/go.sum

@@ -0,0 +1,10 @@
+github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
+github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
+github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4=
+github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
+github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
+github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
+github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
+golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU=
+golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

testcases/promtorture/promkill/main.go

@@ -0,0 +1,57 @@
+/*
+ * hacked up version of the victoria metrics prom exporter demo, for generating
+ * massive cardinality and throwing it at prom.
+ */
+package main
+
+import (
+	"fmt"
+	"strings"
+	"time"
+	"net/http"
+	"log"
+	"math/rand"
+	"github.com/google/uuid"
+	"github.com/VictoriaMetrics/metrics"
+)
+
+func addMetric(basename string, ndims int) {
+	var labels []string = []string{
+		"fixed1=\"constant\"",
+		"fixed2=\"very very very very very very very very very long long long long long long value value value value\"",
+	}
+	for i := 0; i < ndims; i++ {
+		dim := fmt.Sprintf("dim%d=\"%s\"", i, uuid.New().String())
+		labels = append(labels, dim)
+	}
+	metric_name := fmt.Sprintf("%s{%s}", basename, strings.Join(labels,","))
+	// Discard the metric, we're just making noise here. The metric value will be 0.
+	// TODO: remember them a while then age them out?
+	metric_func := func() float64 { return rand.Float64() }
+	_ = metrics.NewGauge(metric_name, metric_func)
+}
+
+func metricsGenerator() {
+	for {
+		addMetric("basic", 1)
+		addMetric("wide", 10)
+		addMetric("extreme", 40)
+
+		time.Sleep(1 * time.Second)
+	}
+}
+
+func main() {
+	// front-load a massive pile of metrics
+	for i := 0; i < 400000; i++ {
+		metric_name := fmt.Sprintf("fixed%d", i)
+		metric_func := func() float64 { return rand.Float64() }
+		_ = metrics.NewGauge(metric_name, metric_func)
+	}
+	// and add more with wide dims regularly
+	go metricsGenerator();
+	http.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
+		metrics.WritePrometheus(w, false)
+	})
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}