promtorture 0.3: kinda useful

Author: ringerc

testcases/promtorture/.gitignore

@@ -1,3 +1,4 @@
 promtorture
 promtool
 .cache
+notes.md

testcases/promtorture/README.md

@@ -0,0 +1,46 @@
+# promtorture
+
+Make prometheus feel really bad. For science.
+
+## quickstart
+
+```
+./scripts/kind-create.sh
+./scripts/kind-deploy.sh
+
+sleep 10
+
+./scripts/resources
+./scripts/promapi -m tsdb | yq .data.headStats
+./scripts/promapi -m tsdb -l 100
+
+```
+
+## promtorture
+
+for args:
+
+```
+go build
+./promtorture --help
+```
+
+## scripts
+
+- [`kind-create.sh`](./scripts/kind-create.sh): Create a kind cluster and deply kube-prometheus
+- [`kind-deploy.sh`](./scripts/kind-deploy.sh): Deploy promtorture, takes CLI options for target and label counts
+- [`promapi`](./scripts/promapi): A simple prometheus API client, see
+  `./scripts/promapi -h`. Can get TSDB info, make snapshots, etc.
+- [`promcmd`](./scripts/promcmd): wrapper for running shell commands on the prometheus pod. Useful for running
+  `promtool` etc. The script source contains some notes on handy commands.
+- [`promlogs`](./scripts/promlogs): tail logs from the prometheus pod
+- [`promnuke`](./scripts/promnuke): delete all prometheus data
+- [`promtorture-args`](./scripts/promtorture-args): report the container args for the running promtorture instance
+- [`resources`](./scripts/resources): dump the resources for the prometheus pod
+- [`socks5`](./scripts/socks5): start a socks5 proxy to the prometheus pod. You can use this with
+  the `http_proxy=socks5://localhost:1081` env-var to proxy requests to the kube cluster, so you can
+  use in-cluster URIs like `http://prometheus-k8s.monitoring.svc.cluster.local:9090` from commands
+  like `curl` and `promtool`. See also the `curl` option `--socks5-hostname`.
+- [`get-promtool`](./scripts/get-promtool): download the `promtool` binary from a prometheus release,
+  for if you want to run `promtool` over SOCKS5 or port-forward rather than in the prom container, e.g
+  for `promtool debug` dump generation.

testcases/promtorture/scripts/get-promtool

@@ -19,19 +19,12 @@ yq '.|select(.kind == "CustomResourceDefinition")' .cache/kube-prometheus.yaml |
 
 "${kubectl[@]}" wait --for=condition=established --timeout=60s crd --all
 
-# this'll re-apply the CRDs but that's harmless
-#
-# ServiceMonitors are omitted deliberately because for this test we only want to see
-# metrics for explicitly named targets.
-#
 yq '
   .
   |select(
     (.kind != "CustomResourceDefinition")
-    and (.kind != "ServiceMonitor")
     and (.kind != "AlertManager")
-    and (.kind != "PodMonitor")
-  )' |  "${kubectl[@]}" apply --server-side -f -
+   )' |  "${kubectl[@]}" apply --server-side -f -
 "${kubectl[@]}" apply --server-side -f .cache/kube-prometheus.yaml
 
 # Scale down to 1 replica and don't deploy alertmanager

testcases/promtorture/scripts/kind-create.sh

@@ -8,16 +8,23 @@ set -e -u -o pipefail
 
 function usage() {
   echo "Usage: meta -k target|targetmeta|tsdb|walreplay|config|flags|runtime|snapshot"
+  echo
+  echo "For tsdb:"
+  echo "  -l nnnn : number of entries in the top-n lists to return"
   exit 1
 }
 
 # what to fetch
 meta=
-while getopts "m:h" opt; do
+limit=
+while getopts "hm:l:" opt; do
   case ${opt} in
     m)
       meta=$OPTARG
       ;;
+    l)
+      limit=$OPTARG
+      ;;
     \?|h|: )
       usage
       ;;
@@ -43,7 +50,11 @@ case "${meta}" in
     apiQuery "/api/v1/targets/metadata" -G --data 'match_target={job="monitoring/promtorture"}'
     ;;
   tsdb)
-    apiQuery "/api/v1/status/tsdb" -G
+    if [ -n "${limit}" ]; then
+      apiQuery "/api/v1/status/tsdb" -G --data "limit=${limit}"
+    else
+      apiQuery "/api/v1/status/tsdb" -G
+    fi
     ;;
   walreplay)
     apiQuery "/api/v1/status/walreplay" -G

testcases/promtorture/scripts/promapi

@@ -12,10 +12,13 @@ prom_pod="$("${kubectl[@]}" get pod -n monitoring -l app.kubernetes.io/instance=
 #
 # handy commands include
 #
+#    promtool query instant http://localhost:9090 'up{job="monitoring/promtorture"}'
+#
 #    promtool check service-discovery /etc/prometheus/config_out/prometheus.env.yaml podMonitor/monitoring/promtorture/0 --timeout=5s
+#
 #    promtool query labels http://localhost:9090 'container'
+#
 #    promtool push metrics ...
-#    promtool debug --server http://localhost:9090 'tsdb' 'head' 'select * from container_cpu_usage_seconds_total limit 1'
 #
 # Alternately you can run it locally, using scripts/get-promtool, run scripts/socks5 for proxy then
 # run with http_proxy env-var and kube uri for prometheus e.g.:
@@ -34,6 +37,12 @@ prom_pod="$("${kubectl[@]}" get pod -n monitoring -l app.kubernetes.io/instance=
 #     # that'll print the snaphot name, so
 #     ./scripts/promcmd du -ms /prometheus/snapshots/${snap_name}
 #
+# and the snapshot can be dumped with
+#
+#     ./scripts/promcmd /prometheus/replay mkdir /prometheus/snapshots/${snap_name}/wal
+#     ./scripts/promcmd promtool tsdb dump --sandbox-dir-root=/prometheus/replay /prometheus/snapshots/${snap_name}
+
+#
 
 
 exec "${kubectl[@]}" exec -it -n monitoring "${prom_pod}" -- "$@"

testcases/promtorture/scripts/promcmd

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+# Print the args promtorture is running with
+#
+
+set -e -u -o pipefail
+source scripts/config
+
+"${kubectl[@]}" get -n default deployment/promtorture -o yaml | yq '.spec.template.spec.containers[]|select(.name=="promtorture").args'
+
+# vim: et ts=2 sw=2 sts=2 ft=bash ai

testcases/promtorture/scripts/promtorture-args

@@ -5,4 +5,23 @@
 set -e -u -o pipefail
 source scripts/config
 
-"${kubectl_bin}" resource-capacity "${kubectl_context[@]}" -n monitoring --pods --pod-labels app.kubernetes.io/instance=k8s,app.kubernetes.io/name=prometheus
+"${kubectl[@]}" top -n monitoring pod prometheus-k8s-0
+echo
+
+if type -p kubectl-resource_capacity &>/dev/null; then
+	"${kubectl_bin}" resource-capacity "${kubectl_context[@]}" -n monitoring \
+		--pods --pods --containers --util \
+		--pod-labels app.kubernetes.io/instance=k8s,app.kubernetes.io/name=prometheus \
+	| awk 'NR==1 { print } $3=="prometheus" { print }'
+  echo
+else
+	echo 1>&2 "no kubectl resource-capacity; use kubectl krew install resource-capacity"
+	echo 1>&2 "skipping"
+  echo 1>&2
+fi
+
+./scripts/promcmd top -b -n 1 -d 0
+
+./scripts/promcmd cat /proc/1/status | awk '/^Name|^Vm|^Rss/ {print}'
+
+# vim: et ts=2 sw=2 sts=2 ft=bash ai