Skip to content

Commit ebc7986

Browse files
dtfranzdtfranz
authored
Performance Alerting (#2081)
Introduces an early-warning series of prometheus alerts to attempt to catch issues with performance at an early stage in development. Signed-off-by: Daniel Franz <[email protected]>
1 parent 36809b3 commit ebc7986

17 files changed

+344
-234
lines changed

‎.github/workflows/e2e.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,21 @@ jobs:
3535
- name: Run e2e tests
3636
run: ARTIFACT_PATH=/tmp/artifacts make test-e2e
3737

38+
- name: alerts-check
39+
# Grab all current alerts, filtering out pending, and print the GH actions warning string
40+
# containing the alert name and description.
41+
#
42+
# NOTE: Leaving this as annotating-only instead of failing the run until we have some more
43+
# finely-tuned alerts.
44+
run: |
45+
if [[ -s /tmp/artifacts/alerts.out ]]; then \
46+
jq -r 'if .state=="firing" then
47+
"::error title=Prometheus Alert Firing::\(.labels.alertname): \(.annotations.description)"
48+
elif .state=="pending" then
49+
"::warning title=Prometheus Alert Pending::\(.labels.alertname): \(.annotations.description)"
50+
end' /tmp/artifacts/alerts.out
51+
fi
52+
3853
- uses: actions/upload-artifact@v4
3954
if: failure()
4055
with:

‎Makefile‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -277,19 +277,23 @@ test-experimental-e2e: run image-registry prometheus experimental-e2e e2e e2e-me
277277
.PHONY: prometheus
278278
prometheus: PROMETHEUS_NAMESPACE := olmv1-system
279279
prometheus: PROMETHEUS_VERSION := v0.83.0
280+
prometheus: TMPDIR := $(shell mktemp -d)
280281
prometheus: #EXHELP Deploy Prometheus into specified namespace
281-
./hack/test/setup-monitoring.sh $(PROMETHEUS_NAMESPACE) $(PROMETHEUS_VERSION) $(KUSTOMIZE)
282-
283-
# The metrics.out file contains raw json data of the metrics collected during a test run.
284-
# In an upcoming PR, this query will be replaced with one that checks for alerts from
285-
# prometheus. Prometheus will gather metrics we currently query for over the test run,
286-
# and provide alerts from the metrics based on the rules that we set.
282+
trap 'echo "Cleaning up $(TMPDIR)"; rm -rf "$(TMPDIR)"' EXIT; \
283+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$(PROMETHEUS_VERSION)/kustomization.yaml" > "$(TMPDIR)/kustomization.yaml"; \
284+
curl -s "https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/refs/tags/$(PROMETHEUS_VERSION)/bundle.yaml" > "$(TMPDIR)/bundle.yaml"; \
285+
(cd $(TMPDIR) && $(KUSTOMIZE) edit set namespace $(PROMETHEUS_NAMESPACE)) && kubectl create -k "$(TMPDIR)"
286+
kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) -l app.kubernetes.io/name=prometheus-operator
287+
$(KUSTOMIZE) build config/overlays/prometheus | sed "s/cert-git-version/cert-$(VERSION)/g" | kubectl apply -f -
288+
kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) -l app.kubernetes.io/name=prometheus-operator --timeout=60s
289+
kubectl wait --for=create pods -n $(PROMETHEUS_NAMESPACE) prometheus-prometheus-0 --timeout=60s
290+
kubectl wait --for=condition=Ready pods -n $(PROMETHEUS_NAMESPACE) prometheus-prometheus-0 --timeout=120s
291+
292+
# The output alerts.out file contains any alerts, pending or firing, collected during a test run in json format.
287293
.PHONY: e2e-metrics
288-
e2e-metrics: #EXHELP Request metrics from prometheus; place in ARTIFACT_PATH if set
289-
curl -X POST \
290-
-H "Content-Type: application/x-www-form-urlencoded" \
291-
--data 'query={pod=~"operator-controller-controller-manager-.*|catalogd-controller-manager-.*"}' \
292-
http://localhost:30900/api/v1/query > $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/metrics.out
294+
e2e-metrics: ALERTS_FILE_PATH := $(if $(ARTIFACT_PATH),$(ARTIFACT_PATH),.)/alerts.out
295+
e2e-metrics: #EXHELP Request metrics from prometheus; select only actively firing alerts; place in ARTIFACT_PATH if set
296+
curl -X GET http://localhost:30900/api/v1/alerts | jq 'if (.data.alerts | length) > 0 then .data.alerts.[] else empty end' > $(ALERTS_FILE_PATH)
293297

294298
.PHONY: extension-developer-e2e
295299
extension-developer-e2e: KIND_CLUSTER_NAME := operator-controller-ext-dev-e2e

‎config/README.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,12 @@ This provides additional configuration support for end-to-end testing, including
2727

2828
This configuration is used to generate `manifests/standard-e2e.yaml`.
2929

30+
## config/overlays/prometheus
31+
32+
Overlay containing manifest files which enable prometheus scraping of the catalogd and operator-controller pods. Used during e2e runs to measure performance over the lifetime of the test.
33+
34+
These manifests will not end up in the `manifests/` folder, as they must be applied in two distinct steps to avoid issues with applying prometheus CRDs and CRs simultaneously.
35+
3036
## config/overlays/experimental
3137

3238
This provides additional configuration used to support experimental features, including CRDs. This configuration requires cert-manager.

‎config/overlays/prometheus/auth_token.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v1
2+
kind: Secret
3+
type: kubernetes.io/service-account-token
4+
metadata:
5+
name: prometheus-metrics-token
6+
namespace: system
7+
annotations:
8+
kubernetes.io/service-account.name: prometheus

‎config/overlays/prometheus/catalogd_service_monitor.yaml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
name: catalogd-controller-manager-metrics-monitor
5+
namespace: system
6+
spec:
7+
endpoints:
8+
- path: /metrics
9+
port: metrics
10+
interval: 10s
11+
scheme: https
12+
authorization:
13+
credentials:
14+
name: prometheus-metrics-token
15+
key: token
16+
tlsConfig:
17+
# NAMESPACE_PLACEHOLDER replaced by replacements in kustomization.yaml
18+
serverName: catalogd-service.NAMESPACE_PLACEHOLDER.svc
19+
insecureSkipVerify: false
20+
ca:
21+
secret:
22+
# CATALOGD_SERVICE_CERT must be replaced by envsubst
23+
name: catalogd-service-cert-git-version
24+
key: ca.crt
25+
cert:
26+
secret:
27+
name: catalogd-service-cert-git-version
28+
key: tls.crt
29+
keySecret:
30+
name: catalogd-service-cert-git-version
31+
key: tls.key
32+
selector:
33+
matchLabels:
34+
app.kubernetes.io/name: catalogd

‎config/overlays/prometheus/kubelet_service_monitor.yaml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
name: kubelet
5+
namespace: system
6+
labels:
7+
k8s-app: kubelet
8+
spec:
9+
jobLabel: k8s-app
10+
endpoints:
11+
- port: https-metrics
12+
scheme: https
13+
path: /metrics
14+
interval: 10s
15+
honorLabels: true
16+
tlsConfig:
17+
insecureSkipVerify: true
18+
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
19+
metricRelabelings:
20+
- action: keep
21+
sourceLabels: [pod,container]
22+
regex: (operator-controller|catalogd).*;manager
23+
- port: https-metrics
24+
scheme: https
25+
path: /metrics/cadvisor
26+
interval: 10s
27+
honorLabels: true
28+
tlsConfig:
29+
insecureSkipVerify: true
30+
bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
31+
metricRelabelings:
32+
- action: keep
33+
sourceLabels: [pod,container]
34+
regex: (operator-controller|catalogd).*;manager
35+
selector:
36+
matchLabels:
37+
k8s-app: kubelet
38+
namespaceSelector:
39+
matchNames:
40+
- kube-system

‎config/overlays/prometheus/kustomization.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: kustomize.config.k8s.io/v1beta1
2+
kind: Kustomization
3+
namespace: olmv1-system
4+
resources:
5+
- prometheus.yaml
6+
- catalogd_service_monitor.yaml
7+
- kubelet_service_monitor.yaml
8+
- operator_controller_service_monitor.yaml
9+
- prometheus_rule.yaml
10+
- auth_token.yaml
11+
- network_policy.yaml
12+
- service.yaml
13+
- rbac
14+
replacements:
15+
- source:
16+
kind: ServiceMonitor
17+
name: catalogd-controller-manager-metrics-monitor
18+
fieldPath: metadata.namespace
19+
targets:
20+
- select:
21+
kind: ServiceMonitor
22+
name: catalogd-controller-manager-metrics-monitor
23+
fieldPaths:
24+
- spec.endpoints.0.tlsConfig.serverName
25+
options:
26+
delimiter: '.'
27+
index: 1
28+
- select:
29+
kind: ServiceMonitor
30+
name: operator-controller-controller-manager-metrics-monitor
31+
fieldPaths:
32+
- spec.endpoints.0.tlsConfig.serverName
33+
options:
34+
delimiter: '.'
35+
index: 1

‎config/overlays/prometheus/network_policy.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
apiVersion: networking.k8s.io/v1
2+
kind: NetworkPolicy
3+
metadata:
4+
name: prometheus
5+
namespace: system
6+
spec:
7+
podSelector:
8+
matchLabels:
9+
app.kubernetes.io/name: prometheus
10+
policyTypes:
11+
- Egress
12+
- Ingress
13+
egress:
14+
- {} # Allows all egress traffic for metrics requests
15+
ingress:
16+
- {} # Allows us to query prometheus

‎config/overlays/prometheus/operator_controller_service_monitor.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: ServiceMonitor
3+
metadata:
4+
name: operator-controller-controller-manager-metrics-monitor
5+
namespace: system
6+
spec:
7+
endpoints:
8+
- path: /metrics
9+
interval: 10s
10+
port: https
11+
scheme: https
12+
authorization:
13+
credentials:
14+
name: prometheus-metrics-token
15+
key: token
16+
tlsConfig:
17+
# NAMESPACE_PLACEHOLDER replaced by replacements in kustomization.yaml
18+
serverName: operator-controller-service.NAMESPACE_PLACEHOLDER.svc
19+
insecureSkipVerify: false
20+
ca:
21+
secret:
22+
name: olmv1-cert
23+
key: ca.crt
24+
cert:
25+
secret:
26+
name: olmv1-cert
27+
key: tls.crt
28+
keySecret:
29+
name: olmv1-cert
30+
key: tls.key
31+
selector:
32+
matchLabels:
33+
control-plane: operator-controller-controller-manager

‎config/overlays/prometheus/prometheus.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: monitoring.coreos.com/v1
2+
kind: Prometheus
3+
metadata:
4+
name: prometheus
5+
namespace: system
6+
spec:
7+
logLevel: debug
8+
serviceAccountName: prometheus
9+
scrapeTimeout: 30s
10+
scrapeInterval: 1m
11+
securityContext:
12+
runAsNonRoot: true
13+
runAsUser: 65534
14+
seccompProfile:
15+
type: RuntimeDefault
16+
ruleSelector: {}
17+
serviceDiscoveryRole: EndpointSlice
18+
serviceMonitorSelector: {}

0 commit comments

Comments
 (0)