Run Pyrra with generic rules

This should allow Grafana dashboards to access the SLO time series.

Author: metalmatze

jsonnet/kube-prometheus/components/pyrra.libsonnet

@@ -167,6 +167,7 @@ function(params) {
       image: pyrra._config.image,
       args: [
         'kubernetes',
+        '--generic-rules',
       ],
       // resources: pyrra._config.resources,
       ports: [{ containerPort: pyrra._config.port }],

jsonnet/kube-prometheus/main.libsonnet

@@ -133,8 +133,8 @@ local utils = import './lib/utils.libsonnet';
     },
     pyrra: {
       namespace: $.values.common.namespace,
-      version: $.values.common.versions.nodeExporter,
-      image: $.values.common.images.nodeExporter,
+      version: $.values.common.versions.pyrra,
+      image: $.values.common.images.pyrra,
     },
   },
 

kustomization.yaml

@@ -36,12 +36,18 @@ resources:
 - ./manifests/kubeStateMetrics-service.yaml
 - ./manifests/kubeStateMetrics-serviceAccount.yaml
 - ./manifests/kubeStateMetrics-serviceMonitor.yaml
+- ./manifests/kubernetesControlPlane-coredns-ServiceMonitor.yaml
+- ./manifests/kubernetesControlPlane-coredns-slo-response-errors.yaml
+- ./manifests/kubernetesControlPlane-coredns-slo-response-latency.yaml
+- ./manifests/kubernetesControlPlane-kubeControllerManagerSLORequestErrors.yaml
+- ./manifests/kubernetesControlPlane-kubeControllerManagerServiceMonitor.yaml
+- ./manifests/kubernetesControlPlane-kubeProxySLORequestErrors.yaml
+- ./manifests/kubernetesControlPlane-kubelet-slo-request-errors.yaml
+- ./manifests/kubernetesControlPlane-kubelet-slo-runtime-errors.yaml
+- ./manifests/kubernetesControlPlane-kubeletServiceMonitor.yaml
 - ./manifests/kubernetesControlPlane-prometheusRule.yaml
 - ./manifests/kubernetesControlPlane-serviceMonitorApiserver.yaml
-- ./manifests/kubernetesControlPlane-serviceMonitorCoreDNS.yaml
-- ./manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml
 - ./manifests/kubernetesControlPlane-serviceMonitorKubeScheduler.yaml
-- ./manifests/kubernetesControlPlane-serviceMonitorKubelet.yaml
 - ./manifests/nodeExporter-clusterRole.yaml
 - ./manifests/nodeExporter-clusterRoleBinding.yaml
 - ./manifests/nodeExporter-daemonset.yaml
@@ -85,6 +91,24 @@ resources:
 - ./manifests/prometheusOperator-service.yaml
 - ./manifests/prometheusOperator-serviceAccount.yaml
 - ./manifests/prometheusOperator-serviceMonitor.yaml
+- ./manifests/prometheusOperator-sloHTTPErrors.yaml
+- ./manifests/prometheusOperator-sloReconcileErrors.yaml
+- ./manifests/pyrra-apiDeployment.yaml
+- ./manifests/pyrra-apiService.yaml
+- ./manifests/pyrra-kubernetesClusterRole.yaml
+- ./manifests/pyrra-kubernetesClusterRoleBinding.yaml
+- ./manifests/pyrra-kubernetesDeployment.yaml
+- ./manifests/pyrra-kubernetesService.yaml
+- ./manifests/pyrra-kubernetesServiceAccount.yaml
+- ./manifests/pyrra-slo-apiserver-read-cluster-latency.yaml
+- ./manifests/pyrra-slo-apiserver-read-namespace-latency.yaml
+- ./manifests/pyrra-slo-apiserver-read-resource-latency.yaml
+- ./manifests/pyrra-slo-apiserver-read-response-errors.yaml
+- ./manifests/pyrra-slo-apiserver-write-response-errors.yaml
+- ./manifests/pyrra-slo-prometheus-notification-errors.yaml
+- ./manifests/pyrra-slo-prometheus-query-errors.yaml
+- ./manifests/pyrra-slo-prometheus-rule-evaluation-failures.yaml
+- ./manifests/pyrra-slo-prometheus-sd-kubernetes-errors.yaml
 - ./manifests/setup/0alertmanagerConfigCustomResourceDefinition.yaml
 - ./manifests/setup/0alertmanagerCustomResourceDefinition.yaml
 - ./manifests/setup/0podmonitorCustomResourceDefinition.yaml
@@ -95,4 +119,5 @@ resources:
 - ./manifests/setup/0scrapeconfigCustomResourceDefinition.yaml
 - ./manifests/setup/0servicemonitorCustomResourceDefinition.yaml
 - ./manifests/setup/0thanosrulerCustomResourceDefinition.yaml
+- ./manifests/setup/crd.yaml
 - ./manifests/setup/namespace.yaml

manifests/kubernetesControlPlane-serviceMonitorCoreDNS.yaml renamed to manifests/kubernetesControlPlane-coredns-ServiceMonitor.yaml

@@ -22,4 +22,4 @@ spec:
     - kube-system
   selector:
     matchLabels:
-      k8s-app: kube-dns
+      k8s-app: coredns

manifests/kubernetesControlPlane-coredns-slo-response-errors.yaml

@@ -0,0 +1,24 @@
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: coredns
+    app.kubernetes.io/part-of: kube-prometheus
+    prometheus: k8s
+    pyrra.dev/component: coredns
+    role: alert-rules
+  name: coredns-response-errors
+  namespace: monitoring
+spec:
+  description: |
+    CoreDNS runs within a Kubernetes cluster and resolves internal requests and forward external requests.
+    If CoreDNS fails to answer requests applications might be unable to make requests.
+  indicator:
+    ratio:
+      errors:
+        metric: coredns_dns_responses_total{job="coredns",rcode="SERVFAIL"}
+      total:
+        metric: coredns_dns_responses_total{job="coredns"}
+  target: "99.99"
+  window: 2w

manifests/kubernetesControlPlane-coredns-slo-response-latency.yaml

@@ -0,0 +1,24 @@
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: coredns
+    app.kubernetes.io/part-of: kube-prometheus
+    prometheus: k8s
+    pyrra.dev/component: coredns
+    role: alert-rules
+  name: coredns-response-latency
+  namespace: monitoring
+spec:
+  description: |
+    CoreDNS runs within a Kubernetes cluster and resolves internal requests and forward external requests.
+    If CoreDNS gets too slow it might have an impact on the latency of other applications in this cluster.
+  indicator:
+    latency:
+      success:
+        metric: coredns_dns_request_duration_seconds_bucket{job="coredns",le="0.032"}
+      total:
+        metric: coredns_dns_request_duration_seconds_count{job="coredns"}
+  target: "99"
+  window: 2w

manifests/kubernetesControlPlane-kubeControllerManagerSLORequestErrors.yaml

@@ -0,0 +1,21 @@
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  labels:
+    app.kubernetes.io/name: kube-controller-manager
+    app.kubernetes.io/part-of: kube-prometheus
+    prometheus: k8s
+    pyrra.dev/component: kube-controller-manager
+    role: alert-rules
+  name: kube-controller-manager-request-errors
+  namespace: monitoring
+spec:
+  description: "The Kubernetes controller manager is a daemon that embeds the core control loops shipped with Kubernetes. \nIn applications of robotics and automation, a control loop is a non-terminating loop that regulates the state of the system. \nIn Kubernetes, a controller is a control loop that watches the shared state of the cluster through the apiserver and makes changes attempting to move the current state towards the desired state. Examples of controllers that ship with Kubernetes today are the replication controller, endpoints controller, namespace controller, and serviceaccounts controller.\n"
+  indicator:
+    ratio:
+      errors:
+        metric: rest_client_requests_total{job="kube-controller-manager",code=~"5..|<error>"}
+      total:
+        metric: rest_client_requests_total{job="kube-controller-manager"}
+  target: "99"
+  window: 2w

manifests/kubernetesControlPlane-serviceMonitorKubeControllerManager.yaml renamed to manifests/kubernetesControlPlane-kubeControllerManagerServiceMonitor.yaml

@@ -0,0 +1,22 @@
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  labels:
+    app.kubernetes.io/component: controller
+    app.kubernetes.io/name: kube-proxy
+    app.kubernetes.io/part-of: kube-prometheus
+    prometheus: k8s
+    pyrra.dev/component: kube-proxy
+    role: alert-rules
+  name: kube-proxy-request-errors
+  namespace: monitoring
+spec:
+  description: ""
+  indicator:
+    ratio:
+      errors:
+        metric: rest_client_requests_total{job="kube-proxy",code=~"5..|<error>"}
+      total:
+        metric: rest_client_requests_total{job="kube-proxy"}
+  target: "90"
+  window: 2w

manifests/kubernetesControlPlane-kubeProxySLORequestErrors.yaml

@@ -0,0 +1,24 @@
+apiVersion: pyrra.dev/v1alpha1
+kind: ServiceLevelObjective
+metadata:
+  labels:
+    app.kubernetes.io/name: kubelet
+    app.kubernetes.io/part-of: kube-prometheus
+    prometheus: k8s
+    pyrra.dev/component: kubelet
+    role: alert-rules
+  name: kubelet-request-errors
+  namespace: monitoring
+spec:
+  description: |
+    The kubelet is the primary “node agent” that runs on each node.
+    The kubelet ensures that the containers are running and healthy.
+    If these requests are failing the Kubelet might not know what to run exactly.
+  indicator:
+    ratio:
+      errors:
+        metric: rest_client_requests_total{job="kubelet", metrics_path="/metrics",code=~"5..|<error>"}
+      total:
+        metric: rest_client_requests_total{job="kubelet", metrics_path="/metrics"}
+  target: "99"
+  window: 2w