diff --git a/clusterloader2/go.mod b/clusterloader2/go.mod index db9d9a0c6d..fca3dd812e 100644 --- a/clusterloader2/go.mod +++ b/clusterloader2/go.mod @@ -49,7 +49,6 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.37.0 golang.org/x/oauth2 v0.29.0 - golang.org/x/sync v0.13.0 golang.org/x/time v0.11.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.32.3 @@ -113,6 +112,7 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/goleak v1.3.0 // indirect golang.org/x/net v0.38.0 // indirect + golang.org/x/sync v0.13.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/term v0.31.0 // indirect golang.org/x/text v0.24.0 // indirect diff --git a/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/linux-exporter-podmonitor.yaml b/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/linux-exporter-podmonitor.yaml new file mode 100644 index 0000000000..33a683bcb2 --- /dev/null +++ b/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/linux-exporter-podmonitor.yaml @@ -0,0 +1,19 @@ +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: node-exporter + namespace: monitoring + labels: + app: node-exporter # must match your Prometheus CR's `podMonitorSelector` +spec: + selector: + matchLabels: + app: node-exporter + namespaceSelector: + matchNames: + - monitoring + podMetricsEndpoints: + - port: metrics + path: /metrics + interval: 5s + diff --git a/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter-daemonset.yaml b/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter-daemonset.yaml new file mode 100644 index 0000000000..e659fef161 --- /dev/null +++ b/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter-daemonset.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + namespace: monitoring + labels: + app: node-exporter +spec: + selector: + matchLabels: + app: node-exporter + template: + metadata: + labels: + app: node-exporter + spec: + hostNetwork: true + hostPID: true + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:latest + args: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--collector.disable-defaults" + - "--collector.cpu" + - "--collector.meminfo" + - "--collector.loadavg" + - "--collector.stat" + ports: + - containerPort: 9100 + hostPort: 9100 + name: metrics + resources: + limits: + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + nodeSelector: + kubernetes.io/os: linux + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + terminationGracePeriodSeconds: 30 \ No newline at end of file diff --git a/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter.yaml b/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter.yaml deleted file mode 100644 index 525fdc1a1e..0000000000 --- a/clusterloader2/pkg/prometheus/manifests/exporters/node_exporter/node-exporter.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: node-exporter - namespace: kube-system - labels: - k8s-app: node-exporter -spec: - containers: - - name: prometheus-node-exporter - image: gcr.io/k8s-testimages/quay.io/prometheus/node-exporter:v1.0.1 - imagePullPolicy: "IfNotPresent" - args: - - --path.sysfs=/host/sys - - --path.rootfs=/host/root - - --no-collector.wifi - - --no-collector.hwmon - - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/pods/.+)($|/) - - --collector.netstat.fields=^.*$ - - --collector.qdisc - ports: - - name: metrics - containerPort: 9100 - hostPort: 9100 - volumeMounts: - - mountPath: /host/sys - mountPropagation: HostToContainer - name: sys - readOnly: true - - mountPath: /host/root - mountPropagation: HostToContainer - name: root - readOnly: true - resources: - limits: - cpu: 10m - memory: 50Mi - requests: - cpu: 10m - memory: 50Mi - hostNetwork: true - hostPID: true - volumes: - - hostPath: - path: /sys - name: sys - - hostPath: - path: / - name: root diff --git a/clusterloader2/pkg/prometheus/prometheus.go b/clusterloader2/pkg/prometheus/prometheus.go index 7eb0918b62..8f6bc6656b 100644 --- a/clusterloader2/pkg/prometheus/prometheus.go +++ b/clusterloader2/pkg/prometheus/prometheus.go @@ -27,7 +27,6 @@ import ( "sync" "time" - "golang.org/x/sync/errgroup" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -59,7 +58,7 @@ const ( kubeStateMetricsManifests = "exporters/kube-state-metrics/*.yaml" masterIPServiceMonitors = "master-ip/*.yaml" metricsServerManifests = "exporters/metrics-server/*.yaml" - nodeExporterPod = "exporters/node_exporter/node-exporter.yaml" + nodeExporterManifests = "exporters/node_exporter/*.yaml" windowsNodeExporterManifests = "exporters/windows_node_exporter/*.yaml" pushgatewayManifests = "pushgateway/*.yaml" ) @@ -260,7 +259,7 @@ func (pc *Controller) SetUpPrometheusStack() error { return err } if pc.clusterLoaderConfig.PrometheusConfig.ScrapeNodeExporter { - if err := pc.runNodeExporter(); err != nil { + if err := pc.applyDefaultManifests(nodeExporterManifests); err != nil { return err } } @@ -520,46 +519,6 @@ func (pc *Controller) configureRBACForMetrics(testClusterClientSet kubernetes.In return nil } -// runNodeExporter adds node-exporter as master's static manifest pod. -// TODO(mborsz): Consider migrating to something less ugly, e.g. daemonset-based approach, -// when master nodes have configured networking. -func (pc *Controller) runNodeExporter() error { - klog.V(2).Infof("Starting node-exporter on master nodes.") - kubemarkFramework, err := framework.NewFramework(&pc.clusterLoaderConfig.ClusterConfig, numK8sClients) - if err != nil { - return err - } - - // Validate masters first - nodes, err := client.ListNodes(kubemarkFramework.GetClientSets().GetClient()) - if err != nil { - return err - } - - var g errgroup.Group - numMasters := 0 - for _, node := range nodes { - node := node - if util.LegacyIsMasterNode(&node) || util.IsControlPlaneNode(&node) { - numMasters++ - g.Go(func() error { - f, err := manifestsFS.Open(nodeExporterPod) - if err != nil { - return fmt.Errorf("unable to open manifest file: %v", err) - } - defer f.Close() - return pc.ssh.Exec("sudo tee /etc/kubernetes/manifests/node-exporter.yaml > /dev/null", &node, f) - }) - } - } - - if numMasters == 0 { - return fmt.Errorf("node-exporter requires master to be registered nodes") - } - - return g.Wait() -} - func (pc *Controller) waitForPrometheusToBeHealthy() error { klog.V(2).Info("Waiting for Prometheus stack to become healthy...") return wait.PollImmediate(