Removed grafana servicemonitor and moved nodeports to helm config

Author: wtripp180901

ansible/roles/kube_prometheus_stack/defaults/main/helm.yml

@@ -52,6 +52,9 @@ kube_prometheus_stack_release_defaults:
       KubeProxyDown: true
       KubeControllerManagerDown: true
   prometheus:
+    service:
+      type: NodePort
+      nodePort: "{{ prometheus_port }}"
     prometheusSpec:
       externalUrl: "{{ kube_prometheus_stack_prometheus_external_url }}"
       podMonitorSelectorNilUsesHelmValues: false
@@ -92,9 +95,12 @@ kube_prometheus_stack_release_defaults:
 
   grafana:
     service:
-      port: "{{ grafana_port }}"
+      type: NodePort
+      nodePort: "{{ grafana_port }}"
+    serviceMonitor:
+      enabled: false
     ingress:
-      path: /node/{{ groups['grafana'].0 }}/30001
+      path: "/node/{{ groups['grafana'].0 }}/{{ grafana_port }}"
     sidecar:
       dashboards:
         searchNamespace: ALL
@@ -127,6 +133,9 @@ kube_prometheus_stack_release_defaults:
     env: "{{ grafana_environment }}"
 
   alertmanager:
+    service:
+      type: NodePort
+      nodePort: "{{ alertmanager_port }}"
     # Don't apply the namespace grouping by default
     config: "{{ alertmanager_config }}"
     alertmanagerSpec:

ansible/roles/kube_prometheus_stack/defaults/main/main.yml

@@ -45,6 +45,11 @@ alertmanager_config:
 
 alertmanager_replicas: 1
 
+# Must be within K3s' reserved port range (default 30000-32767)
+prometheus_port: 30000
+grafana_port: 30001
+alertmanager_port: 30002
+
 ### PREVIOUS ROLE VALUES
 
 prometheus_image_tag: "v2.27.0"
@@ -94,7 +99,7 @@ prometheus_alert_relabel_configs: []
 #     regex: replica
 
 prometheus_global:
-  scrape_interval: 15s
+  scrape_interval: 30s
   scrape_timeout: 10s
   evaluation_interval: 15s
 
@@ -121,10 +126,6 @@ prometheus_scrape_configs:
     static_configs:
       - targets:
         - "{{ control_ip }}:{{ slurm_exporter_port }}"
-  - job_name: "grafana"
-    static_configs:
-    - targets:
-      - "kube-prometheus-stack-grafana:{{ grafana_port }}"
 
 # Alternative config file name, searched in ansible templates path.
 # prometheus_config_file: 'prometheus.yml.j2'
@@ -273,8 +274,6 @@ grafana_instance: "{{ ansible_fqdn | default(ansible_host) | default(inventory_h
 
 grafana_data_dir: "/var/lib/grafana"
 
-grafana_port: 80
-
 # Additional options for grafana "server" section
 # This section WILL omit options for: http_addr, http_port, domain, and root_url, as those settings are set by variables listed before
 # grafana_server:

ansible/roles/kube_prometheus_stack/tasks/main.yml

@@ -193,46 +193,3 @@
     create_namespace: no
     wait: yes
     wait_timeout: "{{ kube_prometheus_stack_wait_timeout }}"
-
-# Again can't loop through these because the k8s module doesn't like templating ints
-- name: Opening NodePort for Prometheus
-  kubernetes.core.k8s:
-    namespace: "{{ kube_prometheus_stack_release_namespace }}"
-    name: prometheus-external
-    definition:
-      kind: Service
-      spec:
-        type: NodePort
-        selector:
-          app.kubernetes.io/name: prometheus
-        ports:
-          - port: 9090
-            nodePort: 30000
-
-- name: Opening NodePort for Grafana
-  kubernetes.core.k8s:
-    namespace: "{{ kube_prometheus_stack_release_namespace }}"
-    name: grafana-external
-    definition:
-      kind: Service
-      spec:
-        type: NodePort
-        selector:
-          app.kubernetes.io/name: grafana
-        ports:
-          - port: 3000
-            nodePort: 30001
-
-- name: Opening NodePort for Alertmanager
-  kubernetes.core.k8s:
-    namespace: "{{ kube_prometheus_stack_release_namespace }}"
-    name: alertmanager-external
-    definition:
-      kind: Service
-      spec:
-        type: NodePort
-        selector:
-          app.kubernetes.io/name: alertmanager
-        ports:
-          - port: 9093
-            nodePort: 30002

environments/common/inventory/group_vars/all/grafana.yml

@@ -5,7 +5,7 @@
 grafana_version: '9.5.21'
 
 # need to copy some role defaults here so we can use in inventory:
-grafana_port: 80
+grafana_port: 30001
 
 # Define where state is stored
 grafana_data_dir: "{{ appliances_state_dir | default('/var/lib') }}/grafana"
@@ -16,7 +16,7 @@ grafana_api_url: "http://{{ grafana_api_address }}:{{ grafana_port }}"
 
 # Configure external address, with external URL depending on whether we are using Open Ondemand as a proxy
 grafana_url_direct: "http://{{ grafana_address }}:{{ grafana_port }}"
-grafana_url_openondemand_proxy: "https://{{ openondemand_servername | default('') }}/node/{{ groups['grafana'].0 }}/30001"
+grafana_url_openondemand_proxy: "https://{{ openondemand_servername | default('') }}/node/{{ groups['grafana'].0 }}/{{ grafana_port }}"
 grafana_url: "{{ grafana_url_openondemand_proxy if groups['openondemand'] | count > 0 else grafana_url_direct }}"
 grafana_serve_from_sub_path: "{{ groups['openondemand'] | count > 0 }}"
 

environments/common/inventory/group_vars/all/prometheus.yml

@@ -28,10 +28,6 @@ prometheus_scrape_configs_default:
   static_configs:
     - targets:
       - "{{ control_ip }}:{{ slurm_exporter_port }}"
-- job_name: "grafana"
-  static_configs:
-  - targets:
-    - "kube-prometheus-stack-grafana:80"
 
 kube_prometheus_stack_release_namespace: monitoring-system