Deploy monitoring in mini-lab (#275)

Author: ostempel

Makefile

@@ -30,6 +30,7 @@ MAX_RETRIES := 30
 # Machine flavors
 ifeq ($(MINI_LAB_FLAVOR),sonic)
 LAB_TOPOLOGY=mini-lab.sonic.yaml
+MONITORING_ENABLED=true
 else ifeq ($(MINI_LAB_FLAVOR),dell_sonic)
 LAB_TOPOLOGY=mini-lab.dell_sonic.yaml
 MINI_LAB_SONIC_IMAGE=r.metal-stack.io/vrnetlab/dell_sonic:$(MINI_LAB_DELL_SONIC_VERSION)

compose.yaml

@@ -21,6 +21,7 @@ services:
       - DOCKER_HUB_USER=${DOCKER_HUB_USER}
       - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN}
       - GARDENER_ENABLED=${GARDENER_ENABLED:-}
+      - MONITORING_ENABLED=${MONITORING_ENABLED:-}
     network_mode: host
     working_dir: /mini-lab
     entrypoint:
@@ -48,6 +49,7 @@ services:
       - CI=${CI}
       - DOCKER_HUB_USER=${DOCKER_HUB_USER}
       - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN}
+      - MONITORING_ENABLED=${MONITORING_ENABLED:-}
     network_mode: host
     working_dir: /mini-lab
     entrypoint:

control-plane/kind.yaml

@@ -11,16 +11,12 @@ nodes:
     extraPortMappings:
       - containerPort: 4443
         hostPort: 4443
-        listenAddress: 0.0.0.0
       - containerPort: 8080
         hostPort: 8080
-        listenAddress: 0.0.0.0
       - containerPort: 4150
         hostPort: 4150
-        listenAddress: 0.0.0.0
       - containerPort: 50051
         hostPort: 50051
-        listenAddress: 0.0.0.0
     # if you want to run gardener operator + metal-stack, you need more pods
     kubeadmConfigPatches:
       - |

deploy_control_plane.yaml

@@ -31,6 +31,12 @@
       tags: auth
     - name: metal-roles/control-plane/roles/metal
       tags: metal
+    - name: metal-roles/control-plane/roles/logging
+      when: monitoring_enabled
+      tags: logging
+    - name: metal-roles/control-plane/roles/monitoring
+      when: monitoring_enabled
+      tags: monitoring
 
 - name: deploy gardener
   import_playbook: deploy_gardener.yaml

deploy_partition.yaml

@@ -87,14 +87,46 @@
     - name: metal-roles/partition/roles/pixiecore
       tags: pixiecore
 
+- name: Deploy monitoring on leaf01
+  hosts: leaf01
+  become: true
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: metal-roles/partition/roles/monitoring/prometheus
+      when: monitoring_enabled
+      tags: prometheus
+    - name: metal-roles/partition/roles/monitoring/ipmi-exporter
+      when: monitoring_enabled
+      tags: ipmi-exporter
+
+- name: Deploy monitoring on leaves
+  hosts: leaves
+  become: true
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: metal-roles/partition/roles/promtail
+      when: monitoring_enabled
+      tags: promtail
+    - name: metal-roles/partition/roles/monitoring/node-exporter
+      when: monitoring_enabled
+      tags: node-exporter
+    - name: metal-roles/partition/roles/monitoring/blackbox-exporter
+      when: monitoring_enabled
+      tags: blackbox-exporter
+    - name: metal-roles/partition/roles/monitoring/sonic-exporter
+      when: monitoring_enabled
+      tags: sonic-exporter
+
 - name: Deploy metal-core
   hosts: leaves
   any_errors_fatal: true
   become: true
   pre_tasks:
-  - name: Wait some time
-    pause:
-      seconds: 120
+    - name: Wait some time
+      pause:
+        seconds: 120
   roles:
     - name: ansible-common
       tags: always
@@ -130,8 +162,8 @@
   any_errors_fatal: true
   gather_facts: false
   pre_tasks:
-  - name: Wait until no route entries have "queued"
-    include_tasks: tasks/check_queued.yaml
+    - name: Wait until no route entries have "queued"
+      include_tasks: tasks/check_queued.yaml
 
 - name: Configure IPv6 and LLDP ports (Enterprise SONiC)
   hosts: dell_sonic
@@ -141,7 +173,7 @@
     - name: Enable IPv6 to also have LLA at VLAN interfaces
       sysctl:
         name: net.ipv6.conf.default.disable_ipv6
-        value: '0'
+        value: "0"
         state: present
         sysctl_file: /etc/sysctl.conf
     - name: Configure LLDP port IDs and descriptions

inventories/group_vars/all/control_plane.yaml

@@ -6,4 +6,7 @@ metal_control_plane_namespace: metal-control-plane
 metal_control_plane_image_pull_policy: Always
 
 helm_additional_params:
-  - '--debug'
+  - "--debug"
+
+# needs to be in all -> paritition and control-plane
+metal_api_bmc_superuser_pwd: change-me

inventories/group_vars/all/monitoring.yaml

@@ -0,0 +1,2 @@
+---
+monitoring_enabled: "{{ lookup('env', 'MONITORING_ENABLED') | default('', false) }}"

inventories/group_vars/all/release_vector.yaml

@@ -6,7 +6,6 @@ metal_stack_release_vectors:
     variable_mapping_path: metal_stack_release.mapping
     include_role_defaults: metal-roles/common/roles/defaults
     oci_cosign_verify_key: "{{ lookup('file', 'cosign.pub') }}"
-
 ##
 ## for development purposes, you can override releases from our image vector here
 ##

inventories/group_vars/control-plane/logging.yaml

@@ -0,0 +1,2 @@
+---
+logging_ingress_loki_tls: no

inventories/group_vars/control-plane/metal.yml

@@ -3,13 +3,16 @@ metal_set_resource_limits: no
 metal_check_api_health_endpoint: http://api.{{ metal_control_plane_ingress_dns }}:8080/metal/v1/health
 metal_api_headscale_control_plane_address: "http://headscale.{{ metal_control_plane_ingress_dns }}:8080"
 
+metal_api_pdb_min_available: 1
 metal_api_replicas: 1
 metal_api_view_key: metal-view
 metal_api_edit_key: metal-edit
 metal_api_admin_key: metal-admin
 
 metal_api_nsq_tcp_address: nsqd:4150
 
+metal_apiserver_pdb_min_available: 1
+
 metal_apiserver_enabled: true
 metal_apiserver_url: http://v2.api.{{ metal_control_plane_ingress_dns }}:8080
 
@@ -21,39 +24,39 @@ metal_apiserver_redis_password: change-me-soon
 metal_apiserver_admin_subjects: "[email protected]@openid-connect"
 
 metal_api_images:
-- id: firewall-ubuntu-3.0
-  name: Firewall 3 Ubuntu
-  description: Firewall 3 Ubuntu Latest Release
-  url: https://images.metal-stack.io/metal-os/stable/firewall/3.0-ubuntu/img.tar.lz4
-  features:
-    - firewall
-- id: ubuntu-24.4
-  name: Ubuntu 24.04
-  description: Ubuntu 24.04 Latest Release
-  url: https://images.metal-stack.io/metal-os/stable/ubuntu/24.04/img.tar.lz4
-  features:
-    - machine
-- id: debian-12.0
-  name: Debian 12
-  description: Debian 12 Latest Release
-  url: https://images.metal-stack.io/metal-os/stable/debian/12/img.tar.lz4
-  features:
-    - machine
+  - id: firewall-ubuntu-3.0
+    name: Firewall 3 Ubuntu
+    description: Firewall 3 Ubuntu Latest Release
+    url: https://images.metal-stack.io/metal-os/stable/firewall/3.0-ubuntu/img.tar.lz4
+    features:
+      - firewall
+  - id: ubuntu-24.4
+    name: Ubuntu 24.04
+    description: Ubuntu 24.04 Latest Release
+    url: https://images.metal-stack.io/metal-os/stable/ubuntu/24.04/img.tar.lz4
+    features:
+      - machine
+  - id: debian-12.0
+    name: Debian 12
+    description: Debian 12 Latest Release
+    url: https://images.metal-stack.io/metal-os/stable/debian/12/img.tar.lz4
+    features:
+      - machine
 
 metal_api_sizes:
-- id: v1-small-x86
-  name: v1-small-x86
-  description: The Tiny Virtual
-  constraints:
-  - type: cores
-    min: 1
-    max: 4
-  - type: memory
-    min: "{{ '500MB' | humanfriendly }}"
-    max: "{{ '4GB' | humanfriendly }}"
-  - type: storage
-    min: "{{ '1GB' | humanfriendly }}"
-    max: "{{ '100GB' | humanfriendly }}"
+  - id: v1-small-x86
+    name: v1-small-x86
+    description: The Tiny Virtual
+    constraints:
+      - type: cores
+        min: 1
+        max: 4
+      - type: memory
+        min: "{{ '500MB' | humanfriendly }}"
+        max: "{{ '4GB' | humanfriendly }}"
+      - type: storage
+        min: "{{ '1GB' | humanfriendly }}"
+        max: "{{ '100GB' | humanfriendly }}"
 
 metal_api_partitions:
   - id: mini-lab
@@ -66,61 +69,61 @@ metal_api_partitions:
     privatenetworkprefixlength: 22
 
 metal_api_networks:
-- id: tenant-super-network-mini-lab
-  name: "Project Super Network"
-  description: "Super network of all project networks"
-  nat: false
-  privatesuper: true
-  underlay: false
-  destinationprefixes: []
-  partitionid: mini-lab
-  defaultchildprefixlength:
-    IPv4: 22
-    IPv6: 96
-  prefixes:
-  - 10.0.0.0/16
-  - 2001:db8:0:10::/64
-  additionalAnnouncableCIDRs:
-  - 10.240.0.0/12
-- id: internet-mini-lab
-  name: "Virtual Internet Network"
-  description: "Virtual Internet Network for mini-lab"
-  nat: true
-  privatesuper: false
-  underlay: false
-  destinationprefixes:
-    - 0.0.0.0/0
-    - ::/0
-  partitionid: "mini-lab"
-  vrf: 104009
-  prefixes:
-  - 203.0.113.128/25
-  - 2001:db8:0:113::/64
-  labels:
-    network.metal-stack.io/default: ""
-    network.metal-stack.io/default-external: ""
-- id: underlay-mini-lab
-  name: "Underlay Network"
-  description: "Underlay Network for mini-lab"
-  nat: false
-  privatesuper: false
-  underlay: true
-  destinationprefixes: []
-  partitionid: "mini-lab"
-  prefixes:
-  - 10.1.0.0/24
+  - id: tenant-super-network-mini-lab
+    name: "Project Super Network"
+    description: "Super network of all project networks"
+    nat: false
+    privatesuper: true
+    underlay: false
+    destinationprefixes: []
+    partitionid: mini-lab
+    defaultchildprefixlength:
+      IPv4: 22
+      IPv6: 96
+    prefixes:
+      - 10.0.0.0/16
+      - 2001:db8:0:10::/64
+    additionalAnnouncableCIDRs:
+      - 10.240.0.0/12
+  - id: internet-mini-lab
+    name: "Virtual Internet Network"
+    description: "Virtual Internet Network for mini-lab"
+    nat: true
+    privatesuper: false
+    underlay: false
+    destinationprefixes:
+      - 0.0.0.0/0
+      - ::/0
+    partitionid: "mini-lab"
+    vrf: 104009
+    prefixes:
+      - 203.0.113.128/25
+      - 2001:db8:0:113::/64
+    labels:
+      network.metal-stack.io/default: ""
+      network.metal-stack.io/default-external: ""
+  - id: underlay-mini-lab
+    name: "Underlay Network"
+    description: "Underlay Network for mini-lab"
+    nat: false
+    privatesuper: false
+    underlay: true
+    destinationprefixes: []
+    partitionid: "mini-lab"
+    prefixes:
+      - 10.1.0.0/24
 
 metal_api_ips:
-- name: "reserve v4"
-  description: "Reserve IPv4 (for testing purposes)"
-  networkid: internet-mini-lab
-  projectid: 00000000-0000-0000-0000-000000000000
-  ipaddress: "203.0.113.142"
-- name: "reserve v6"
-  description: "Reserve IPv6 (for testing purposes)"
-  networkid: internet-mini-lab
-  projectid: 00000000-0000-0000-0000-000000000000
-  ipaddress: "2001:db8:0:113::142"
+  - name: "reserve v4"
+    description: "Reserve IPv4 (for testing purposes)"
+    networkid: internet-mini-lab
+    projectid: 00000000-0000-0000-0000-000000000000
+    ipaddress: "203.0.113.142"
+  - name: "reserve v6"
+    description: "Reserve IPv6 (for testing purposes)"
+    networkid: internet-mini-lab
+    projectid: 00000000-0000-0000-0000-000000000000
+    ipaddress: "2001:db8:0:113::142"
 
 metal_masterdata_api_tls_ca: "{{ lookup('file', 'certs/ca.pem') }}"
 metal_masterdata_api_tls_cert: "{{ lookup('file', 'certs/masterdata-api/server.pem') }}"
@@ -132,23 +135,23 @@ metal_masterdata_api_tls_client_key: "{{ lookup('file', 'certs/masterdata-api/cl
 metal_masterdata_api_port: 8445
 
 metal_masterdata_api_tenants:
-- meta:
-    id: metal-stack
-    kind: Tenant
-    apiversion: v1
-    version: 0
-  name: metal-stack
-  iam_config:
-  description: metal-stack tenant, which is provider
+  - meta:
+      id: metal-stack
+      kind: Tenant
+      apiversion: v1
+      version: 0
+    name: metal-stack
+    iam_config:
+    description: metal-stack tenant, which is provider
 
 metal_masterdata_api_projects:
-- meta:
-    id: 00000000-0000-0000-0000-000000000001
-    kind: Project
-    apiversion: v1
-    version: 0
-  name: sample-project
-  description: Sample project with static id
+  - meta:
+      id: 00000000-0000-0000-0000-000000000001
+      kind: Project
+      apiversion: v1
+      version: 0
+    name: sample-project
+    description: Sample project with static id
 
 metal_console_enabled: false