ceph_migrate: Enable firewall by default and enhance Prometheus/EDPM integration

rebtoor · rebtoor · commit b493fc5e199a · 2025-10-15T09:42:24.000+02:00
This commit modernizes the ceph_migrate role with several key improvements:

**Firewall Configuration:**
- Enable firewall by default (`ceph_firewall_enabled: true`)
- Update nftables rules path from TripleO to EDPM conventions
  (`/etc/nftables/tripleo-rules.nft` → `/etc/nftables/edpm-rules.nft`)
- Replace TRIPLEO_INPUT chain references with EDPM_INPUT in nftables rules
- Add proper file permissions (mode: 0644) for nftables configuration
- Remove firewall service stopping logic that temporarily disabled iptables/nftables
- Add insertbefore directive for proper rule placement before INPUT chain lockdown
- Update firewall enablement conditions to use new default (true)

**Prometheus Integration:**
- Add configurable Prometheus server settings:
  - `ceph_prometheus_server_port: 9283` (default)
  - `ceph_prometheus_server_addr: "0.0.0.0"` (default)
- Implement Prometheus module configuration in monitoring tasks
- Add firewall rules for Prometheus port (9283) in both iptables and nftables
- Enable Prometheus module automatically during ceph-mgr setup

**Firewall Rules Improvements:**
- Consolidate and reorganize nftables rules with cleaner structure
- Add missing port 12049 for ceph_nfs backend in nftables rules
- Improve rule comments for better readability and consistency
- Remove redundant ceph rgw rule (122) that duplicated existing ports
- Simplify ceph_dashboard rule (123) by removing duplicate ports

**Post-migration Enhancements:**
- Add mgr failover task delegation to ComputeHCI nodes with proper conditionals
- Include safety checks for ComputeHCI group existence and ceph_cli definition

**Documentation:**
- Document new Prometheus configuration variables in README.md

These changes align the role with EDPM (Edge Deployment and Management)
conventions while improving security through default firewall enablement
and enhanced monitoring capabilities.

Signed-off-by: Roberto Alfieri &lt;ralfieri@redhat.com&gt;
diff --git a/tests/roles/ceph_migrate/README.md b/tests/roles/ceph_migrate/README.md
@@ -66,6 +66,10 @@ ceph_prometheus_container_image: "quay.io/prometheus/prometheus:v2.43.0"
 ceph_spec_render_dir: "/home/tripleo-admin"
 endif::[]
 
+# Prometheus module configuration
+ceph_prometheus_server_port: 9283
+ceph_prometheus_server_addr: "0.0.0.0"
+
 ceph_rgw_virtual_ips_list:
   - 172.17.3.99/24
 #  - 10.0.0.99/24 # this requires the external network on the cephstorage node
diff --git a/tests/roles/ceph_migrate/defaults/main.yaml b/tests/roles/ceph_migrate/defaults/main.yaml
@@ -32,11 +32,11 @@ ceph_wait_mon_timeout: 10
 ceph_keep_mon_ipaddr: true
 
 # firewall section
-ceph_firewall_enabled: false
+ceph_firewall_enabled: true
 ceph_iptables_path:
   - "/etc/sysconfig/iptables"
   - "/etc/sysconfig/ip6tables"
-ceph_nftables_path: "/etc/nftables/tripleo-rules.nft"
+ceph_nftables_path: "/etc/nftables/edpm-rules.nft"
 ceph_firewall_type: nftables
 
 # DEFAULT Ceph Reef container images
@@ -46,6 +46,10 @@ ceph_alertmanager_container_image: "quay.io/prometheus/alertmanager:v0.25.0"
 ceph_grafana_container_image: "quay.io/ceph/ceph-grafana:9.4.7"
 ceph_node_exporter_container_image: "quay.io/prometheus/node-exporter:v1.5.0"
 ceph_prometheus_container_image: "quay.io/prometheus/prometheus:v2.43.0"
+
+# Prometheus module configuration
+ceph_prometheus_server_port: 9283
+ceph_prometheus_server_addr: "0.0.0.0"
 ceph_storagenfs_nic: "nic2"
 ceph_storagenfs_vlan_id: "70"
 rhoso_namespace: "openstack"
diff --git a/tests/roles/ceph_migrate/handlers/main.yml b/tests/roles/ceph_migrate/handlers/main.yml
@@ -3,3 +3,8 @@
   become: true
   ansible.builtin.command:
     "{{ ceph_cli }} mgr fail"
+  delegate_to: "{{ groups['ComputeHCI'][0] | default(inventory_hostname) }}"
+  when:
+    - groups['ComputeHCI'] is defined
+    - groups['ComputeHCI'] | length > 0
+    - ceph_cli is defined
diff --git a/tests/roles/ceph_migrate/tasks/firewall.yaml b/tests/roles/ceph_migrate/tasks/firewall.yaml
@@ -1,17 +1,9 @@
 # Add firewall rules for all the Ceph Services
 
-- name: Ensure firewall is temporarily stopped
-  delegate_to: "{{ node }}"
-  become: true
-  ansible.builtin.systemd:
-    name: "{{ item }}"
-    state: stopped
-  loop:
-    - iptables
-    - nftables
-
 - name: Manage Ceph iptables rules
-  when: ceph_firewall_type == "iptables"
+  when:
+    - ceph_firewall_enabled | bool | default(true)
+    - ceph_firewall_type == "iptables"
   block:
     - name: Ceph Migration - Apply the Ceph cluster rules (iptables)
       delegate_to: "{{ node }}"
@@ -29,10 +21,13 @@
           -A INPUT -p tcp -m tcp --dport 2049 -m conntrack --ctstate NEW -m comment --comment "111 ceph_nfs ipv4" -j ACCEPT
           -A INPUT -p tcp -m tcp --dport 12049 -m conntrack --ctstate NEW -m comment --comment "111 ceph_nfs_backend ipv4" -j ACCEPT
           -A INPUT -p tcp -m tcp --dport 6800:7300 -m conntrack --ctstate NEW -m comment --comment "112 ceph_mds_mgr ipv4" -j ACCEPT
+          -A INPUT -p tcp -m tcp --dport 9283 -m conntrack --ctstate NEW -m comment --comment "113 ceph_prometheus ipv4" -j ACCEPT
       loop: "{{ ceph_iptables_path }}"
 
     - name: Ensure firewall is enabled/started - iptables
-      when: ceph_firewall_enabled | bool | default(false)
+      when:
+        - ceph_firewall_enabled | bool | default(true)
+        - ceph_firewall_type == "iptables"
       delegate_to: "{{ node }}"
       become: true
       ansible.builtin.systemd:
@@ -41,7 +36,9 @@
         enabled: true
 
 - name: Manage Ceph nftables rules
-  when: ceph_firewall_type == "nftables"
+  when:
+    - ceph_firewall_enabled | bool | default(true)
+    - ceph_firewall_type == "nftables"
   block:
     - name: Ceph Migration - Apply the Ceph cluster rules (nftables)
       delegate_to: "{{ node }}"
@@ -50,32 +47,34 @@
         marker_begin: "BEGIN ceph firewall rules"
         marker_end: "END ceph firewall rules"
         path: "{{ ceph_nftables_path }}"
+        mode: "0644"
         block: |
-           # 100 ceph_alertmanager {'dport': [9093]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 9093 } ct state new counter accept comment "100 ceph_alertmanager"
-           # 100 ceph_dashboard {'dport': [8443]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 8443 } ct state new counter accept comment "100 ceph_dashboard"
-           # 100 ceph_grafana {'dport': [3100]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 3100 } ct state new counter accept comment "100 ceph_grafana"
-           # 100 ceph_prometheus {'dport': [9092]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 9092 } ct state new counter accept comment "100 ceph_prometheus"
-           # 100 ceph_rgw {'dport': ['8080']}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 8080 } ct state new counter accept comment "100 ceph_rgw"
-           # 110 ceph_mon {'dport': [6789, 3300, '9100']}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 6789,3300,9100 } ct state new counter accept comment "110 ceph_mon"
-           # 112 ceph_mds {'dport': ['6800-7300', '9100']}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 6800-7300,9100 } ct state new counter accept comment "112 ceph_mds"
-           # 113 ceph_mgr {'dport': ['6800-7300', 8444]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 6800-7300,8444 } ct state new counter accept comment "113 ceph_mgr"
-           # 120 ceph_nfs {'dport': ['12049', '2049']}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 2049 } ct state new counter accept comment "120 ceph_nfs"
-           # 122 ceph rgw {'dport': ['8080', '8080', '9100']}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 8080,8080,9100 } ct state new counter accept comment "122 ceph rgw"
-           # 123 ceph_dashboard {'dport': [3100, 9090, 9092, 9093, 9094, 9100, 9283]}
-           add rule inet filter TRIPLEO_INPUT tcp dport { 3100,9090,9092,9093,9094,9100,9283 } ct state new counter accept comment "123 ceph_dashboard"
+           # 100 ceph_alertmanager (9093)
+           add rule inet filter EDPM_INPUT tcp dport { 9093 } ct state new counter accept comment "100 ceph_alertmanager"
+           # 100 ceph_dashboard (8443)
+           add rule inet filter EDPM_INPUT tcp dport { 8443 } ct state new counter accept comment "100 ceph_dashboard"
+           # 100 ceph_grafana (3100)
+           add rule inet filter EDPM_INPUT tcp dport { 3100 } ct state new counter accept comment "100 ceph_grafana"
+           # 100 ceph_prometheus (9092)
+           add rule inet filter EDPM_INPUT tcp dport { 9092 } ct state new counter accept comment "100 ceph_prometheus"
+           # 100 ceph_rgw (8080)
+           add rule inet filter EDPM_INPUT tcp dport { 8080 } ct state new counter accept comment "100 ceph_rgw"
+           # 110 ceph_mon (6789, 3300, 9100)
+           add rule inet filter EDPM_INPUT tcp dport { 6789,3300,9100 } ct state new counter accept comment "110 ceph_mon"
+           # 112 ceph_mds (6800-7300, 9100)
+           add rule inet filter EDPM_INPUT tcp dport { 6800-7300,9100 } ct state new counter accept comment "112 ceph_mds"
+           # 113 ceph_mgr (6800-7300, 8444)
+           add rule inet filter EDPM_INPUT tcp dport { 6800-7300,8444 } ct state new counter accept comment "113 ceph_mgr"
+           # 120 ceph_nfs (2049, 12049)
+           add rule inet filter EDPM_INPUT tcp dport { 2049,12049 } ct state new counter accept comment "120 ceph_nfs"
+           # 123 ceph_dashboard (9090, 9094, 9283)
+           add rule inet filter EDPM_INPUT tcp dport { 9090,9094,9283 } ct state new counter accept comment "123 ceph_dashboard"
+        insertbefore: '^# Lock down INPUT chains'
 
     - name: Ensure firewall is enabled/started - nftables
-      when: ceph_firewall_enabled | bool | default(false)
+      when:
+        - ceph_firewall_enabled | bool | default(true)
+        - ceph_firewall_type == "nftables"
       delegate_to: "{{ node }}"
       become: true
       ansible.builtin.systemd:
diff --git a/tests/roles/ceph_migrate/tasks/monitoring.yaml b/tests/roles/ceph_migrate/tasks/monitoring.yaml
@@ -24,6 +24,23 @@
       ansible.builtin.command: |
         {{ ceph_cli }} mgr module enable dashboard
 
+- name: Set ceph-mgr prometheus port configuration
+  # cephadm runs w/ root privileges
+  become: true
+  block:
+    - name: Set the prometheus server port
+      ansible.builtin.command: |
+        {{ ceph_cli }} config set mgr mgr/prometheus/server_port {{ ceph_prometheus_server_port }}
+      changed_when: false
+    - name: Set the prometheus server address
+      ansible.builtin.command: |
+        {{ ceph_cli }} config set mgr mgr/prometheus/server_addr {{ ceph_prometheus_server_addr }}
+      changed_when: false
+    - name: Enable prometheus module
+      ansible.builtin.command: |
+        {{ ceph_cli }} mgr module enable prometheus
+      changed_when: false
+
 # - Expand labels to the whole hostmap
 - name: Apply Monitoring label to the overcloud nodes
   ansible.builtin.import_tasks: labels.yaml
