ufm telemetry support [metrics,syslog]

common/library/module_utils/input_validation/schema/telemetry_config.json

@@ -94,9 +94,37 @@
             }
           },
           "required": ["metrics_enabled", "collection_targets"]
+        },
+        "ufm": {
+          "type": "object",
+          "description": "NVIDIA UFM InfiniBand Fabric telemetry.",
+          "properties": {
+            "metrics_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM InfiniBand metrics collection."
+            },
+            "logs_enabled": {
+              "type": "boolean",
+              "default": false,
+              "description": "Enable or disable UFM syslog logs collection."
+            },
+            "collection_targets": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": ["victoria_metrics", "victoria_logs"]
+              },
+              "minItems": 1,
+              "uniqueItems": true,
+              "default": ["victoria_metrics"],
+              "description": "UFM supports victoria_metrics (metrics) and victoria_logs (logs)."
+            }
+          },
+          "required": ["metrics_enabled", "collection_targets"]
         }
       },
-      "required": ["idrac", "ldms", "dcgm", "powerscale"]
+      "required": ["idrac", "ldms", "dcgm", "powerscale", "ufm"]
     },
     "telemetry_bridges": {
       "type": "object",
@@ -398,7 +426,55 @@
         }
       },
       "required": ["otel_collector_storage_size", "csm_observability_values_file_path"]
+    },
+    "ufm_configuration": {
+      "type": "object",
+      "description": "UFM telemetry detailed configurations.",
+      "properties": {
+        "ufm_endpoint": {
+          "type": "string",
+          "minLength": 1,
+          "description": "UFM appliance IP address or hostname."
+        },
+        "ufm_metrics_port": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 65535,
+          "default": 9001,
+          "description": "UFM Prometheus exporter port."
+        },
+        "scrape_interval": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "30s",
+          "description": "Prometheus scrape interval for UFM metrics."
+        },
+        "scrape_timeout": {
+          "type": "string",
+          "pattern": "^[0-9]+[smh]$",
+          "default": "15s",
+          "description": "Prometheus scrape timeout (must be <= scrape_interval)."
+        },
+        "tls_mode": {
+          "type": "string",
+          "enum": ["self_signed", "ca_signed"],
+          "default": "self_signed",
+          "description": "TLS mode for connecting to UFM Prometheus endpoint."
+        },
+        "ufm_ca_cert_path": {
+          "type": "string",
+          "default": "",
+          "description": "Path to CA certificate file for UFM TLS verification."
+        },
+        "auth_mode": {
+          "type": "string",
+          "enum": ["basic", "none"],
+          "default": "basic",
+          "description": "Authentication mode for UFM Prometheus endpoint."
+        }
+      },
+      "required": ["ufm_endpoint"]
     }
   },
-  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations"]
+  "required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"]
 }

common/library/module_utils/input_validation/validation_flows/telemetry_validation.py

@@ -315,6 +315,7 @@ def validate_telemetry_config(
     idrac_source = telemetry_sources.get("idrac", {})
     ldms_source = telemetry_sources.get("ldms", {})
     powerscale_source = telemetry_sources.get("powerscale", {})
+    ufm_source = telemetry_sources.get("ufm", {})
 
     idrac_telemetry_support = idrac_source.get("metrics_enabled", False)
     idrac_collection_targets = idrac_source.get("collection_targets", [])
@@ -370,6 +371,17 @@ def validate_telemetry_config(
             list(invalid_powerscale_targets),
             f"Invalid collection targets for PowerScale. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_powerscale_targets}"
         ))
+
+    # UFM: supports victoria_metrics and victoria_logs
+    ufm_targets = set(ufm_source.get("collection_targets", []))
+    allowed_ufm_targets = {"victoria_metrics", "victoria_logs"}
+    invalid_ufm_targets = ufm_targets - allowed_ufm_targets
+    if invalid_ufm_targets:
+        errors.append(create_error_msg(
+            "telemetry_sources.ufm.collection_targets",
+            list(invalid_ufm_targets),
+            f"Invalid collection targets for UFM. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_ufm_targets}"
+        ))
 
     # =========================================================================
     # Validate service cluster and slurm cluster
@@ -771,4 +783,59 @@ def validate_telemetry_config(
         is_service_cluster_defined, config_paths, logger, errors
     )
 
+    # =========================================================================
+    # Validate UFM telemetry configuration
+    # =========================================================================
+    ufm_metrics_enabled = ufm_source.get("metrics_enabled", False)
+    ufm_logs_enabled = ufm_source.get("logs_enabled", False)
+    ufm_detailed_config = data.get("ufm_configuration", {})
+
+    if ufm_metrics_enabled or ufm_logs_enabled:
+        # Check required UFM endpoint
+        ufm_endpoint = ufm_detailed_config.get("ufm_endpoint", "")
+        if not ufm_endpoint or (isinstance(ufm_endpoint, str) and ufm_endpoint.strip() == ""):
+            errors.append(create_error_msg(
+                "ufm_configuration.ufm_endpoint",
+                ufm_endpoint,
+                "ufm_endpoint is required when UFM telemetry is enabled. Provide the UFM appliance IP address or hostname."
+            ))
+
+        # Validate UFM metrics port if metrics enabled
+        if ufm_metrics_enabled:
+            ufm_metrics_port = ufm_detailed_config.get("ufm_metrics_port", 9001)
+            if not isinstance(ufm_metrics_port, int) or ufm_metrics_port < 1 or ufm_metrics_port > 65535:
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_metrics_port",
+                    ufm_metrics_port,
+                    "ufm_metrics_port must be an integer between 1 and 65535."
+                ))
+
+        # Validate TLS mode
+        tls_mode = ufm_detailed_config.get("tls_mode", "self_signed")
+        if tls_mode not in ["self_signed", "ca_signed"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.tls_mode",
+                tls_mode,
+                "tls_mode must be 'self_signed' or 'ca_signed'."
+            ))
+
+        # Validate CA certificate path when tls_mode is ca_signed
+        if tls_mode == "ca_signed":
+            ca_cert_path = ufm_detailed_config.get("ufm_ca_cert_path", "")
+            if not ca_cert_path or (isinstance(ca_cert_path, str) and ca_cert_path.strip() == ""):
+                errors.append(create_error_msg(
+                    "ufm_configuration.ufm_ca_cert_path",
+                    ca_cert_path,
+                    "ufm_ca_cert_path is required when tls_mode is 'ca_signed'. Provide path to CA certificate file."
+                ))
+
+        # Validate auth mode
+        auth_mode = ufm_detailed_config.get("auth_mode", "basic")
+        if auth_mode not in ["basic", "none"]:
+            errors.append(create_error_msg(
+                "ufm_configuration.auth_mode",
+                auth_mode,
+                "auth_mode must be 'basic' or 'none'."
+            ))
+
     return errors

common/library/modules/fetch_telemetry_status.py

@@ -72,6 +72,13 @@ def main():
     if telemetry_sources.get("idrac", {}).get("metrics_enabled", False):
         telemetry_status_list.append("idrac_telemetry")
 
+    # Check UFM telemetry
+    ufm_config = telemetry_sources.get("ufm", {})
+    if ufm_config.get("metrics_enabled", False):
+        telemetry_status_list.append("ufm_telemetry")
+    if ufm_config.get("logs_enabled", False):
+        telemetry_status_list.append("ufm_logs")
+
     module.exit_json(
             changed=False,
             telemetry_status_list=telemetry_status_list

input/telemetry_config.yml

@@ -146,6 +146,29 @@ telemetry_sources:
       - "victoria_metrics"
       - "victoria_logs"
 
+  # --------------------------------------------------------------------------
+  # UFM — NVIDIA UFM InfiniBand Fabric Telemetry
+  # --------------------------------------------------------------------------
+  # Collects: IB port state, transmit/receive data, error counters, fabric topology
+  # Requires: NVIDIA UFM appliance with Prometheus exporter enabled (port 9090)
+  # Data path: UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+  #
+  # NOTE: Omnia does NOT deploy UFM itself. Omnia configures the existing
+  # vmagent to scrape the UFM Prometheus endpoint for IB fabric metrics.
+  ufm:
+    # Enable or disable UFM InfiniBand metrics collection
+    # Default: false
+    metrics_enabled: false
+
+    # Enable or disable UFM syslog logs collection
+    # Default: false
+    logs_enabled: false
+
+    # UFM uses vmagent(shared) for metrics and VLAgent for logs
+    collection_targets:
+      - "victoria_metrics"
+      - "victoria_logs"
+
 
 # ============================================================================
 # TELEMETRY BRIDGES (Data Routers)
@@ -349,3 +372,54 @@ powerscale_configurations:
   # Required when powerscale_configurations.powerscale_telemetry_support: true
   # Reference: https://raw.githubusercontent.com/dell/helm-charts/refs/heads/release-v1.16.3/charts/karavi-observability/values.yaml
   csm_observability_values_file_path: ""
+
+# --------------------------------------------------------------------------
+# UFM Telemetry Configuration
+# --------------------------------------------------------------------------
+# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
+# using Prometheus scraping.
+#
+# DATA PIPELINE:
+#   UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
+#
+# NOTE: UFM does NOT use Vector bridges. It uses the shared vmagent instance
+#       for metrics collection.
+ufm_configuration:
+  # UFM appliance IP address or hostname
+  # Required when telemetry_sources.ufm.metrics_enabled is true
+  # Example: "172.20.44.180" or "ufm.example.com"
+  ufm_endpoint: ""
+
+  # UFM Prometheus exporter port
+  # Default: 9001 (UFM default Prometheus port)
+  ufm_metrics_port: 9001
+
+  # Prometheus scrape interval for UFM metrics
+  # Accepted values: Prometheus duration format (e.g., "15s", "30s", "1m")
+  # Default: "30s"
+  scrape_interval: "30s"
+
+  # Prometheus scrape timeout (must be <= scrape_interval)
+  # Accepted values: Prometheus duration format (e.g., "10s", "15s")
+  # Default: "15s"
+  scrape_timeout: "15s"
+
+  # TLS mode for connecting to UFM Prometheus endpoint
+  # Accepted values: "self_signed", "ca_signed"
+  # - self_signed: Skip TLS verification (insecure_skip_verify=true)
+  # - ca_signed: Use CA certificate for TLS verification
+  # Default: "self_signed"
+  tls_mode: "self_signed"
+
+  # Path to CA certificate file for UFM TLS verification
+  # Required when tls_mode is "ca_signed"
+  # Must be a valid PEM-format certificate file
+  # Default: "" (empty — not used when tls_mode is "self_signed")
+  ufm_ca_cert_path: ""
+
+  # Authentication mode for UFM Prometheus endpoint
+  # Accepted values: "basic", "none"
+  # - basic: Use ufm_username/ufm_password from omnia_config_credentials.yml
+  # - none: No authentication (UFM endpoint is open)
+  # Default: "basic"
+  auth_mode: "basic"

provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2

@@ -321,6 +321,20 @@
               ipAddressPools:
               - first-pool
 
+{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
+        - path: /root/telemetry.sh
+          owner: root:root
+          permissions: '0755'
+          content: |
+            {{ lookup('template', 'templates/telemetry/telemetry.sh.j2') | indent(12) }}
+{% endif %}
+{% for pv_entry in cloud_init_groups_dict[functional_group_name].powervault_scripts | default([]) %}
+        - path: /usr/local/bin/setup_iscsi_storage_{{ pv_entry.name }}.sh
+          permissions: '{{ file_mode_755 }}'
+          content: |
+            {{ pv_entry.content | indent(12) }}
+{% endfor %}
+
       runcmd:
         - /usr/local/bin/set-ssh.sh
         - "systemctl enable chronyd"
@@ -1059,7 +1073,7 @@
 {% include 'powerscale/deploy_powerscale_telemetry.sh.j2' %}
 {% endif %}
 
-{% if telemetry_enabled %}
+{% if telemetry_enabled or ldms_support or ufm_telemetry_support%}
             echo "Applying Telemetry Kubernetes deployments"
             {{ k8s_client_mount_path }}/telemetry/telemetry.sh
 {% endif %}