Skip to content

Commit 9adea7c

Browse files
pullan1abhishek-sa1
pullan1
and
abhishek-sa1
authored
ufm telemetry support [metrics,syslog] (#4405)
* ufm telemetry support [metrics,syslog] Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * ansible lint fix Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * ansible lint fix Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * ansible lint fix Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * updated ufm config Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * added verbosity to the msg Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * reference of deploy_powerscale_logs.yml removed Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * added ufm credential in config credentials utils Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * resolved merge conflicts Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * Removed debug msgs Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * updated the ufm secrets flow Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * Fixes for merge conflicts and ufm Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * merge conflicts resolved Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> * Update telemetry_secret_creation.yaml.j2 Signed-off-by: Abhishek S A <abhishek.sa3@dell.com> * Update main.yml Signed-off-by: Abhishek S A <abhishek.sa3@dell.com> --------- Signed-off-by: pullan1 <sudha.pullalaravu@dell.com> Signed-off-by: Abhishek S A <abhishek.sa3@dell.com> Co-authored-by: Abhishek S A <abhishek.sa3@dell.com>
1 parent 0764fbb commit 9adea7c

21 files changed

Lines changed: 667 additions & 14 deletions

File tree

‎common/library/module_utils/input_validation/schema/credential_rules.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,5 +161,17 @@
161161
"maxLength": 128,
162162
"pattern": "^.+$",
163163
"description": "Password for Dell OpenManage Enterprise (OME). Length must be between 1 and 128 characters."
164+
},
165+
"ufm_username": {
166+
"minLength": 1,
167+
"maxLength": 64,
168+
"pattern": "^[^\\\\'\";]+$",
169+
"description": "Username for NVIDIA UFM (Unified Fabric Manager). Must not contain backslashes (\\), single quotes ('), double quotes (\"), or semicolons (;). Length must be between 1 and 64 characters."
170+
},
171+
"ufm_password": {
172+
"minLength": 1,
173+
"maxLength": 128,
174+
"pattern": "^.+$",
175+
"description": "Password for NVIDIA UFM (Unified Fabric Manager). Length must be between 1 and 128 characters."
164176
}
165177
}

‎common/library/module_utils/input_validation/schema/telemetry_config.json‎

Lines changed: 78 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -94,9 +94,37 @@
9494
}
9595
},
9696
"required": ["metrics_enabled", "collection_targets"]
97+
},
98+
"ufm": {
99+
"type": "object",
100+
"description": "NVIDIA UFM InfiniBand Fabric telemetry.",
101+
"properties": {
102+
"metrics_enabled": {
103+
"type": "boolean",
104+
"default": false,
105+
"description": "Enable or disable UFM InfiniBand metrics collection."
106+
},
107+
"logs_enabled": {
108+
"type": "boolean",
109+
"default": false,
110+
"description": "Enable or disable UFM syslog logs collection."
111+
},
112+
"collection_targets": {
113+
"type": "array",
114+
"items": {
115+
"type": "string",
116+
"enum": ["victoria_metrics", "victoria_logs"]
117+
},
118+
"minItems": 1,
119+
"uniqueItems": true,
120+
"default": ["victoria_metrics"],
121+
"description": "UFM supports victoria_metrics (metrics) and victoria_logs (logs)."
122+
}
123+
},
124+
"required": ["metrics_enabled", "collection_targets"]
97125
}
98126
},
99-
"required": ["idrac", "ldms", "dcgm", "powerscale"]
127+
"required": ["idrac", "ldms", "dcgm", "powerscale", "ufm"]
100128
},
101129
"telemetry_bridges": {
102130
"type": "object",
@@ -398,7 +426,55 @@
398426
}
399427
},
400428
"required": ["otel_collector_storage_size", "csm_observability_values_file_path"]
429+
},
430+
"ufm_configuration": {
431+
"type": "object",
432+
"description": "UFM telemetry detailed configurations.",
433+
"properties": {
434+
"ufm_endpoint": {
435+
"type": "string",
436+
"minLength": 1,
437+
"description": "UFM appliance IP address or hostname."
438+
},
439+
"ufm_metrics_port": {
440+
"type": "integer",
441+
"minimum": 1,
442+
"maximum": 65535,
443+
"default": 9001,
444+
"description": "UFM Prometheus exporter port."
445+
},
446+
"scrape_interval": {
447+
"type": "string",
448+
"pattern": "^[0-9]+[smh]$",
449+
"default": "30s",
450+
"description": "Prometheus scrape interval for UFM metrics."
451+
},
452+
"scrape_timeout": {
453+
"type": "string",
454+
"pattern": "^[0-9]+[smh]$",
455+
"default": "15s",
456+
"description": "Prometheus scrape timeout (must be <= scrape_interval)."
457+
},
458+
"tls_mode": {
459+
"type": "string",
460+
"enum": ["self_signed", "ca_signed"],
461+
"default": "self_signed",
462+
"description": "TLS mode for connecting to UFM Prometheus endpoint."
463+
},
464+
"ufm_ca_cert_path": {
465+
"type": "string",
466+
"default": "",
467+
"description": "Path to CA certificate file for UFM TLS verification."
468+
},
469+
"auth_mode": {
470+
"type": "string",
471+
"enum": ["basic", "none"],
472+
"default": "basic",
473+
"description": "Authentication mode for UFM Prometheus endpoint."
474+
}
475+
},
476+
"required": ["ufm_endpoint"]
401477
}
402478
},
403-
"required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations"]
479+
"required": ["telemetry_sources", "telemetry_bridges", "telemetry_sinks", "ldms_configurations", "powerscale_configurations", "ufm_configuration"]
404480
}

‎common/library/module_utils/input_validation/validation_flows/telemetry_validation.py‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -328,6 +328,7 @@ def validate_telemetry_config(
328328
idrac_source = telemetry_sources.get("idrac", {})
329329
ldms_source = telemetry_sources.get("ldms", {})
330330
powerscale_source = telemetry_sources.get("powerscale", {})
331+
ufm_source = telemetry_sources.get("ufm", {})
331332

332333
idrac_telemetry_support = idrac_source.get("metrics_enabled", False)
333334
idrac_collection_targets = idrac_source.get("collection_targets", [])
@@ -383,6 +384,17 @@ def validate_telemetry_config(
383384
list(invalid_powerscale_targets),
384385
f"Invalid collection targets for PowerScale. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_powerscale_targets}"
385386
))
387+
388+
# UFM: supports victoria_metrics and victoria_logs
389+
ufm_targets = set(ufm_source.get("collection_targets", []))
390+
allowed_ufm_targets = {"victoria_metrics", "victoria_logs"}
391+
invalid_ufm_targets = ufm_targets - allowed_ufm_targets
392+
if invalid_ufm_targets:
393+
errors.append(create_error_msg(
394+
"telemetry_sources.ufm.collection_targets",
395+
list(invalid_ufm_targets),
396+
f"Invalid collection targets for UFM. Only 'victoria_metrics' and 'victoria_logs' are supported. Found: {invalid_ufm_targets}"
397+
))
386398

387399
# =========================================================================
388400
# Validate service cluster and slurm cluster
@@ -784,4 +796,59 @@ def validate_telemetry_config(
784796
is_service_cluster_defined, config_paths, logger, errors
785797
)
786798

799+
# =========================================================================
800+
# Validate UFM telemetry configuration
801+
# =========================================================================
802+
ufm_metrics_enabled = ufm_source.get("metrics_enabled", False)
803+
ufm_logs_enabled = ufm_source.get("logs_enabled", False)
804+
ufm_detailed_config = data.get("ufm_configuration", {})
805+
806+
if ufm_metrics_enabled or ufm_logs_enabled:
807+
# Check required UFM endpoint
808+
ufm_endpoint = ufm_detailed_config.get("ufm_endpoint", "")
809+
if not ufm_endpoint or (isinstance(ufm_endpoint, str) and ufm_endpoint.strip() == ""):
810+
errors.append(create_error_msg(
811+
"ufm_configuration.ufm_endpoint",
812+
ufm_endpoint,
813+
"ufm_endpoint is required when UFM telemetry is enabled. Provide the UFM appliance IP address or hostname."
814+
))
815+
816+
# Validate UFM metrics port if metrics enabled
817+
if ufm_metrics_enabled:
818+
ufm_metrics_port = ufm_detailed_config.get("ufm_metrics_port", 9001)
819+
if not isinstance(ufm_metrics_port, int) or ufm_metrics_port < 1 or ufm_metrics_port > 65535:
820+
errors.append(create_error_msg(
821+
"ufm_configuration.ufm_metrics_port",
822+
ufm_metrics_port,
823+
"ufm_metrics_port must be an integer between 1 and 65535."
824+
))
825+
826+
# Validate TLS mode
827+
tls_mode = ufm_detailed_config.get("tls_mode", "self_signed")
828+
if tls_mode not in ["self_signed", "ca_signed"]:
829+
errors.append(create_error_msg(
830+
"ufm_configuration.tls_mode",
831+
tls_mode,
832+
"tls_mode must be 'self_signed' or 'ca_signed'."
833+
))
834+
835+
# Validate CA certificate path when tls_mode is ca_signed
836+
if tls_mode == "ca_signed":
837+
ca_cert_path = ufm_detailed_config.get("ufm_ca_cert_path", "")
838+
if not ca_cert_path or (isinstance(ca_cert_path, str) and ca_cert_path.strip() == ""):
839+
errors.append(create_error_msg(
840+
"ufm_configuration.ufm_ca_cert_path",
841+
ca_cert_path,
842+
"ufm_ca_cert_path is required when tls_mode is 'ca_signed'. Provide path to CA certificate file."
843+
))
844+
845+
# Validate auth mode
846+
auth_mode = ufm_detailed_config.get("auth_mode", "basic")
847+
if auth_mode not in ["basic", "none"]:
848+
errors.append(create_error_msg(
849+
"ufm_configuration.auth_mode",
850+
auth_mode,
851+
"auth_mode must be 'basic' or 'none'."
852+
))
853+
787854
return errors

‎common/library/modules/fetch_telemetry_status.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,13 @@ def main():
7272
if telemetry_sources.get("idrac", {}).get("metrics_enabled", False):
7373
telemetry_status_list.append("idrac_telemetry")
7474

75+
# Check UFM telemetry
76+
ufm_config = telemetry_sources.get("ufm", {})
77+
if ufm_config.get("metrics_enabled", False):
78+
telemetry_status_list.append("ufm_telemetry")
79+
if ufm_config.get("logs_enabled", False):
80+
telemetry_status_list.append("ufm_logs")
81+
7582
module.exit_json(
7683
changed=False,
7784
telemetry_status_list=telemetry_status_list

‎input/telemetry_config.yml‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,29 @@ telemetry_sources:
146146
- "victoria_metrics"
147147
- "victoria_logs"
148148

149+
# --------------------------------------------------------------------------
150+
# UFM — NVIDIA UFM InfiniBand Fabric Telemetry
151+
# --------------------------------------------------------------------------
152+
# Collects: IB port state, transmit/receive data, error counters, fabric topology
153+
# Requires: NVIDIA UFM appliance with Prometheus exporter enabled (port 9090)
154+
# Data path: UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
155+
#
156+
# NOTE: Omnia does NOT deploy UFM itself. Omnia configures the existing
157+
# vmagent to scrape the UFM Prometheus endpoint for IB fabric metrics.
158+
ufm:
159+
# Enable or disable UFM InfiniBand metrics collection
160+
# Default: false
161+
metrics_enabled: false
162+
163+
# Enable or disable UFM syslog logs collection
164+
# Default: false
165+
logs_enabled: false
166+
167+
# UFM uses vmagent(shared) for metrics and VLAgent for logs
168+
collection_targets:
169+
- "victoria_metrics"
170+
- "victoria_logs"
171+
149172

150173
# ============================================================================
151174
# TELEMETRY BRIDGES (Data Routers)
@@ -349,3 +372,54 @@ powerscale_configurations:
349372
# Required when powerscale_configurations.powerscale_telemetry_support: true
350373
# Reference: https://raw.githubusercontent.com/dell/helm-charts/refs/heads/release-v1.16.3/charts/karavi-observability/values.yaml
351374
csm_observability_values_file_path: ""
375+
376+
# --------------------------------------------------------------------------
377+
# UFM Telemetry Configuration
378+
# --------------------------------------------------------------------------
379+
# UFM telemetry collects InfiniBand fabric metrics from NVIDIA UFM appliances
380+
# using Prometheus scraping.
381+
#
382+
# DATA PIPELINE:
383+
# UFM Prometheus Exporter → vmagent(shared) → victoria_metrics
384+
#
385+
# NOTE: UFM does NOT use Vector bridges. It uses the shared vmagent instance
386+
# for metrics collection.
387+
ufm_configuration:
388+
# UFM appliance IP address or hostname
389+
# Required when telemetry_sources.ufm.metrics_enabled is true
390+
# Example: "172.20.44.180" or "ufm.example.com"
391+
ufm_endpoint: ""
392+
393+
# UFM Prometheus exporter port
394+
# Default: 9001 (UFM default Prometheus port)
395+
ufm_metrics_port: 9001
396+
397+
# Prometheus scrape interval for UFM metrics
398+
# Accepted values: Prometheus duration format (e.g., "15s", "30s", "1m")
399+
# Default: "30s"
400+
scrape_interval: "30s"
401+
402+
# Prometheus scrape timeout (must be <= scrape_interval)
403+
# Accepted values: Prometheus duration format (e.g., "10s", "15s")
404+
# Default: "15s"
405+
scrape_timeout: "15s"
406+
407+
# TLS mode for connecting to UFM Prometheus endpoint
408+
# Accepted values: "self_signed", "ca_signed"
409+
# - self_signed: Skip TLS verification (insecure_skip_verify=true)
410+
# - ca_signed: Use CA certificate for TLS verification
411+
# Default: "self_signed"
412+
tls_mode: "self_signed"
413+
414+
# Path to CA certificate file for UFM TLS verification
415+
# Required when tls_mode is "ca_signed"
416+
# Must be a valid PEM-format certificate file
417+
# Default: "" (empty — not used when tls_mode is "self_signed")
418+
ufm_ca_cert_path: ""
419+
420+
# Authentication mode for UFM Prometheus endpoint
421+
# Accepted values: "basic", "none"
422+
# - basic: Use ufm_username/ufm_password from omnia_config_credentials.yml
423+
# - none: No authentication (UFM endpoint is open)
424+
# Default: "basic"
425+
auth_mode: "basic"

‎prepare_oim/prepare_oim.yml‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,9 @@
102102
(telemetry_config.telemetry_sources.ldms.metrics_enabled | default(false) | bool) or
103103
(telemetry_config.telemetry_sources.powerscale.metrics_enabled | default(false) | bool) or
104104
(telemetry_config.telemetry_sources.powerscale.logs_enabled | default(false) | bool) or
105-
(telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool)
105+
(telemetry_config.telemetry_sources.dcgm.metrics_enabled | default(false) | bool) or
106+
(telemetry_config.telemetry_sources.ufm.metrics_enabled | default(false) | bool) or
107+
(telemetry_config.telemetry_sources.ufm.logs_enabled | default(false) | bool)
106108
107109
- name: Check discovery configuration for OME
108110
block:

‎provision/roles/configure_ochami/templates/cloud_init/ci-group-service_kube_control_plane_first_x86_64.yaml.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1109,4 +1109,4 @@
11091109
kubectl get pods --all-namespaces -o wide
11101110
echo "Cloud-Init finished successfully after the reboot."
11111111

1112-
fi
1112+
fi

0 commit comments

Comments
 (0)