disable and enable powerscale telemetry

common/library/module_utils/input_validation/validation_flows/provision_validation.py

@@ -394,6 +394,11 @@ def validate_mapping_file_entries(mapping_file_path):
     if not reader.fieldnames:
         raise ValueError("CSV header not found in mapping file.")
 
+    # Check for leading/trailing whitespace in header names
+    for fn in reader.fieldnames:
+        if fn != fn.strip():
+            raise ValueError(f"Header '{fn}' has leading or trailing whitespace. Please remove all whitespace from header names in mapping file.")
+
     # Map header names case-insensitively to original names
     fieldname_map = {fn.strip().upper(): fn for fn in reader.fieldnames}
 
@@ -411,6 +416,12 @@ def validate_mapping_file_entries(mapping_file_path):
     row_seen = False
     for row_idx, row in enumerate(reader, start=2):  # start=2 approximates CSV row number
         row_seen = True
+
+        # Check for leading/trailing whitespace in all field values
+        for col, val in row.items():
+            if val is not None and val != val.strip():
+                raise ValueError(f"Field '{col}' at CSV row {row_idx} has leading or trailing whitespace. Please remove all whitespace from field values in mapping file.")
+
         # Check presence and non-empty for all required headers
         for hdr in required_headers:
             col = fieldname_map[hdr]

provision/roles/telemetry/tasks/check_kube_vip_reachability.yml → common/tasks/common/check_kube_vip_reachability.yml

@@ -17,7 +17,6 @@
   when:
     - kube_vip is defined
     - kube_vip | length > 0
-  tags: telemetry_deployment
   block:
     - name: Set kube_vip reachability fact to false initially
       ansible.builtin.set_fact:

common/tasks/common/load_ha_config.yml

@@ -0,0 +1,33 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Load high_availability_config.yml
+  ansible.builtin.include_vars:
+    file: "{{ ha_config_file }}"
+  register: ha_config_loaded
+  ignore_errors: true
+
+- name: Set kube_vip fact
+  ansible.builtin.set_fact:
+    kube_vip: "{{ service_k8s_cluster_ha[0].virtual_ip_address | default('') }}"
+  when: ha_config_loaded is succeeded
+
+- name: Fail if kube_vip is empty
+  ansible.builtin.fail:
+    msg: "kube_vip is not set in high_availability_config.yml. Please configure service_k8s_cluster_ha[0].virtual_ip_address"
+  when:
+    - ha_config_loaded is succeeded
+    - kube_vip is defined
+    - kube_vip | length == 0

provision/roles/telemetry/tasks/main.yml

@@ -28,7 +28,7 @@
   ansible.builtin.include_tasks: load_service_images.yml
 
 - name: Check kube_vip reachability for validation
-  ansible.builtin.include_tasks: check_kube_vip_reachability.yml
+  ansible.builtin.include_tasks: "{{ playbook_dir }}/../common/tasks/telemetry/check_kube_vip_reachability.yml"
   when:
     - victoria_metrics_support | default(false) | bool
     - kube_vip is defined

provision/roles/telemetry/templates/telemetry/telemetry.sh.j2

@@ -100,4 +100,50 @@ else
 fi
 {% endif %}
 
+{% if victoria_logs_support %}
+# Check reachability of additional log write endpoints
+{% if telemetry_config.telemetry_sinks.victoria_logs.additional_log_write_endpoints | default([]) %}
+echo "Checking reachability of additional log write endpoints..."
+# Wait for VLAgent to be ready before checking endpoint reachability
+echo "  Waiting for VLAgent to be ready..."
+kubectl wait --for=condition=ready --timeout=300s statefulset/vlagent -n telemetry || echo "  WARNING: VLAgent not ready within timeout"
+
+VLAGENT_POD=$(kubectl get pod -n telemetry -l app.kubernetes.io/name=vlagent -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+if [ -n "$VLAGENT_POD" ]; then
+  {% for endpoint in telemetry_config.telemetry_sinks.victoria_logs.additional_log_write_endpoints %}
+  echo "  Testing connectivity to: {{ endpoint.url }}"
+  # Test connectivity using wget (more reliable than curl in minimal containers)
+  kubectl exec -n telemetry "$VLAGENT_POD" -- wget -T 5 -q --spider "{{ endpoint.url }}" 2>/dev/null && \
+    echo "    ✓ Endpoint reachable" || \
+    echo "    WARNING: Endpoint unreachable - logs may not be forwarded to {{ endpoint.url }}"
+  {% endfor %}
+else
+  echo "  WARNING: Could not find VLAgent pod to check endpoint reachability"
+fi
+{% endif %}
+{% endif %}
+
+{% if victoria_metrics_support %}
+# Check reachability of additional metric remote write endpoints
+{% if telemetry_config.telemetry_sinks.victoria_metrics.additional_metric_remote_write_endpoints | default([]) %}
+echo "Checking reachability of additional metric remote write endpoints..."
+# Wait for vmagent to be ready before checking endpoint reachability
+echo "  Waiting for vmagent to be ready..."
+kubectl wait --for=condition=ready --timeout=300s deployment/vmagent -n telemetry || echo "  WARNING: vmagent not ready within timeout"
+
+VMAGENT_POD=$(kubectl get pod -n telemetry -l app.kubernetes.io/name=vmagent -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+if [ -n "$VMAGENT_POD" ]; then
+  {% for endpoint in telemetry_config.telemetry_sinks.victoria_metrics.additional_metric_remote_write_endpoints %}
+  echo "  Testing connectivity to: {{ endpoint.url }}"
+  # Test connectivity using wget (more reliable than curl in minimal containers)
+  kubectl exec -n telemetry "$VMAGENT_POD" -- wget -T 5 -q --spider "{{ endpoint.url }}" 2>/dev/null && \
+    echo "    ✓ Endpoint reachable" || \
+    echo "    WARNING: Endpoint unreachable - metrics may not be forwarded to {{ endpoint.url }}"
+  {% endfor %}
+else
+  echo "  WARNING: Could not find vmagent pod to check endpoint reachability"
+fi
+{% endif %}
+{% endif %}
+
 echo "===== Telemetry Stack Deployment Complete ====="

provision/roles/telemetry/templates/telemetry/victoria/csi-volume-exporter.yaml.j2

@@ -84,8 +84,9 @@ spec:
           command: ["/bin/sh", "-c"]
           args:
             - |
-              pip3 install prometheus_client==0.20.0 kubernetes==33.1.0 \
-                --find-links="{{ offline_pip_module_path }}/prometheus_client==0.20.0/" \
+              pip3 install \
+                "{{ offline_pip_module_path }}/prometheus_client==0.20.0/prometheus_client-0.20.0-py3-none-any.whl" \
+                "{{ offline_pip_module_path }}/kubernetes==33.1.0/kubernetes-33.1.0-py2.py3-none-any.whl" \
                 --trusted-host "{{ pulp_server_ip }}" \
                 --no-index || \
               pip3 install prometheus_client kubernetes
@@ -132,7 +133,6 @@ spec:
                   'Total PowerScale PVCs by phase',
                   ['phase'])
 
-
               # ── Health event metrics (from CSI external-health-monitor-controller) ──
               volume_condition_abnormal = Gauge(
                   'powerscale_volume_health_abnormal',

telemetry/roles/telemetry_disable/tasks/disable_powerscale_metrics.yml

@@ -0,0 +1,60 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Scale down OTEL Collector
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} otel-collector
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale down karavi-metrics-powerscale
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} karavi-metrics-powerscale
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale down csi-volume-exporter
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} csi-volume-exporter
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale down karavi-observability-cert-manager
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} karavi-observability-cert-manager
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale down karavi-observability-cert-manager-cainjector
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} karavi-observability-cert-manager-cainjector
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale down karavi-observability-cert-manager-webhook
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=0 -n {{ telemetry_namespace }} karavi-observability-cert-manager-webhook
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Display PowerScale metric workloads scaled down
+  ansible.builtin.debug:
+    msg: "{{ powerscale_metrics_scaled_down_msg }}"

telemetry/roles/telemetry_disable/tasks/main.yml

@@ -0,0 +1,37 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Prerequisite setup
+  tags: always
+  block:
+    - name: Fail if no tags provided
+      ansible.builtin.fail:
+        msg: "{{ tags_required_msg }}"
+      when: ansible_run_tags | default(['all']) | length == 1 and 'all' in ansible_run_tags | default(['all'])
+
+    - name: Load telemetry configuration
+      ansible.builtin.include_vars:
+        file: "{{ telemetry_config_file }}"
+
+    - name: Load HA configuration
+      ansible.builtin.include_tasks: "{{ playbook_dir }}/../common/tasks/common/load_ha_config.yml"
+
+- name: Disable PowerScale metrics
+  tags:
+    - powerscale
+  when: kube_vip is defined and kube_vip | length > 0
+  block:
+    - name: Disable PowerScale metrics
+      ansible.builtin.include_tasks: disable_powerscale_metrics.yml

telemetry/roles/telemetry_disable/vars/main.yml

@@ -0,0 +1,17 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+tags_required_msg: "No tags provided. Please run this playbook with the --tags flag. Example: ansible-playbook telemetry_disable.yml --tags powerscale"
+powerscale_metrics_scaled_down_msg: "PowerScale metrics workloads have been scaled down"

telemetry/roles/telemetry_enable/tasks/enable_powerscale_metrics.yml

@@ -0,0 +1,74 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Scale up karavi-observability-cert-manager to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} karavi-observability-cert-manager
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale up karavi-observability-cert-manager-cainjector to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} karavi-observability-cert-manager-cainjector
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale up karavi-observability-cert-manager-webhook to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} karavi-observability-cert-manager-webhook
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale up karavi-metrics-powerscale to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} karavi-metrics-powerscale
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale up csi-volume-exporter to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} csi-volume-exporter
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Wait for csi-volume-exporter to be ready
+  ansible.builtin.command:
+    kubectl wait deployment csi-volume-exporter -n {{ telemetry_namespace }} --for condition=available --timeout=5m
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Scale up OTEL Collector to replica count 1
+  ansible.builtin.command:
+    kubectl scale deployment --replicas=1 -n {{ telemetry_namespace }} otel-collector
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Wait for OTEL Collector to be ready
+  ansible.builtin.command:
+    kubectl wait deployment otel-collector -n {{ telemetry_namespace }} --for condition=available --timeout=5m
+  delegate_to: "{{ kube_vip }}"
+  failed_when: false
+  changed_when: false
+
+- name: Display PowerScale metric workloads scaled up
+  ansible.builtin.debug:
+    msg: "{{ powerscale_metrics_scaled_up_msg }}"

telemetry/roles/telemetry_enable/tasks/main.yml

@@ -0,0 +1,37 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+- name: Prerequisite setup
+  tags: always
+  block:
+    - name: Fail if no tags provided
+      ansible.builtin.fail:
+        msg: "{{ tags_required_msg }}"
+      when: ansible_run_tags | default(['all']) | length == 1 and 'all' in ansible_run_tags | default(['all'])
+
+    - name: Load telemetry configuration
+      ansible.builtin.include_vars:
+        file: "{{ telemetry_config_file }}"
+
+    - name: Load HA configuration
+      ansible.builtin.include_tasks: "{{ playbook_dir }}/../common/tasks/common/load_ha_config.yml"
+
+- name: Enable PowerScale metrics
+  tags:
+    - powerscale
+  when: kube_vip is defined and kube_vip | length > 0
+  block:
+    - name: Enable PowerScale metrics
+      ansible.builtin.include_tasks: enable_powerscale_metrics.yml

telemetry/roles/telemetry_enable/vars/main.yml

@@ -0,0 +1,17 @@
+# Copyright 2026 Dell Inc. or its subsidiaries. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+tags_required_msg: "No tags provided. Please run this playbook with the --tags flag. Example: ansible-playbook telemetry_enable.yml --tags powerscale"
+powerscale_metrics_scaled_up_msg: "PowerScale metrics workloads have been scaled up"