fix: add missing Spark import/export support for metrics

natoscott · natoscott · commit 8436ff0aec22 · 2025-04-17T18:17:04.000+10:00
Resolves Red Hat issue RHEL-17564

Signed-off-by: Nathan Scott &lt;nathans@redhat.com&gt;
diff --git a/.ostree/packages-runtime-CentOS-10.txt b/.ostree/packages-runtime-CentOS-10.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-runtime-CentOS-9.txt b/.ostree/packages-runtime-CentOS-9.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-runtime-Fedora.txt b/.ostree/packages-runtime-Fedora.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-runtime-RedHat-10.txt b/.ostree/packages-runtime-RedHat-10.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-runtime-RedHat-8.txt b/.ostree/packages-runtime-RedHat-8.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-runtime-RedHat-9.txt b/.ostree/packages-runtime-RedHat-9.txt
@@ -2,6 +2,7 @@ bpftrace
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-mssql
 pcp-system-tools
diff --git a/.ostree/packages-testing-CentOS-10.txt b/.ostree/packages-testing-CentOS-10.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/.ostree/packages-testing-CentOS-9.txt b/.ostree/packages-testing-CentOS-9.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/.ostree/packages-testing-Fedora.txt b/.ostree/packages-testing-Fedora.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/.ostree/packages-testing-RedHat-10.txt b/.ostree/packages-testing-RedHat-10.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/.ostree/packages-testing-RedHat-8.txt b/.ostree/packages-testing-RedHat-8.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/.ostree/packages-testing-RedHat-9.txt b/.ostree/packages-testing-RedHat-9.txt
@@ -3,6 +3,7 @@ cyrus-sasl-scram
 grafana
 grafana-pcp
 pcp-export-pcp2elasticsearch
+pcp-export-pcp2spark
 pcp-pmda-bpftrace
 pcp-pmda-elasticsearch
 pcp-pmda-mssql
diff --git a/README.md b/README.md
@@ -85,6 +85,14 @@ Boolean flag allowing metric values to be exported into Elasticsearch.
 
 Boolean flag allowing metrics from Elasticsearch to be made available.
 
+### metrics_into_spark: false
+
+Boolean flag allowing metric values to be exported into Spark.
+
+### metrics_from_spark: false
+
+Boolean flag allowing metrics from Spark to be made available.
+
 ### metrics_from_postfix: false
 
 Boolean flag allowing metrics from Postfix to be made available.
diff --git a/defaults/main.yml b/defaults/main.yml
@@ -25,6 +25,12 @@ metrics_into_elasticsearch: false
 # Make metrics available from an Elasticsearch instance
 metrics_from_elasticsearch: false
 
+# Export metrics to an Apache Spark data analysis setup
+metrics_into_spark: false
+
+# Make metrics available from an Apache Spark instance
+metrics_from_spark: false
+
 # Make metrics available from a SQL Server database
 metrics_from_mssql: false
 
diff --git a/examples/pcp_spark.yml b/examples/pcp_spark.yml
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: MIT
+---
+- name: Set up Spark for use with PCP
+  hosts: monitoring
+
+  roles:
+    - role: linux-system-roles.metrics
+      vars:
+        metrics_provider: pcp
+        metrics_from_spark: true
+        metrics_into_spark: true
diff --git a/tasks/main.yml b/tasks/main.yml
@@ -49,6 +49,18 @@
     metrics_into_elasticsearch | d(false) | bool
 # yamllint enable rule:line-length
 
+- name: Configure Spark metrics
+  vars:
+    spark_metrics_agent: "{{ metrics_from_spark | d(false) | bool }}"
+    spark_export_metrics: "{{ metrics_into_spark | d(false) | bool }}"
+    spark_metrics_provider: "{{ metrics_provider }}"
+  include_role:
+    # noqa role-name[path]
+    name: "{{ role_path }}/roles/spark"
+  when: >
+    metrics_from_spark | d(false) | bool or
+    metrics_into_spark | d(false) | bool
+
 - name: Configure SQL Server metrics.
   vars:
     mssql_metrics_provider: "{{ metrics_provider }}"
diff --git a/tests/check_from_spark.yml b/tests/check_from_spark.yml
@@ -0,0 +1,9 @@
+# SPDX-License-Identifier: MIT
+---
+- name: Check if OpenMetrics PMDA has Spark metrics registered
+  command: pmprobe -I openmetrics.control.status
+  register: status
+  until: status.stdout.find("spark") != -1
+  retries: 10
+  delay: 1
+  changed_when: false
diff --git a/tests/check_into_spark.yml b/tests/check_into_spark.yml
@@ -0,0 +1,11 @@
+# SPDX-License-Identifier: MIT
+---
+- name: Check if export to Spark is installed
+  command: test -x /usr/bin/pcp2spark
+  changed_when: false
+
+- name: Check the ansible_managed header in pcp2spark.service
+  vars:
+    __test_config_path: >-
+      {{ __spark_service_path }}/pcp2spark.service
+  include_tasks: check_header.yml
diff --git a/tests/tests_verify_from_spark.yml b/tests/tests_verify_from_spark.yml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: MIT
+---
+- name: Test import from Spark
+  hosts: all
+  vars:
+    metrics_from_spark: true
+
+  pre_tasks:
+    - name: Stop test
+      meta: end_host
+      when: (ansible_distribution in ['RedHat', 'CentOS'] and
+             ansible_distribution_major_version | int < 7) or
+             ansible_distribution not in ['Fedora', 'RedHat', 'CentOS']
+
+    - name: Save state of services
+      import_tasks: get_services_state.yml
+
+  tasks:
+    - name: Run test
+      block:
+        - name: Run the metrics role to configure Spark
+          include_role:
+            name: linux-system-roles.metrics
+            public: true
+
+        - name: Check if import from Spark works
+          include_tasks: check_from_spark.yml
+
+        - name: Flush handlers
+          meta: flush_handlers
+
+      rescue:
+        - name: Handle failure case
+          include_tasks: handle_test_failure.yml
+
+      always:
+        - name: Restore state of services
+          import_tasks: restore_services_state.yml
diff --git a/tests/tests_verify_into_spark.yml b/tests/tests_verify_into_spark.yml
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: MIT
+---
+- name: Test import to Spark
+  hosts: all
+  vars:
+    metrics_into_spark: true
+
+  pre_tasks:
+    - name: Stop test
+      meta: end_host
+      when: (ansible_distribution in ['RedHat'] and
+             (ansible_facts['distribution_version'] is version('8.4', '<'))) or
+             ansible_distribution not in ['Fedora', 'RedHat']
+
+    - name: Save state of services
+      import_tasks: get_services_state.yml
+
+  tasks:
+    - name: Run test
+      block:
+        - name: Run the role
+          include_role:
+            name: linux-system-roles.metrics
+            public: true
+
+        - name: Check if import to Spark works
+          include_tasks: check_into_spark.yml
+
+        - name: Flush handlers
+          meta: flush_handlers
+
+      rescue:
+        - name: Handle failure case
+          include_tasks: handle_test_failure.yml
+
+      always:
+        - name: Restore state of services
+          import_tasks: restore_services_state.yml
