chore: ensure metrics are correctly exposed (#619)

* fix examples

* adapted changelog

* add metrics service for spark connect

* fix comment

* precommit

* add metrics test for the history server

* add metrics test for connect server

* add test for connect executor metrics

* update docs

---------

Co-authored-by: Razvan-Daniel Mihai <[email protected]>

Author: maltesander

CHANGELOG.md

@@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file.
 - Add experimental support for Spark 4 ([#589])
 - Helm: Allow Pod `priorityClassName` to be configured ([#608]).
 - Support for Spark 3.5.7 ([#610]).
+- Add metrics service with `prometheus.io/path|port|scheme` annotations for spark history server ([#619]).
+- Add metrics service with `prometheus.io/path|port|scheme` annotations for spark connect ([#619]).
 
 ### Fixed
 
@@ -35,6 +37,7 @@ All notable changes to this project will be documented in this file.
 [#610]: https://github.com/stackabletech/spark-k8s-operator/pull/610
 [#611]: https://github.com/stackabletech/spark-k8s-operator/pull/611
 [#617]: https://github.com/stackabletech/spark-k8s-operator/pull/617
+[#619]: https://github.com/stackabletech/spark-k8s-operator/pull/619
 
 ## [25.7.0] - 2025-07-23
 

apps/ny_tlc_report.py

@@ -9,6 +9,7 @@
             need to be submitted along with the job.
 --output    Path to write the report as a CSV file.
 """
+
 import argparse
 
 from argparse import Namespace

docs/modules/spark-k8s/pages/usage-guide/history-server.adoc

@@ -157,10 +157,9 @@ By setting up port forwarding on 18080 the UI can be opened by pointing your bro
 
 image::history-server-ui.png[History Server Console]
 
-== Metrics
+== Monitoring
 
-[NOTE]
-====
-Starting with version 25.7, the built-in Prometheus servlet is enabled in addition to the existing JMX exporter.
-The JMX exporter is still available but it is deprecated and will be removed in a future release.
-====
+The operator creates a Kubernetes service dedicated specifically to collect metrics for Spark History instances with Prometheus.
+These metrics are exported via the JMX exporter as the history server doesn't support the built in Spark prometheus servlet.
+The service name follows the convention `<stacklet name>-history-metrics`.
+Metrics can be scraped at the endpoint `<service name>:18081/metrics`.

docs/modules/spark-k8s/pages/usage-guide/operations/applications.adoc

@@ -9,7 +9,7 @@ As the operator creates the necessary resources, the status of the application t
 NOTE: The operator never reconciles an application once it has been created.
 To resubmit an application, a new SparkApplication resource must be created.
 
-== Metrics
+== Monitoring
 
 [NOTE]
 ====

docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc

@@ -26,12 +26,14 @@ include::example$example-spark-connect.yaml[]
 <7> Customize the driver properties in the `server` role. The number of cores here is not related to Kubernetes cores!
 <8> Customize `spark.executor.\*` and `spark.kubernetes.executor.*` in the `executor` role.
 
-== Metrics
+== Monitoring
 
-The server pod exposes Prometheus metrics at the following endpoints:
+The operator creates a Kubernetes service dedicated specifically to collect metrics for Spark Connect instances with Prometheus.
+The service name follows the convention `<stacklet name>-server-metrics`.
+This service exposes Prometheus metrics at the following endpoints:
 
-* `/metrics/prometheus` for driver instances.
-* `/metrics/executors/prometheus` for executor instances.
+* `<service name>:4040/metrics/prometheus` for driver instances.
+* `<service name>:4040/metrics/executors/prometheus` for executor instances.
 
 To customize the metrics configuration use the `spec.server.configOverrides' like this:
 
@@ -47,8 +49,8 @@ The example above adds a new endpoint for application metrics.
 
 == Spark History Server
 
-Unforunately integration with the Spark History Server is not supported yet.
-The connect server seems to ignore the `spark.eventLog` properties while also prohibiting clients to set them programatically.
+Unfortunately integration with the Spark History Server is not supported yet.
+The connect server seems to ignore the `spark.eventLog` properties while also prohibiting clients to set them programmatically.
 
 == Notable Omissions
 

examples/README-examples.md

@@ -50,7 +50,7 @@ Several resources are needed in this store. These can be loaded like this:
 
 ```text
 kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/'
-kubectl cp examples/ny-tlc-report-1.1.0-3.5.7.jar  minio-mc-0:/tmp
+kubectl cp tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.7.jar  minio-mc-0:/tmp
 kubectl cp apps/ny_tlc_report.py  minio-mc-0:/tmp
 kubectl cp examples/yellow_tripdata_2021-07.csv  minio-mc-0:/tmp
 kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.7.jar test-minio/my-bucket

kind/assert-pvc-jars.yaml

@@ -13,7 +13,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: assert-pvc-jars
-          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"

kind/kind-pvc.yaml

@@ -24,7 +24,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: aws-deps
-          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"

kind/minio.yaml

@@ -29,6 +29,6 @@ spec:
     spec:
       containers:
         - name: minio-mc
-          image: bitnamilegacy/minio:2022-debian-10
+          image: docker.io/bitnamilegacy/minio:2024-debian-12
           stdin: true
           tty: true

rust/operator-binary/src/connect/controller.rs

@@ -21,7 +21,7 @@ use strum::{EnumDiscriminants, IntoStaticStr};
 use super::crd::{CONNECT_APP_NAME, CONNECT_CONTROLLER_NAME, v1alpha1};
 use crate::{
     Ctx,
-    connect::{common, crd::SparkConnectServerStatus, executor, server},
+    connect::{common, crd::SparkConnectServerStatus, executor, server, service},
     crd::constants::{OPERATOR_NAME, SPARK_IMAGE_BASE_NAME},
 };
 
@@ -47,7 +47,7 @@ pub enum Error {
     ServerProperties { source: server::Error },
 
     #[snafu(display("failed to build spark connect service"))]
-    BuildService { source: server::Error },
+    BuildService { source: service::Error },
 
     #[snafu(display("failed to build spark connect executor config map for {name}"))]
     BuildExecutorConfigMap {
@@ -67,9 +67,6 @@ pub enum Error {
         name: String,
     },
 
-    #[snafu(display("spark connect object has no namespace"))]
-    ObjectHasNoNamespace,
-
     #[snafu(display("failed to update the connect server stateful set"))]
     ApplyStatefulSet {
         source: stackable_operator::cluster_resources::Error,
@@ -208,12 +205,22 @@ pub async fn reconcile(
         .context(ApplyRoleBindingSnafu)?;
 
     // Headless service used by executors connect back to the driver
-    let service =
-        server::build_internal_service(scs, &resolved_product_image.app_version_label_value)
+    let headless_service =
+        service::build_headless_service(scs, &resolved_product_image.app_version_label_value)
             .context(BuildServiceSnafu)?;
 
-    let applied_internal_service = cluster_resources
-        .add(client, service.clone())
+    let applied_headless_service = cluster_resources
+        .add(client, headless_service.clone())
+        .await
+        .context(ApplyServiceSnafu)?;
+
+    // Metrics service used for scraping
+    let metrics_service =
+        service::build_metrics_service(scs, &resolved_product_image.app_version_label_value)
+            .context(BuildServiceSnafu)?;
+
+    cluster_resources
+        .add(client, metrics_service.clone())
         .await
         .context(ApplyServiceSnafu)?;
 
@@ -224,7 +231,7 @@ pub async fn reconcile(
         server::server_properties(
             scs,
             &server_config,
-            &applied_internal_service,
+            &applied_headless_service,
             &service_account,
             &resolved_product_image,
         )