diff --git a/CHANGELOG.md b/CHANGELOG.md index 2940f6928..0b8394caf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -123,6 +123,7 @@ All notable changes to this project will be documented in this file. - nifi: Remove `2.2.0` ([#1114]). - kafka: Remove `3.7.1` and `3.8.0` ([#1117]). - spark-connect-client: Remove `3.5.5` ([#1142]). +- spark-k8s: Remove the JMX exporter jar ([#1157]). [nifi-iceberg-bundle]: https://github.com/stackabletech/nifi-iceberg-bundle [#1025]: https://github.com/stackabletech/docker-images/pull/1025 @@ -184,6 +185,7 @@ All notable changes to this project will be documented in this file. [#1151]: https://github.com/stackabletech/docker-images/pull/1151 [#1152]: https://github.com/stackabletech/docker-images/pull/1152 [#1156]: https://github.com/stackabletech/docker-images/pull/1156 +[#1157]: https://github.com/stackabletech/docker-images/pull/1157 [#1163]: https://github.com/stackabletech/docker-images/pull/1163 [#1165]: https://github.com/stackabletech/docker-images/pull/1165 diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 55b719131..628acf5cc 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -131,7 +131,6 @@ ARG AZURE_KEYVAULT_CORE ARG JACKSON_DATAFORMAT_XML ARG STAX2_API ARG WOODSTOX_CORE -ARG JMX_EXPORTER ARG TARGETARCH ARG TINI ARG STACKABLE_USER_UID @@ -206,8 +205,6 @@ RUN cp /stackable/spark-${PRODUCT}/connector/connect/server/target/spark-connect && cp /stackable/spark-${PRODUCT}/connector/connect/common/target/spark-connect-common_*-${PRODUCT}.jar . \ && cp /stackable/spark-${PRODUCT}/connector/connect/client/jvm/target/spark-connect-client-jvm_2.12-${PRODUCT}.jar . -COPY spark-k8s/stackable/jmx /stackable/jmx - WORKDIR /stackable/spark-${PRODUCT}/dist/extra-jars RUN <<>Value" - name: spark_master_$1 - - # These come from the worker - # Example: worker.coresFree - - pattern: "metrics<>Value" - name: spark_worker_$1 - - # These come from the application driver - # Example: app-20160809000059-0000.driver.DAGScheduler.stage.failedStages - - pattern: "metrics<>Value" - name: spark_driver_$2_$3 - type: GAUGE - labels: - app_id: "$1" - - # These come from the application driver - # Emulate timers for DAGScheduler like messagePRocessingTime - - pattern: "metrics<>Count" - name: spark_driver_DAGScheduler_$2_total - type: COUNTER - labels: - app_id: "$1" - - - pattern: "metrics<>Count" - name: spark_driver_HiveExternalCatalog_$2_total - type: COUNTER - labels: - app_id: "$1" - - # These come from the application driver - # Emulate histograms for CodeGenerator - - pattern: "metrics<>Count" - name: spark_driver_CodeGenerator_$2_total - type: COUNTER - labels: - app_id: "$1" - - # These come from the application driver - # Emulate timer (keep only count attribute) plus counters for LiveListenerBus - - pattern: "metrics<>Count" - name: spark_driver_LiveListenerBus_$2_total - type: COUNTER - labels: - app_id: "$1" - - # Get Gauge type metrics for LiveListenerBus - - pattern: "metrics<>Value" - name: spark_driver_LiveListenerBus_$2 - type: GAUGE - labels: - app_id: "$1" - - # These come from the application driver if it's a streaming application - # Example: app-20160809000059-0000.driver.com.example.ClassName.StreamingMetrics.streaming.lastCompletedBatch_schedulingDelay - - pattern: "metrics<>Value" - name: spark_driver_streaming_$3 - labels: - app_id: "$1" - app_name: "$2" - - # These come from the application driver if it's a structured streaming application - # Example: app-20160809000059-0000.driver.spark.streaming.QueryName.inputRate-total - - pattern: "metrics<>Value" - name: spark_driver_structured_streaming_$3 - labels: - app_id: "$1" - query_name: "$2" - - # These come from the application executors - # Examples: - # app-20160809000059-0000.0.executor.threadpool.activeTasks (value) - # app-20160809000059-0000.0.executor.JvmGCtime (counter) - - # filesystem metrics are declared as gauge metrics, but are actually counters - - pattern: "metrics<>Value" - name: spark_executor_filesystem_$3_total - type: COUNTER - labels: - app_id: "$1" - executor_id: "$2" - - - pattern: "metrics<>Value" - name: spark_executor_$3 - type: GAUGE - labels: - app_id: "$1" - executor_id: "$2" - - - pattern: "metrics<>Count" - name: spark_executor_$3_total - type: COUNTER - labels: - app_id: "$1" - executor_id: "$2" - - - pattern: "metrics<>Value" - name: spark_executor_$3 - type: GAUGE - labels: - app_id: "$1" - executor_id: "$2" - - # These come from the application executors - # Example: app-20160809000059-0000.0.jvm.threadpool.activeTasks - - pattern: "metrics<>Value" - name: spark_executor_$3_$4 - type: GAUGE - labels: - app_id: "$1" - executor_id: "$2" - - - pattern: "metrics<>Count" - name: spark_executor_HiveExternalCatalog_$3_total - type: COUNTER - labels: - app_id: "$1" - executor_id: "$2" - - # These come from the application driver - # Emulate histograms for CodeGenerator - - pattern: "metrics<>Count" - name: spark_executor_CodeGenerator_$3_total - type: COUNTER - labels: - app_id: "$1" - executor_id: "$2" diff --git a/spark-k8s/versions.py b/spark-k8s/versions.py index 105429502..bd1ec5a4f 100644 --- a/spark-k8s/versions.py +++ b/spark-k8s/versions.py @@ -13,7 +13,6 @@ "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "vector": "0.47.0", - "jmx_exporter": "1.3.0", "tini": "0.19.0", "hbase_connector": "1.0.1", }, @@ -31,7 +30,6 @@ "stax2_api": "4.2.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "woodstox_core": "6.5.1", # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 "vector": "0.47.0", - "jmx_exporter": "1.3.0", "tini": "0.19.0", "hbase_connector": "1.0.1", },