[SPARK-51953] Support Java 24

### What changes were proposed in this pull request?

This PR aims to support Java 24 as a preparation of Java 25 LTS.

- Update `lombok`, `jacoco`, and `mockito` libraries to support Java 24.
- Disable `pmd` and `spotbugs` plugins.
- Add `--enable-native-access=ALL-UNNAMED` to `DEFAULT_JVM_OPTS`.
- Add `SPARK_USER=spark` environment to the operator pod explicitly.
- Update `4.0.0-preview2` dependency to `4.0.1-SNAPSHOT` to validate with the upcoming `Apache Spark 4.0.0 RC5`.
- Use `azul/zulu-openjdk:24-jre` as the docker base image.
- Add `Java 24` to the CI.

### Why are the changes needed?

To support Java 24 as a preparation of Java 25 LTS.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Pass the CIs.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes apache#178 from dongjoon-hyun/SPARK-51953.

Authored-by: Dongjoon Hyun <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>

Author: dongjoon-hyun

.github/workflows/build_and_test.yml

@@ -33,7 +33,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ 'ubuntu-latest', 'ubuntu-24.04-arm' ]
-        java-version: [ 17, 21 ]
+        java-version: [ 17, 21, 24 ]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

build-tools/docker/Dockerfile

@@ -22,7 +22,7 @@ COPY . .
 
 RUN --mount=type=cache,target=/home/gradle/.gradle/caches gradle --no-daemon clean build -x check
 
-FROM azul/zulu-openjdk:21-jre
+FROM azul/zulu-openjdk:24-jre
 
 ARG APP_VERSION=0.1.0-SNAPSHOT
 ARG SPARK_UID=185

build-tools/helm/spark-kubernetes-operator/values.yaml

@@ -43,6 +43,8 @@ operatorDeployment:
     operatorContainer:
       jvmArgs: "-Dfile.encoding=UTF8"
       env:
+        - name: "SPARK_USER"
+          value: "spark"
       envFrom:
       volumeMounts: { }
       resources:

build.gradle

@@ -58,6 +58,9 @@ subprojects {
 
   repositories {
     mavenCentral()
+    maven {
+      url = "https://repository.apache.org/content/repositories/snapshots/"
+    }
   }
 
   apply plugin: 'checkstyle'
@@ -68,22 +71,24 @@ subprojects {
     showViolations = true
   }
 
-  apply plugin: 'pmd'
-  pmd {
-    ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml")
-    toolVersion = libs.versions.pmd.get()
-    consoleOutput = true
-    ignoreFailures = false
-  }
+  if (JavaVersion.current() < JavaVersion.VERSION_24) {
+    apply plugin: 'pmd'
+    pmd {
+      ruleSetFiles = files("$rootDir/config/pmd/ruleset.xml")
+      toolVersion = libs.versions.pmd.get()
+      consoleOutput = true
+      ignoreFailures = false
+    }
 
-  apply plugin: 'com.github.spotbugs'
-  spotbugs {
-    toolVersion = libs.versions.spotbugs.tool.get()
-    afterEvaluate {
-      reportsDir = file("${project.reporting.baseDir}/findbugs")
+    apply plugin: 'com.github.spotbugs'
+    spotbugs {
+      toolVersion = libs.versions.spotbugs.tool.get()
+      afterEvaluate {
+        reportsDir = file("${project.reporting.baseDir}/findbugs")
+      }
+      excludeFilter = file("$rootDir/config/spotbugs/spotbugs_exclude.xml")
+      ignoreFailures = false
     }
-    excludeFilter = file("$rootDir/config/spotbugs/spotbugs_exclude.xml")
-    ignoreFailures = false
   }
 
   apply plugin: 'jacoco'

gradle/libs.versions.toml

@@ -16,17 +16,17 @@
 # under the License.
 [versions]
 fabric8 = "7.1.0"
-lombok = "1.18.32"
+lombok = "1.18.38"
 operator-sdk = "4.9.0"
 okhttp = "4.12.0"
 dropwizard-metrics = "4.2.25"
-spark = "4.0.0-preview2"
+spark = "4.0.1-SNAPSHOT"
 log4j = "2.24.2"
 
 # Test
 junit = "5.10.2"
-jacoco = "0.8.12"
-mockito = "5.11.0"
+jacoco = "0.8.13"
+mockito = "5.17.0"
 powermock = "2.0.9"
 
 # Build Analysis

gradlew

@@ -210,7 +210,7 @@ fi
 
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m" "--enable-native-access=ALL-UNNAMED"'
 
 # Collect all arguments for the java command:
 #   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,