fix(spark): JAVA_HOME needs to point to OpenJDK

Author: razvan

spark-k8s/Dockerfile

@@ -306,6 +306,7 @@ RUN <<EOF
 microdnf update
 # procps: required for spark startup scripts
 # java-*-openjdk-devel: This is needed by the Spark UI to display process information using jps and jmap
+#                       The spark-connect server also needs it to compile auto-generated classes on the fly.
 #                       Copying just the binaries from the builder stage failed.
 microdnf install \
   gzip \
@@ -318,6 +319,10 @@ microdnf install \
 microdnf clean all
 rm -rf /var/cache/yum
 
+# The base image (java-base) defines this to point to a JRE installation.
+# Spark Connect requires it to point to a JDK installation.
+ENV JAVA_HOME="/usr/lib/jvm/java-${JAVA_VERSION}-openjdk"
+
 ln -s /usr/bin/python${PYTHON} /usr/bin/python
 ln -s /usr/bin/pip-${PYTHON} /usr/bin/pip