Merge branch 'main' into chore/tool-bumps-25.7.0

Author: NickLarsenNZ

.github/workflows/build_spark-connect-client.yaml

@@ -26,7 +26,7 @@ jobs:
     name: Reusable Workflow
     uses: ./.github/workflows/reusable_build_image.yaml
     secrets:
-      harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_SDP_GITHUB_ACTION_BUILD_SECRET }}
+      harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
       slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }}
     with:
       product-name: spark-connect-client

.github/workflows/mirror.yaml

@@ -1,7 +1,7 @@
 ---
 name: Mirror Container Image
 run-name: |
-  Mirror Container Image (attempt #${{ github.run_attempt }})
+  Mirror Container Image (${{ inputs.image-repository-uri }}:${{ inputs.image-index-manifest-tag }}, attempt #${{ github.run_attempt }})
 
 on:
   workflow_dispatch:

.github/workflows/reusable_build_image.yaml

@@ -67,7 +67,7 @@ jobs:
         uses: stackabletech/actions/publish-image@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
         with:
           image-registry-uri: oci.stackable.tech
-          image-registry-username: robot$sdp+github-action-build
+          image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build
           image-registry-password: ${{ secrets.harbor-robot-secret }}
           image-repository: ${{ inputs.registry-namespace }}/${{ inputs.product-name }}
           image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
@@ -93,7 +93,7 @@ jobs:
         uses: stackabletech/actions/publish-index-manifest@2d3d7ddad981ae09901d45a0f6bf30c2658b1b78 # 0.7.0
         with:
           image-registry-uri: oci.stackable.tech
-          image-registry-username: robot$sdp+github-action-build
+          image-registry-username: robot$${{ inputs.registry-namespace }}+github-action-build
           image-registry-password: ${{ secrets.harbor-robot-secret }}
           image-repository: ${{ inputs.registry-namespace }}/${{ inputs.product-name }}
           image-index-manifest-tag: ${{ matrix.versions }}-stackable${{ inputs.sdp-version }}

CHANGELOG.md

@@ -36,6 +36,8 @@ All notable changes to this project will be documented in this file.
 
 - ubi-rust-builder: Bump Rust toolchain to 1.85.0, cargo-cyclonedx to 0.5.7, and cargo-auditable to 0.6.6 ([#1050]).
 - spark-k8s: Include spark-connect jars. Replace OpenJDK with Temurin JDK. Cleanup. ([#1034]).
+- spark-k8s: Include spark-connect jars. Replace OpenJDK with Temurin JDK. Cleanup. ([#1034])
+- spark-connect-client: Image is now completely based on spark-k8s and includes JupyterLab and other demo dependencies ([#1071])
 - jmx_exporter: Bump products to use `1.2.0` ([#1090]).
 - kubectl: Bump products to use `1.33.0` ([#1090]).
 - yq: Bump products to use `4.45.2` ([#1090]).

spark-connect-client/Dockerfile

@@ -3,8 +3,6 @@
 # spark-builder: provides client libs for spark-connect
 FROM stackable/image/spark-k8s AS spark-builder
 
-FROM stackable/image/java-base
-
 ARG PRODUCT
 ARG PYTHON
 ARG RELEASE
@@ -18,42 +16,36 @@ LABEL name="Stackable Spark Connect Examples" \
     summary="Spark Connect Examples" \
     description="Spark Connect client libraries for Python and the JVM, including some examples."
 
+# Need root to install setuptools
+USER root
 
-ENV HOME=/stackable
-
-COPY spark-connect-client/stackable/spark-connect-examples /stackable/spark-connect-examples
-COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark/connect /stackable/spark/connect
+COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/spark-connect-examples /stackable/spark-connect-examples
+COPY --chown=${STACKABLE_USER_UID}:0 spark-connect-client/stackable/.jupyter /stackable/.jupyter
 
 RUN <<EOF
 microdnf update
 # python{version}-setuptools: needed to build the pyspark[connect] package
 microdnf install --nodocs \
-  "python${PYTHON}" \
-  "python${PYTHON}-pip" \
   "python${PYTHON}-setuptools"
 microdnf clean all
 rm -rf /var/cache/yum
 
-ln -s /usr/bin/python${PYTHON} /usr/bin/python
-ln -s /usr/bin/pip-${PYTHON} /usr/bin/pip
-
-# Install python libraries for the spark connect client
-# shellcheck disable=SC2102
-pip install --no-cache-dir pyspark[connect]==${PRODUCT}
-
 # All files and folders owned by root group to support running as arbitrary users.
 # This is best practice as all container users will belong to the root group (0).
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 chmod -R g=u /stackable
 EOF
 
-# ----------------------------------------
-# Attention: We are changing the group of all files in /stackable directly above
-# If you do any file based actions (copying / creating etc.) below this comment you
-# absolutely need to make sure that the correct permissions are applied!
-# chown ${STACKABLE_USER_UID}:0
-# ----------------------------------------
-
 USER ${STACKABLE_USER_UID}
 
+# Install python packages.
+# Packages are intentionally installed in "user mode" to reduce the container attack surface.
+# - pyspark[connect] = spark connect client libs
+# - jupyterlab = notebook client used in demos
+RUN pip install --no-cache-dir --user \
+  "pyspark[connect]==${PRODUCT}" \
+  "jupyterlab==4.4.1" \
+  "scikit-learn==1.3.1" \
+  "matplotlib==3.10.1"
+
 WORKDIR /stackable/spark-connect-examples/python