diff --git a/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml b/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml similarity index 89% rename from .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml rename to .github/workflows/dev_spark-k8s-with-scikit-learn.yaml index f9b497e5..635cc33f 100644 --- a/.github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml +++ b/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml @@ -1,9 +1,9 @@ --- -name: Build and publish pyspark-k8s-with-scikit-learn +name: Build and publish spark-k8s-with-scikit-learn env: - IMAGE_NAME: pyspark-k8s-with-scikit-learn - IMAGE_VERSION: 3.4.0-stackable0.0.0-dev + IMAGE_NAME: spark-k8s-with-scikit-learn + IMAGE_VERSION: 3.5.0-stackable24.3.0 REGISTRY_PATH: stackable DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile" @@ -15,7 +15,7 @@ on: paths: - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt - - .github/workflows/dev_pyspark-k8s-with-scikit-learn.yaml + - .github/workflows/dev_spark-k8s-with-scikit-learn.yaml jobs: build: @@ -27,7 +27,9 @@ jobs: matrix: runner: - {name: "ubuntu-latest", arch: "amd64"} - - {name: "ubicloud-standard-8-arm", arch: "arm64"} + # TODO: the image 3.5.0-stackable24.3.0 does not have an arm64 build. + # Re-activate the arm runner when the image is updated to one that does. + #- {name: "ubicloud-standard-8-arm", arch: "arm64"} steps: - name: Checkout Repository uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile index 445be34e..232b7c8b 100644 --- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile +++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.stackable.tech/stackable/spark-k8s:3.5.1-stackable24.7.0 +FROM docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable24.3.0 COPY demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt . diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt index 27812422..899fea70 100644 --- a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt +++ b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt @@ -1,2 +1,2 @@ -scikit-learn==1.1.3 -pandas==1.5.1 \ No newline at end of file +scikit-learn==1.3.1 +pandas==2.0.3 diff --git a/stacks/_templates/jupyterhub.yaml b/stacks/_templates/jupyterhub.yaml index 81481c9a..793929a7 100644 --- a/stacks/_templates/jupyterhub.yaml +++ b/stacks/_templates/jupyterhub.yaml @@ -1,3 +1,4 @@ +--- releaseName: jupyterhub name: jupyterhub repo: @@ -11,7 +12,7 @@ options: allowed_users: - admin DummyAuthenticator: - password: {{ jupyterHubAdminPassword }} + password: {{jupyterHubAdminPassword}} JupyterHub: authenticator_class: dummy labels: @@ -39,7 +40,7 @@ options: # Inspect the Dockerfile at: # https://github.com/jupyter/docker-stacks/tree/HEAD/datascience-notebook/Dockerfile name: jupyter/pyspark-notebook - tag: python-3.9 + tag: python-3.11 serviceAccountName: spark networkPolicy: enabled: false diff --git a/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb b/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb index 19dcd3f4..7e0cc949 100644 --- a/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb +++ b/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb @@ -34,7 +34,7 @@ "spark = (SparkSession\n", " .builder\n", " .master(f'k8s://https://{os.environ[\"KUBERNETES_SERVICE_HOST\"]}:{os.environ[\"KUBERNETES_SERVICE_PORT\"]}')\n", - " .config(\"spark.kubernetes.container.image\", \"docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable23.4\")\n", + " .config(\"spark.kubernetes.container.image\", \"docker.stackable.tech/demos/spark-k8s-with-scikit-learn:3.5.0-stackable24.3.0\")\n", " .config(\"spark.driver.port\", \"2222\")\n", " .config(\"spark.driver.blockManager.port\", \"7777\")\n", " .config(\"spark.driver.host\", \"driver-service.default.svc.cluster.local\")\n",