Remove Python 3.8 to fix build issues

.github/workflows/build.yml

@@ -7,28 +7,29 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     env:
-      ASYNC_TEST_TIMEOUT: 30
+      ASYNC_TEST_TIMEOUT: 60
+      KERNEL_LAUNCH_TIMEOUT: 120
       CONDA_HOME: /usr/share/miniconda
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           clean: true
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+#        with:
+#          python_version: "${{ matrix.python-version }}"
       - name: Display dependency info
         run: |
           python --version
           pip --version
           conda --version
-      - name: Add SBT launcher
-        run: |
-          mkdir -p $HOME/.sbt/launchers/1.3.12
-          curl -L -o $HOME/.sbt/launchers/1.3.12/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.3.12/sbt-launch.jar
+      - name: Setup sbt launcher
+        uses: sbt/setup-sbt@v1
       - name: Install Python dependencies
         run: |
           pip install ".[test]"
@@ -50,15 +51,26 @@ jobs:
           command: |
             make test
       - name: Run integration tests
-        run: |
-          make itest-yarn
+        uses: nick-invision/[email protected]
+        with:
+          timeout_minutes: 3
+          max_attempts: 1
+          command: |
+            export PREP_TIMEOUT=180
+            # Run integration tests with debug output
+            make itest-yarn-debug
       - name: Collect logs
         if: success() || failure()
         run: |
           python --version
           pip --version
           pip list
+          echo "==== Docker Container Logs ===="
           docker logs itest-yarn
+          echo "==== Docker Container Status ===="
+          docker ps -a
+          echo "==== Enterprise Gateway Log ===="
+          docker exec -it itest-yarn cat /usr/local/share/jupyter/enterprise-gateway.log || true
       - name: Run linters
         run: |
           make lint
@@ -70,7 +82,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - name: Install Python dependencies
         run: |
           pip install ".[test]"
@@ -85,6 +100,8 @@ jobs:
         uses: actions/checkout@v4
       - name: Base Setup
         uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - name: Build Docs
         run: make docs
 
@@ -94,10 +111,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.8"
-      - uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1
+          python_version: "3.9"
+      - name: Install dependencies with minimum versions
+        run: |
+          pip install ".[test]"
       - name: Run the unit tests
         run: |
           pytest -vv -W default || pytest -vv -W default --lf
@@ -109,6 +132,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - uses: jupyterlab/maintainer-tools/.github/actions/make-sdist@v1
 
   test_sdist:
@@ -118,6 +143,8 @@ jobs:
     timeout-minutes: 20
     steps:
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - uses: jupyterlab/maintainer-tools/.github/actions/test-sdist@v1
 
   python_tests_check: # This job does nothing and is only used for the branch protection

.gitignore

@@ -43,6 +43,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .pytest_cache/
+.bsp
 
 # Translations
 *.mo
@@ -60,10 +61,6 @@ target/
 .DS_Store
 .ipynb_checkpoints/
 
-# PyCharm
-.idea/
-*.iml
-
 # Build-related
 .image-*
 
@@ -74,11 +71,11 @@ _site/
 # Debug-related
 .kube/
 
+# PyCharm
+.idea/
+*.iml
+
 # vscode ide stuff
 *.code-workspace
 .history/
 .vscode/
-
-# jetbrains ide stuff
-*.iml
-.idea/

Makefile

@@ -15,7 +15,7 @@ MULTIARCH_BUILD?=
 TARGET_ARCH?=undefined
 
 VERSION?=3.3.0.dev0
-SPARK_VERSION?=3.2.1
+SPARK_VERSION?=3.2.4
 
 ifeq (dev, $(findstring dev, $(VERSION)))
     TAG:=dev
@@ -224,7 +224,7 @@ PREP_TIMEOUT?=60
 itest-yarn-prep:
 	@-docker rm -f itest-yarn >> /dev/null
 	@echo "Starting enterprise-gateway container (run \`docker logs itest-yarn\` to see container log)..."
-	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:$(TAG) --gateway
+	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:dev --gateway
 	@(r="1"; attempts=0; while [ "$$r" == "1" -a $$attempts -lt $(PREP_TIMEOUT) ]; do echo "Waiting for enterprise-gateway to start..."; sleep 2; ((attempts++)); docker logs itest-yarn |grep --regexp "Jupyter Enterprise Gateway .* is available at http"; r=$$?; done; if [ $$attempts -ge $(PREP_TIMEOUT) ]; then echo "Wait for startup timed out!"; exit 1; fi;)
 
 

docs/environment.yml

@@ -5,6 +5,6 @@ channels:
   - free
 dependencies:
   - pip
-  - python=3.8
+  - python=3.9
   - pip:
       - -r doc-requirements.txt

enterprise_gateway/itests/test_r_kernel.py

@@ -34,7 +34,7 @@ def test_restart(self):
         self.assertTrue(self.kernel.restart())
 
         error_result, has_error = self.kernel.execute("y = x + 1")
-        self.assertRegex(error_result, "Error in eval")
+        self.assertRegex(error_result, r"Error: object '(\w+)' not found")
         self.assertEqual(has_error, True)
 
     def test_interrupt(self):

etc/Makefile

@@ -87,7 +87,8 @@ kernel-launchers/scala/lib: $(TOREE_LAUNCHER_FILES)
 	-rm -rf kernel-launchers/scala/lib
 	mkdir -p kernel-launchers/scala/lib
 	@(cd kernel-launchers/scala/toree-launcher; sbt -Dversion=$(VERSION) -Dspark_version=$(SPARK_VERSION) package; cp target/scala-2.12/*.jar ../lib)
-	curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
+	# curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
+	mvn org.apache.maven.plugins:maven-dependency-plugin:3.3.0:get -DrepoUrl=https://repository.apache.org/content/repositories/releases/ -Dartifact=org.apache.toree:toree-assembly:0.5.0-incubating -Ddest=./etc/kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
 
 KERNEL_IMAGE_FILE:=../dist/jupyter_enterprise_gateway_kernel_image_files-$(VERSION).tar.gz
 kernel_image_files: ../build/kernel_image_files

etc/docker/demo-base/Dockerfile

@@ -9,23 +9,23 @@ ARG NB_GID="100"
 
 USER root
 
-ENV HADOOP_HOME=/usr/hdp/current/hadoop \
-    ANACONDA_HOME=/opt/conda
-
-ENV SHELL=/bin/bash \
-    NB_USER=$NB_USER \
-    NB_UID=$NB_UID \
-    NB_GID=$NB_GID \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8 \
-    JAVA_HOME=/usr/lib/jvm/java \
-    SPARK_HOME=/usr/hdp/current/spark2-client \
-    PYSPARK_PYTHON=$ANACONDA_HOME/bin/python \
-    HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-
-ENV HOME=/home/$NB_USER \
-    PATH=$JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH
+ENV HADOOP_HOME /usr/hdp/current/hadoop
+ENV ANACONDA_HOME /opt/conda
+
+ENV SHELL /bin/bash \
+    NB_USER $NB_USER \
+    NB_UID $NB_UID \
+    NB_GID $NB_GID \
+    LC_ALL en_US.UTF-8 \
+    LANG en_US.UTF-8 \
+    LANGUAGE en_US.UTF-8 \
+    JAVA_HOME /usr/lib/jvm/java \
+    SPARK_HOME /usr/hdp/current/spark2-client \
+    PYSPARK_PYTHON $ANACONDA_HOME/bin/python \
+    HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
+
+ENV HOME /home/$NB_USER
+ENV PATH $JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH
 
 ENV SPARK_VER $SPARK_VERSION
 ENV HADOOP_VER 3.3.1
@@ -56,7 +56,7 @@ RUN dpkg --purge --force-depends ca-certificates-java \
     && apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security main' \
     && apt-add-repository 'deb http://deb.debian.org/debian/ sid main' \
     && apt-get update && apt-get install -yq --no-install-recommends  \
-    openjdk-8-jre-headless \
+    openjdk-8-jdk-headless \
     ca-certificates-java \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
@@ -71,14 +71,14 @@ ADD fix-permissions /usr/local/bin/fix-permissions
 # and make sure these dirs are writable by the `users` group.
 RUN groupadd wheel -g 11 && \
     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
-    useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    useradd -m -s /bin/bash -N -u "$NB_UID" "$NB_USER" && \
     mkdir -p /usr/hdp/current && \
     mkdir -p /usr/local/share/jupyter && \
-    chown $NB_USER:$NB_GID $ANACONDA_HOME && \
+    chown "$NB_USER":"$NB_GID" "$ANACONDA_HOME" && \
     chmod g+w /etc/passwd && \
     chmod +x /usr/local/bin/fix-permissions && \
-    fix-permissions $HOME && \
-    fix-permissions $ANACONDA_HOME && \
+    fix-permissions "$HOME" && \
+    fix-permissions "$ANACONDA_HOME" && \
     fix-permissions /usr/hdp/current && \
     fix-permissions /usr/local/share/jupyter
 
@@ -90,8 +90,8 @@ RUN useradd -m -s /bin/bash -N -u 1111 elyra && \
 USER $NB_UID
 
 # Setup work directory for backward-compatibility
-RUN mkdir /home/$NB_USER/work && \
-    fix-permissions /home/$NB_USER
+RUN mkdir "/home/$NB_USER/work" && \
+    fix-permissions "/home/$NB_USER"
 
 # DOWNLOAD HADOOP AND SPARK
 RUN curl -sL https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VER/hadoop-$HADOOP_VER.tar.gz | tar -xz -C /usr/hdp/current
@@ -106,10 +106,11 @@ RUN conda install mamba -n base -c conda-forge && \
     'jupyter' \
     'r-devtools' \
     'r-stringr' \
-    'r-argparse' && \
+    'r-argparse' \
+    'python=3.10' && \
     mamba clean -y --all &&\
-    fix-permissions $ANACONDA_HOME && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "$ANACONDA_HOME" && \
+    fix-permissions "/home/$NB_USER"
 
 USER $NB_UID
 
@@ -141,8 +142,8 @@ RUN cd /tmp && \
     pip install /tmp/toree-0.5.0.tar.gz && \
     jupyter toree install --spark_home=$SPARK_HOME --kernel_name="Spark $SPARK_VER" --interpreters=Scala && \
     rm -f /tmp/toree-0.5.0.tar.gz && \
-    fix-permissions $ANACONDA_HOME && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "$ANACONDA_HOME" && \
+    fix-permissions "/home/$NB_USER"
 
 # SETUP PASSWORDLESS SSH FOR $NB_USER
 RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \
@@ -152,7 +153,7 @@ RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \
 USER root
 
 # SETUP PASSWORDLESS SSH
-RUN yes y | ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \
+RUN yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_dsa_key && \
     yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
     yes y | ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa && \
     cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

etc/docker/demo-base/README.md

@@ -4,7 +4,7 @@
 - Hadoop 2.7.7
 - Apache Spark 2.4.6
 - Java 1.8 runtime
-- Mini-conda latest (python 3.8) with R packages
+- Mini-conda latest (python 3.9) with R packages
 - Toree 0.4.0-incubating
 - `jovyan` service user, with system users `elyra`, `bob`, and `alice`. The jovyan uid is `1000` to match other jupyter
   images.

etc/docker/demo-base/fix-permissions

@@ -20,10 +20,10 @@ set -e
 for d in "$@"; do
   find "$d" \
     ! \( \
-      -group $NB_GID \
+      -group "$NB_GID" \
       -a -perm -g+rwX  \
     \) \
-    -exec chgrp $NB_GID {} \; \
+    -exec chgrp "$NB_GID" {} \; \
     -exec chmod g+rwX {} \;
   # setuid,setgid *on directories only*
   find "$d" \

etc/docker/enterprise-gateway-demo/Dockerfile

@@ -10,8 +10,8 @@ FROM $BASE_CONTAINER
 # use an ARG instruction without a value inside of a build stage:
 ARG SPARK_VERSION
 
-ENV NB_USER="jovyan"
-ENV SPARK_VER=${SPARK_VERSION}
+ENV NB_USER "jovyan"
+ENV SPARK_VER ${SPARK_VERSION}
 
 USER $NB_USER
 

etc/docker/kernel-py/Dockerfile

@@ -2,7 +2,7 @@
 ARG BASE_CONTAINER=jupyter/scipy-notebook:2023-03-13
 FROM $BASE_CONTAINER
 
-ENV PATH=$PATH:$CONDA_DIR/bin
+ENV PATH $PATH:$CONDA_DIR/bin
 
 # Add debugger support
 RUN pip install --upgrade ipykernel

etc/docker/kernel-spark-r/Dockerfile

@@ -10,7 +10,7 @@ USER root
 
 ENV SPARK_VER $SPARK_VERSION
 ENV SPARK_HOME /opt/spark
-ENV KERNEL_LANGUAGE=R
+ENV KERNEL_LANGUAGE R
 ENV R_LIBS_USER $R_LIBS_USER:${R_HOME}/library:${SPARK_HOME}/R/lib
 ENV PATH $PATH:$SPARK_HOME/bin
 

etc/docker/kernel-tf-gpu-py/Dockerfile

@@ -2,7 +2,7 @@
 ARG BASE_CONTAINER=tensorflow/tensorflow:2.9.1-gpu
 FROM $BASE_CONTAINER
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && apt-get install -yq \
     build-essential \

etc/kernel-launchers/scala/toree-launcher/build.sbt

@@ -14,4 +14,4 @@ resolvers += "Typesafe Repo" at "https://repo.typesafe.com/typesafe/releases/"
 resolvers += "Sonatype Maven Central Mirror" at "https://maven-central.storage-download.googleapis.com/maven2/"
 
 libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" // Apache v2
-libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating" from "https://repository.apache.org/content/repositories/orgapachetoree-1020/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar"
+libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating"

pyproject.toml

@@ -15,12 +15,11 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
   "docker>=3.5.0",
   "future",