diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 720d2fc21..30405594a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,28 +7,29 @@ jobs: build: runs-on: ${{ matrix.os }} env: - ASYNC_TEST_TIMEOUT: 30 + ASYNC_TEST_TIMEOUT: 60 + KERNEL_LAUNCH_TIMEOUT: 120 CONDA_HOME: /usr/share/miniconda strategy: fail-fast: false matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.9", "3.10", "3.11"] steps: - name: Checkout uses: actions/checkout@v4 with: clean: true - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 +# with: +# python_version: "${{ matrix.python-version }}" - name: Display dependency info run: | python --version pip --version conda --version - - name: Add SBT launcher - run: | - mkdir -p $HOME/.sbt/launchers/1.3.12 - curl -L -o $HOME/.sbt/launchers/1.3.12/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.3.12/sbt-launch.jar + - name: Setup sbt launcher + uses: sbt/setup-sbt@v1 - name: Install Python dependencies run: | pip install ".[test]" @@ -50,15 +51,26 @@ jobs: command: | make test - name: Run integration tests - run: | - make itest-yarn + uses: nick-invision/retry@v3.0.0 + with: + timeout_minutes: 3 + max_attempts: 1 + command: | + export PREP_TIMEOUT=180 + # Run integration tests with debug output + make itest-yarn-debug - name: Collect logs if: success() || failure() run: | python --version pip --version pip list + echo "==== Docker Container Logs ====" docker logs itest-yarn + echo "==== Docker Container Status ====" + docker ps -a + echo "==== Enterprise Gateway Log ====" + docker exec -it itest-yarn cat /usr/local/share/jupyter/enterprise-gateway.log || true - name: Run linters run: | make lint @@ -70,7 +82,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' - name: Install Python dependencies run: | pip install ".[test]" @@ -85,6 +100,8 @@ jobs: uses: actions/checkout@v4 - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: "3.11" - name: Build Docs run: make docs @@ -94,10 +111,16 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v5 + with: + python-version: '3.9' - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 with: - python_version: "3.8" - - uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1 + python_version: "3.9" + - name: Install dependencies with minimum versions + run: | + pip install ".[test]" - name: Run the unit tests run: | pytest -vv -W default || pytest -vv -W default --lf @@ -109,6 +132,8 @@ jobs: steps: - uses: actions/checkout@v4 - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: "3.11" - uses: jupyterlab/maintainer-tools/.github/actions/make-sdist@v1 test_sdist: @@ -118,6 +143,8 @@ jobs: timeout-minutes: 20 steps: - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + with: + python_version: "3.11" - uses: jupyterlab/maintainer-tools/.github/actions/test-sdist@v1 python_tests_check: # This job does nothing and is only used for the branch protection diff --git a/.gitignore b/.gitignore index 42d69573d..596d74d5a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ nosetests.xml coverage.xml *,cover .pytest_cache/ +.bsp # Translations *.mo @@ -60,10 +61,6 @@ target/ .DS_Store .ipynb_checkpoints/ -# PyCharm -.idea/ -*.iml - # Build-related .image-* @@ -74,11 +71,11 @@ _site/ # Debug-related .kube/ +# PyCharm +.idea/ +*.iml + # vscode ide stuff *.code-workspace .history/ .vscode/ - -# jetbrains ide stuff -*.iml -.idea/ diff --git a/Makefile b/Makefile index 1c5b1e854..44e33fd63 100644 --- a/Makefile +++ b/Makefile @@ -15,7 +15,7 @@ MULTIARCH_BUILD?= TARGET_ARCH?=undefined VERSION?=3.3.0.dev0 -SPARK_VERSION?=3.2.1 +SPARK_VERSION?=3.2.4 ifeq (dev, $(findstring dev, $(VERSION))) TAG:=dev @@ -224,7 +224,7 @@ PREP_TIMEOUT?=60 itest-yarn-prep: @-docker rm -f itest-yarn >> /dev/null @echo "Starting enterprise-gateway container (run \`docker logs itest-yarn\` to see container log)..." - @-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:$(TAG) --gateway + @-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:dev --gateway @(r="1"; attempts=0; while [ "$$r" == "1" -a $$attempts -lt $(PREP_TIMEOUT) ]; do echo "Waiting for enterprise-gateway to start..."; sleep 2; ((attempts++)); docker logs itest-yarn |grep --regexp "Jupyter Enterprise Gateway .* is available at http"; r=$$?; done; if [ $$attempts -ge $(PREP_TIMEOUT) ]; then echo "Wait for startup timed out!"; exit 1; fi;) diff --git a/docs/environment.yml b/docs/environment.yml index 48cfd61e1..286e86a4e 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -5,6 +5,6 @@ channels: - free dependencies: - pip - - python=3.8 + - python=3.9 - pip: - -r doc-requirements.txt diff --git a/enterprise_gateway/itests/test_r_kernel.py b/enterprise_gateway/itests/test_r_kernel.py index b32cbde96..8d53b27a8 100644 --- a/enterprise_gateway/itests/test_r_kernel.py +++ b/enterprise_gateway/itests/test_r_kernel.py @@ -34,7 +34,7 @@ def test_restart(self): self.assertTrue(self.kernel.restart()) error_result, has_error = self.kernel.execute("y = x + 1") - self.assertRegex(error_result, "Error in eval") + self.assertRegex(error_result, r"Error: object '(\w+)' not found") self.assertEqual(has_error, True) def test_interrupt(self): diff --git a/etc/Makefile b/etc/Makefile index 420aff25e..20ad46aac 100644 --- a/etc/Makefile +++ b/etc/Makefile @@ -87,7 +87,8 @@ kernel-launchers/scala/lib: $(TOREE_LAUNCHER_FILES) -rm -rf kernel-launchers/scala/lib mkdir -p kernel-launchers/scala/lib @(cd kernel-launchers/scala/toree-launcher; sbt -Dversion=$(VERSION) -Dspark_version=$(SPARK_VERSION) package; cp target/scala-2.12/*.jar ../lib) - curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar + # curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar + mvn org.apache.maven.plugins:maven-dependency-plugin:3.3.0:get -DrepoUrl=https://repository.apache.org/content/repositories/releases/ -Dartifact=org.apache.toree:toree-assembly:0.5.0-incubating -Ddest=./etc/kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar KERNEL_IMAGE_FILE:=../dist/jupyter_enterprise_gateway_kernel_image_files-$(VERSION).tar.gz kernel_image_files: ../build/kernel_image_files diff --git a/etc/docker/demo-base/Dockerfile b/etc/docker/demo-base/Dockerfile index 9b484c507..6d8ad0cf9 100644 --- a/etc/docker/demo-base/Dockerfile +++ b/etc/docker/demo-base/Dockerfile @@ -9,23 +9,23 @@ ARG NB_GID="100" USER root -ENV HADOOP_HOME=/usr/hdp/current/hadoop \ - ANACONDA_HOME=/opt/conda - -ENV SHELL=/bin/bash \ - NB_USER=$NB_USER \ - NB_UID=$NB_UID \ - NB_GID=$NB_GID \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 \ - JAVA_HOME=/usr/lib/jvm/java \ - SPARK_HOME=/usr/hdp/current/spark2-client \ - PYSPARK_PYTHON=$ANACONDA_HOME/bin/python \ - HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop - -ENV HOME=/home/$NB_USER \ - PATH=$JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH +ENV HADOOP_HOME /usr/hdp/current/hadoop +ENV ANACONDA_HOME /opt/conda + +ENV SHELL /bin/bash \ + NB_USER $NB_USER \ + NB_UID $NB_UID \ + NB_GID $NB_GID \ + LC_ALL en_US.UTF-8 \ + LANG en_US.UTF-8 \ + LANGUAGE en_US.UTF-8 \ + JAVA_HOME /usr/lib/jvm/java \ + SPARK_HOME /usr/hdp/current/spark2-client \ + PYSPARK_PYTHON $ANACONDA_HOME/bin/python \ + HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop + +ENV HOME /home/$NB_USER +ENV PATH $JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH ENV SPARK_VER $SPARK_VERSION ENV HADOOP_VER 3.3.1 @@ -56,7 +56,7 @@ RUN dpkg --purge --force-depends ca-certificates-java \ && apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security main' \ && apt-add-repository 'deb http://deb.debian.org/debian/ sid main' \ && apt-get update && apt-get install -yq --no-install-recommends \ - openjdk-8-jre-headless \ + openjdk-8-jdk-headless \ ca-certificates-java \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -71,14 +71,14 @@ ADD fix-permissions /usr/local/bin/fix-permissions # and make sure these dirs are writable by the `users` group. RUN groupadd wheel -g 11 && \ echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \ - useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \ + useradd -m -s /bin/bash -N -u "$NB_UID" "$NB_USER" && \ mkdir -p /usr/hdp/current && \ mkdir -p /usr/local/share/jupyter && \ - chown $NB_USER:$NB_GID $ANACONDA_HOME && \ + chown "$NB_USER":"$NB_GID" "$ANACONDA_HOME" && \ chmod g+w /etc/passwd && \ chmod +x /usr/local/bin/fix-permissions && \ - fix-permissions $HOME && \ - fix-permissions $ANACONDA_HOME && \ + fix-permissions "$HOME" && \ + fix-permissions "$ANACONDA_HOME" && \ fix-permissions /usr/hdp/current && \ fix-permissions /usr/local/share/jupyter @@ -90,8 +90,8 @@ RUN useradd -m -s /bin/bash -N -u 1111 elyra && \ USER $NB_UID # Setup work directory for backward-compatibility -RUN mkdir /home/$NB_USER/work && \ - fix-permissions /home/$NB_USER +RUN mkdir "/home/$NB_USER/work" && \ + fix-permissions "/home/$NB_USER" # DOWNLOAD HADOOP AND SPARK RUN curl -sL https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VER/hadoop-$HADOOP_VER.tar.gz | tar -xz -C /usr/hdp/current @@ -106,10 +106,11 @@ RUN conda install mamba -n base -c conda-forge && \ 'jupyter' \ 'r-devtools' \ 'r-stringr' \ - 'r-argparse' && \ + 'r-argparse' \ + 'python=3.10' && \ mamba clean -y --all &&\ - fix-permissions $ANACONDA_HOME && \ - fix-permissions /home/$NB_USER + fix-permissions "$ANACONDA_HOME" && \ + fix-permissions "/home/$NB_USER" USER $NB_UID @@ -141,8 +142,8 @@ RUN cd /tmp && \ pip install /tmp/toree-0.5.0.tar.gz && \ jupyter toree install --spark_home=$SPARK_HOME --kernel_name="Spark $SPARK_VER" --interpreters=Scala && \ rm -f /tmp/toree-0.5.0.tar.gz && \ - fix-permissions $ANACONDA_HOME && \ - fix-permissions /home/$NB_USER + fix-permissions "$ANACONDA_HOME" && \ + fix-permissions "/home/$NB_USER" # SETUP PASSWORDLESS SSH FOR $NB_USER RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \ @@ -152,7 +153,7 @@ RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \ USER root # SETUP PASSWORDLESS SSH -RUN yes y | ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \ +RUN yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_dsa_key && \ yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \ yes y | ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa && \ cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys diff --git a/etc/docker/demo-base/README.md b/etc/docker/demo-base/README.md index 84bcd7432..155f5b2c2 100644 --- a/etc/docker/demo-base/README.md +++ b/etc/docker/demo-base/README.md @@ -4,7 +4,7 @@ - Hadoop 2.7.7 - Apache Spark 2.4.6 - Java 1.8 runtime -- Mini-conda latest (python 3.8) with R packages +- Mini-conda latest (python 3.9) with R packages - Toree 0.4.0-incubating - `jovyan` service user, with system users `elyra`, `bob`, and `alice`. The jovyan uid is `1000` to match other jupyter images. diff --git a/etc/docker/demo-base/fix-permissions b/etc/docker/demo-base/fix-permissions index 659b2763e..485e5e911 100755 --- a/etc/docker/demo-base/fix-permissions +++ b/etc/docker/demo-base/fix-permissions @@ -20,10 +20,10 @@ set -e for d in "$@"; do find "$d" \ ! \( \ - -group $NB_GID \ + -group "$NB_GID" \ -a -perm -g+rwX \ \) \ - -exec chgrp $NB_GID {} \; \ + -exec chgrp "$NB_GID" {} \; \ -exec chmod g+rwX {} \; # setuid,setgid *on directories only* find "$d" \ diff --git a/etc/docker/enterprise-gateway-demo/Dockerfile b/etc/docker/enterprise-gateway-demo/Dockerfile index f8c2ea35c..01d0985ab 100644 --- a/etc/docker/enterprise-gateway-demo/Dockerfile +++ b/etc/docker/enterprise-gateway-demo/Dockerfile @@ -10,8 +10,8 @@ FROM $BASE_CONTAINER # use an ARG instruction without a value inside of a build stage: ARG SPARK_VERSION -ENV NB_USER="jovyan" -ENV SPARK_VER=${SPARK_VERSION} +ENV NB_USER "jovyan" +ENV SPARK_VER ${SPARK_VERSION} USER $NB_USER diff --git a/etc/docker/kernel-py/Dockerfile b/etc/docker/kernel-py/Dockerfile index e967509bb..54c688f33 100644 --- a/etc/docker/kernel-py/Dockerfile +++ b/etc/docker/kernel-py/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_CONTAINER=jupyter/scipy-notebook:2023-03-13 FROM $BASE_CONTAINER -ENV PATH=$PATH:$CONDA_DIR/bin +ENV PATH $PATH:$CONDA_DIR/bin # Add debugger support RUN pip install --upgrade ipykernel diff --git a/etc/docker/kernel-spark-r/Dockerfile b/etc/docker/kernel-spark-r/Dockerfile index 5e92caeaa..8274cc511 100644 --- a/etc/docker/kernel-spark-r/Dockerfile +++ b/etc/docker/kernel-spark-r/Dockerfile @@ -10,7 +10,7 @@ USER root ENV SPARK_VER $SPARK_VERSION ENV SPARK_HOME /opt/spark -ENV KERNEL_LANGUAGE=R +ENV KERNEL_LANGUAGE R ENV R_LIBS_USER $R_LIBS_USER:${R_HOME}/library:${SPARK_HOME}/R/lib ENV PATH $PATH:$SPARK_HOME/bin diff --git a/etc/docker/kernel-tf-gpu-py/Dockerfile b/etc/docker/kernel-tf-gpu-py/Dockerfile index d6b6c5d27..a02b5117a 100644 --- a/etc/docker/kernel-tf-gpu-py/Dockerfile +++ b/etc/docker/kernel-tf-gpu-py/Dockerfile @@ -2,7 +2,7 @@ ARG BASE_CONTAINER=tensorflow/tensorflow:2.9.1-gpu FROM $BASE_CONTAINER -ENV DEBIAN_FRONTEND=noninteractive +ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -yq \ build-essential \ diff --git a/etc/kernel-launchers/scala/toree-launcher/build.sbt b/etc/kernel-launchers/scala/toree-launcher/build.sbt index 05f1c09ba..1174bc98c 100644 --- a/etc/kernel-launchers/scala/toree-launcher/build.sbt +++ b/etc/kernel-launchers/scala/toree-launcher/build.sbt @@ -14,4 +14,4 @@ resolvers += "Typesafe Repo" at "https://repo.typesafe.com/typesafe/releases/" resolvers += "Sonatype Maven Central Mirror" at "https://maven-central.storage-download.googleapis.com/maven2/" libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" // Apache v2 -libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating" from "https://repository.apache.org/content/repositories/orgapachetoree-1020/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar" +libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating" diff --git a/pyproject.toml b/pyproject.toml index 8eff50492..829c6be41 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,12 +15,11 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ] -requires-python = ">=3.8" +requires-python = ">=3.9" dependencies = [ "docker>=3.5.0", "future",