From da7190c6c509066efad671259a8d4755acd51d0d Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 22 Jun 2025 18:38:51 -0700
Subject: [PATCH 01/43] Remove Python 3.8 to fix build issues

---
 .github/workflows/build.yml    | 4 ++--
 docs/environment.yml           | 2 +-
 etc/docker/demo-base/README.md | 2 +-
 pyproject.toml                 | 3 +--
 4 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 720d2fc21..47ffd948f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -96,7 +96,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.8"
+          python_version: "3.9"
       - uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1
       - name: Run the unit tests
         run: |
diff --git a/docs/environment.yml b/docs/environment.yml
index 48cfd61e1..286e86a4e 100644
--- a/docs/environment.yml
+++ b/docs/environment.yml
@@ -5,6 +5,6 @@ channels:
   - free
 dependencies:
   - pip
-  - python=3.8
+  - python=3.9
   - pip:
       - -r doc-requirements.txt
diff --git a/etc/docker/demo-base/README.md b/etc/docker/demo-base/README.md
index 84bcd7432..155f5b2c2 100644
--- a/etc/docker/demo-base/README.md
+++ b/etc/docker/demo-base/README.md
@@ -4,7 +4,7 @@
 - Hadoop 2.7.7
 - Apache Spark 2.4.6
 - Java 1.8 runtime
-- Mini-conda latest (python 3.8) with R packages
+- Mini-conda latest (python 3.9) with R packages
 - Toree 0.4.0-incubating
 - `jovyan` service user, with system users `elyra`, `bob`, and `alice`. The jovyan uid is `1000` to match other jupyter
   images.
diff --git a/pyproject.toml b/pyproject.toml
index 8eff50492..829c6be41 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,12 +15,11 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
 ]
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 dependencies = [
   "docker>=3.5.0",
   "future",

From d926a9168c50ec9f5da0aca84c4ddcd540b31048 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 22 Jun 2025 23:55:47 -0700
Subject: [PATCH 02/43] Update sbr launch

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 47ffd948f..e2681ed2d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,8 +27,8 @@ jobs:
           conda --version
       - name: Add SBT launcher
         run: |
-          mkdir -p $HOME/.sbt/launchers/1.3.12
-          curl -L -o $HOME/.sbt/launchers/1.3.12/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.3.12/sbt-launch.jar
+          mkdir -p $HOME/.sbt/launchers/1.9.3
+          curl -L -o $HOME/.sbt/launchers/1.9.3/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.9.3/sbt-launch-1.9.3.jar
       - name: Install Python dependencies
         run: |
           pip install ".[test]"

From 39cfbb02ed22993b432a22459018bf71c58f55d2 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 00:34:27 -0700
Subject: [PATCH 03/43] Add sbt launch action

---
 .github/workflows/build.yml | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e2681ed2d..dfef77a12 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -25,10 +25,12 @@ jobs:
           python --version
           pip --version
           conda --version
-      - name: Add SBT launcher
-        run: |
-          mkdir -p $HOME/.sbt/launchers/1.9.3
-          curl -L -o $HOME/.sbt/launchers/1.9.3/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.9.3/sbt-launch-1.9.3.jar
+#      - name: Add SBT launcher
+#        run: |
+#          mkdir -p $HOME/.sbt/launchers/1.9.3
+#          curl -L -o $HOME/.sbt/launchers/1.9.3/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.9.3/sbt-launch-1.9.3.jar
+      - name: Setup sbt launcher
+        uses: sbt/setup-sbt@v1
       - name: Install Python dependencies
         run: |
           pip install ".[test]"

From 07189ca8e52c2cdd43392ab4c12e166cde4797e5 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:05:22 -0700
Subject: [PATCH 04/43] Fix pyzmq build issue

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index dfef77a12..d317e4992 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -7,6 +7,7 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     env:
+      ZMQ_SYSTEM_LIBZMQ: 1
       ASYNC_TEST_TIMEOUT: 30
       CONDA_HOME: /usr/share/miniconda
     strategy:

From ffc8609454ecbe28c33a9aeb9a028fbf6e25e5b4 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:20:03 -0700
Subject: [PATCH 05/43] Downgrade gcc

---
 .github/workflows/build.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d317e4992..b0ee05b5f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -16,6 +16,12 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11"]
     steps:
+      - name: Downgrade GCC to version 12
+        run: |
+          sudo apt-get install -y gcc-12 g++-12
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
+          gcc --version
       - name: Checkout
         uses: actions/checkout@v4
         with:

From 546f9ea98417c1e64dd8fadb1d3c2100a635f725 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:20:25 -0700
Subject: [PATCH 06/43] use python 3.11 for check links

---
 .github/workflows/build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index b0ee05b5f..e1a51c300 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,6 +79,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
       - name: Install Python dependencies
         run: |

From b158df3941e4aa12df7989186f4cc3c461752c33 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:24:37 -0700
Subject: [PATCH 07/43] Set python_version to be used with base-setup action

---
 .github/workflows/build.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e1a51c300..4bc2b6cb0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,6 +77,8 @@ jobs:
 
   link_check:
     runs-on: ubuntu-latest
+    env:
+      PYTHON_VERSION: '3.11'
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python 3.11

From 898c6786965a953ee146bf36cc234152e77eed00 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:30:58 -0700
Subject: [PATCH 08/43] downgrade gcc for link check

---
 .github/workflows/build.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4bc2b6cb0..ca2f16e93 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -78,6 +78,7 @@ jobs:
   link_check:
     runs-on: ubuntu-latest
     env:
+      ZMQ_SYSTEM_LIBZMQ: 1
       PYTHON_VERSION: '3.11'
     steps:
       - uses: actions/checkout@v4
@@ -85,6 +86,12 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
+      - name: Downgrade GCC to version 12
+        run: |
+          sudo apt-get install -y gcc-12 g++-12
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
+          gcc --version
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
       - name: Install Python dependencies
         run: |

From cd180c877b247495bdf41ce876297f50cfff1846 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:39:05 -0700
Subject: [PATCH 09/43] update libzmq3

---
 .github/workflows/build.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ca2f16e93..81ffed2c0 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -92,6 +92,8 @@ jobs:
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
           gcc --version
+      - name: Install system ZeroMQ
+        run: sudo apt-get update && sudo apt-get install -y libzmq3-dev
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
       - name: Install Python dependencies
         run: |

From 49fdd11c45b6941d6994d36031aaff163e01a447 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:46:52 -0700
Subject: [PATCH 10/43] force python 3.11

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 81ffed2c0..3090e095f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,6 +77,7 @@ jobs:
 
   link_check:
     runs-on: ubuntu-latest
+    python-version: ['3.11']
     env:
       ZMQ_SYSTEM_LIBZMQ: 1
       PYTHON_VERSION: '3.11'

From 2b7e2486527780ebe34ba53a3fb7444340fe22a4 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 01:55:06 -0700
Subject: [PATCH 11/43] Remove duplicate python config

---
 .github/workflows/build.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3090e095f..ee27f0ac1 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -80,13 +80,8 @@ jobs:
     python-version: ['3.11']
     env:
       ZMQ_SYSTEM_LIBZMQ: 1
-      PYTHON_VERSION: '3.11'
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.11'
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12

From d4c3d1e9050e8bb4b4830f0ce31a78e288e525b4 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:03:33 -0700
Subject: [PATCH 12/43] Another try for python 3.11

---
 .github/workflows/build.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ee27f0ac1..bec4dd489 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -77,11 +77,16 @@ jobs:
 
   link_check:
     runs-on: ubuntu-latest
-    python-version: ['3.11']
     env:
       ZMQ_SYSTEM_LIBZMQ: 1
     steps:
       - uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12

From cfc312b2b47e168fa403847e8302e886dd6c3541 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:19:08 -0700
Subject: [PATCH 13/43] display python version

---
 .github/workflows/build.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index bec4dd489..d6e3034e4 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -87,6 +87,9 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Display Python version
+        run: python -c "import sys; print(sys.version)"
+
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12

From 0cd05e31cbeda4e0b7ef53becbad00cdb6360f07 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:23:03 -0700
Subject: [PATCH 14/43] Set python_version env

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d6e3034e4..c8f4e74d9 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,6 +79,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       ZMQ_SYSTEM_LIBZMQ: 1
+      PYTHON_VERSION: '3.11'
     steps:
       - uses: actions/checkout@v4
 

From 63095525e0de58af60de170d9eeb639155a7de48 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:39:57 -0700
Subject: [PATCH 15/43] update python with alternative

---
 .github/workflows/build.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c8f4e74d9..8a677d275 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -91,6 +91,13 @@ jobs:
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
 
+      - name: Force Python 3.11
+        run: |
+          sudo update-alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1
+          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
+          sudo update-alternatives --config python3
+          
+
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12

From 96cfe4788484309d52609f54e7f4f01425bd7e93 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:42:41 -0700
Subject: [PATCH 16/43] Whereis python

---
 .github/workflows/build.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 8a677d275..d477823ce 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -89,7 +89,9 @@ jobs:
           python-version: '3.11'
 
       - name: Display Python version
-        run: python -c "import sys; print(sys.version)"
+        run: |
+          python -c "import sys; print(sys.version)"
+          whereis python
 
       - name: Force Python 3.11
         run: |

From 59e2537be53f73309d7f37f8ffc5d92252ecb98a Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:45:56 -0700
Subject: [PATCH 17/43] update python config

---
 .github/workflows/build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d477823ce..a322424ea 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -95,8 +95,8 @@ jobs:
 
       - name: Force Python 3.11
         run: |
-          sudo update-alternatives --install /usr/bin/python3 python /usr/bin/python3.11 1
-          sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1
+          sudo update-alternatives --install /usr/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+          sudo update-alternatives --install /usr/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
           sudo update-alternatives --config python3
           
 

From e6359a2dcfa009817f7c910ae7f6400c8b1011ec Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:48:54 -0700
Subject: [PATCH 18/43] add apt-python

---
 .github/workflows/build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index a322424ea..3b309ab08 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -103,6 +103,7 @@ jobs:
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12
+          sudo apt-get install -y python3-apt
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
           gcc --version

From 8f53b0a74bc191f46845b772f4000109b8d78430 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 02:57:37 -0700
Subject: [PATCH 19/43] update path with python

---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 3b309ab08..7cc63c38f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -95,15 +95,15 @@ jobs:
 
       - name: Force Python 3.11
         run: |
-          sudo update-alternatives --install /usr/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
-          sudo update-alternatives --install /usr/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+          sudo update-alternatives --install /usr/local/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+          sudo update-alternatives --install /usr/local/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+          echo "/usr/local/bin:$PATH" >> $GITHUB_PATH
           sudo update-alternatives --config python3
           
 
       - name: Downgrade GCC to version 12
         run: |
           sudo apt-get install -y gcc-12 g++-12
-          sudo apt-get install -y python3-apt
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
           sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
           gcc --version

From 556ffdf29c0f50ae63ffdf4aa3911e4328b3780d Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 03:05:53 -0700
Subject: [PATCH 20/43] update python env var

---
 .github/workflows/build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7cc63c38f..e95d045e5 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -92,14 +92,15 @@ jobs:
         run: |
           python -c "import sys; print(sys.version)"
           whereis python
+          whereis python3
 
       - name: Force Python 3.11
         run: |
           sudo update-alternatives --install /usr/local/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
           sudo update-alternatives --install /usr/local/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
           echo "/usr/local/bin:$PATH" >> $GITHUB_PATH
+          echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
           sudo update-alternatives --config python3
-          
 
       - name: Downgrade GCC to version 12
         run: |

From 565d017b3291cd684565dbe445778622fc037203 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 03:15:20 -0700
Subject: [PATCH 21/43] Update with python

---
 .github/workflows/build.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e95d045e5..5afa1103f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -111,7 +111,11 @@ jobs:
       - name: Install system ZeroMQ
         run: sudo apt-get update && sudo apt-get install -y libzmq3-dev
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python-version: '3.11'
       - name: Install Python dependencies
+        with:
+          python-version: '3.11'
         run: |
           pip install ".[test]"
       - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1

From 0f0c726b78a16fd214b16b9794cf4f66ddbbb7f4 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 22:55:01 -0700
Subject: [PATCH 22/43] playing with python_version

---
 .github/workflows/build.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5afa1103f..6d557e234 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,6 +27,8 @@ jobs:
         with:
           clean: true
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - name: Display dependency info
         run: |
           python --version
@@ -114,8 +116,6 @@ jobs:
         with:
           python-version: '3.11'
       - name: Install Python dependencies
-        with:
-          python-version: '3.11'
         run: |
           pip install ".[test]"
       - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1
@@ -129,6 +129,8 @@ jobs:
         uses: actions/checkout@v4
       - name: Base Setup
         uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - name: Build Docs
         run: make docs
 
@@ -153,6 +155,8 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - uses: jupyterlab/maintainer-tools/.github/actions/make-sdist@v1
 
   test_sdist:
@@ -162,6 +166,8 @@ jobs:
     timeout-minutes: 20
     steps:
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+        with:
+          python_version: "3.11"
       - uses: jupyterlab/maintainer-tools/.github/actions/test-sdist@v1
 
   python_tests_check: # This job does nothing and is only used for the branch protection

From 2e7a2c716c0cf9e287f2a1ffabc9daa83fe06438 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Mon, 23 Jun 2025 23:13:31 -0700
Subject: [PATCH 23/43] Cleanup

---
 .github/workflows/build.yml | 64 +++++++++++++++++--------------------
 1 file changed, 30 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 6d557e234..c4066514f 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -7,7 +7,6 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     env:
-      ZMQ_SYSTEM_LIBZMQ: 1
       ASYNC_TEST_TIMEOUT: 30
       CONDA_HOME: /usr/share/miniconda
     strategy:
@@ -16,12 +15,12 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11"]
     steps:
-      - name: Downgrade GCC to version 12
-        run: |
-          sudo apt-get install -y gcc-12 g++-12
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
-          gcc --version
+      # - name: Downgrade GCC to version 12
+      #   run: |
+      #     sudo apt-get install -y gcc-12 g++-12
+      #     sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+      #     sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
+      #     gcc --version
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -79,48 +78,45 @@ jobs:
 
   link_check:
     runs-on: ubuntu-latest
-    env:
-      ZMQ_SYSTEM_LIBZMQ: 1
-      PYTHON_VERSION: '3.11'
     steps:
       - uses: actions/checkout@v4
-
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 
-      - name: Display Python version
-        run: |
-          python -c "import sys; print(sys.version)"
-          whereis python
-          whereis python3
+      # - name: Display Python version
+      #   run: |
+      #     python -c "import sys; print(sys.version)"
+      #     whereis python
+      #     whereis python3
 
-      - name: Force Python 3.11
-        run: |
-          sudo update-alternatives --install /usr/local/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
-          sudo update-alternatives --install /usr/local/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
-          echo "/usr/local/bin:$PATH" >> $GITHUB_PATH
-          echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
-          sudo update-alternatives --config python3
+      # - name: Force Python 3.11
+      #   run: |
+      #     sudo update-alternatives --install /usr/local/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+      #     sudo update-alternatives --install /usr/local/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
+      #     echo "/usr/local/bin:$PATH" >> $GITHUB_PATH
+      #     echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
+      #     sudo update-alternatives --config python3
 
-      - name: Downgrade GCC to version 12
-        run: |
-          sudo apt-get install -y gcc-12 g++-12
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
-          gcc --version
-      - name: Install system ZeroMQ
-        run: sudo apt-get update && sudo apt-get install -y libzmq3-dev
-      - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-        with:
-          python-version: '3.11'
+      # - name: Downgrade GCC to version 12
+      #   run: |
+      #     sudo apt-get install -y gcc-12 g++-12
+      #     sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
+      #     sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
+      #     gcc --version
+      # - name: Install system ZeroMQ
+      #   run: sudo apt-get update && sudo apt-get install -y libzmq3-dev
+      # - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+      #   with:
+      #     python-version: '3.11'
       - name: Install Python dependencies
         run: |
           pip install ".[test]"
       - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1
         with:
           ignore_links: "http://my-gateway-server.com:8888"
+          python-version: '3.11'
 
   build_docs:
     runs-on: windows-latest

From 87dcfa5a7e9cec844e818aa0fae0be4c692c5158 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 20:08:26 -0700
Subject: [PATCH 24/43] install python 3.11 on minimal test action

---
 .github/workflows/build.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c4066514f..d736a6eef 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -136,10 +136,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.9"
-      - uses: jupyterlab/maintainer-tools/.github/actions/install-minimums@v1
+          python_version: "3.11"
+      - name: Install dependencies with minimum versions
+        run: |
+          pip install ".[test]"
       - name: Run the unit tests
         run: |
           pytest -vv -W default || pytest -vv -W default --lf

From 4a6a39221f906581d8cb3a2ec08f2dae57be40b8 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 20:18:46 -0700
Subject: [PATCH 25/43] Use matrix python versions

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index d736a6eef..170bef255 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -27,7 +27,7 @@ jobs:
           clean: true
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.11"
+          python_version: "${{ matrix.python-version }}"
       - name: Display dependency info
         run: |
           python --version

From 595ebcb3a680ef94dc0d5e2976b0765b4e7fa394 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 20:52:50 -0700
Subject: [PATCH 26/43] cleanup

---
 .github/workflows/build.yml | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 170bef255..94feef82a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -15,12 +15,6 @@ jobs:
         os: [ubuntu-latest]
         python-version: ["3.9", "3.10", "3.11"]
     steps:
-      # - name: Downgrade GCC to version 12
-      #   run: |
-      #     sudo apt-get install -y gcc-12 g++-12
-      #     sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
-      #     sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
-      #     gcc --version
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -33,10 +27,6 @@ jobs:
           python --version
           pip --version
           conda --version
-#      - name: Add SBT launcher
-#        run: |
-#          mkdir -p $HOME/.sbt/launchers/1.9.3
-#          curl -L -o $HOME/.sbt/launchers/1.9.3/sbt-launch.jar https://repo1.maven.org/maven2/org/scala-sbt/sbt-launch/1.9.3/sbt-launch-1.9.3.jar
       - name: Setup sbt launcher
         uses: sbt/setup-sbt@v1
       - name: Install Python dependencies
@@ -84,32 +74,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-
-      # - name: Display Python version
-      #   run: |
-      #     python -c "import sys; print(sys.version)"
-      #     whereis python
-      #     whereis python3
-
-      # - name: Force Python 3.11
-      #   run: |
-      #     sudo update-alternatives --install /usr/local/bin/python python /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
-      #     sudo update-alternatives --install /usr/local/bin/python3 python3 /opt/hostedtoolcache/Python/3.11.13/x64/bin/python 1
-      #     echo "/usr/local/bin:$PATH" >> $GITHUB_PATH
-      #     echo "PYTHON_VERSION=3.11" >> $GITHUB_ENV
-      #     sudo update-alternatives --config python3
-
-      # - name: Downgrade GCC to version 12
-      #   run: |
-      #     sudo apt-get install -y gcc-12 g++-12
-      #     sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100
-      #     sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100
-      #     gcc --version
-      # - name: Install system ZeroMQ
-      #   run: sudo apt-get update && sudo apt-get install -y libzmq3-dev
-      # - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-      #   with:
-      #     python-version: '3.11'
       - name: Install Python dependencies
         run: |
           pip install ".[test]"

From 97943c8a0e0bfdf55092019783187ea92f100cf4 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 20:53:04 -0700
Subject: [PATCH 27/43] use py 3.9

---
 .github/workflows/build.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 94feef82a..ed6292ef7 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -100,13 +100,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python 3.9
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: '3.9'
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
         with:
-          python_version: "3.11"
+          python_version: "3.9"
       - name: Install dependencies with minimum versions
         run: |
           pip install ".[test]"

From 5b7391b774b0995da87790bb6d964d5846fd8683 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 21:06:21 -0700
Subject: [PATCH 28/43] Increase timeouts and enable logs

---
 .github/workflows/build.yml | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index ed6292ef7..e9895d8b6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -7,7 +7,8 @@ jobs:
   build:
     runs-on: ${{ matrix.os }}
     env:
-      ASYNC_TEST_TIMEOUT: 30
+      ASYNC_TEST_TIMEOUT: 60
+      KERNEL_LAUNCH_TIMEOUT: 120
       CONDA_HOME: /usr/share/miniconda
     strategy:
       fail-fast: false
@@ -51,14 +52,24 @@ jobs:
             make test
       - name: Run integration tests
         run: |
-          make itest-yarn
+          # Increase timeout for integration tests
+          export PREP_TIMEOUT=180
+          # Set specific Spark version to match what's expected in tests
+          export SPARK_VERSION=3.2.1
+          # Run integration tests with debug output
+          make LOG_LEVEL=DEBUG itest-yarn
       - name: Collect logs
         if: success() || failure()
         run: |
           python --version
           pip --version
           pip list
+          echo "==== Docker Container Logs ===="
           docker logs itest-yarn
+          echo "==== Docker Container Status ===="
+          docker ps -a
+          echo "==== Enterprise Gateway Log ===="
+          docker exec -it itest-yarn cat /usr/local/share/jupyter/enterprise-gateway.log || true
       - name: Run linters
         run: |
           make lint

From 10302006dbe19187bf5ade2e8ad42bb935589458 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 23:33:54 -0700
Subject: [PATCH 29/43] update gitignore

---
 .gitignore | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/.gitignore b/.gitignore
index 42d69573d..596d74d5a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,6 +43,7 @@ nosetests.xml
 coverage.xml
 *,cover
 .pytest_cache/
+.bsp
 
 # Translations
 *.mo
@@ -60,10 +61,6 @@ target/
 .DS_Store
 .ipynb_checkpoints/
 
-# PyCharm
-.idea/
-*.iml
-
 # Build-related
 .image-*
 
@@ -74,11 +71,11 @@ _site/
 # Debug-related
 .kube/
 
+# PyCharm
+.idea/
+*.iml
+
 # vscode ide stuff
 *.code-workspace
 .history/
 .vscode/
-
-# jetbrains ide stuff
-*.iml
-.idea/

From 60502ea5efa31e03dea762647616dcd3315ea076 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Wed, 25 Jun 2025 23:34:59 -0700
Subject: [PATCH 30/43] Update itest with debug

---
 .github/workflows/build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e9895d8b6..c785a4a98 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -57,7 +57,7 @@ jobs:
           # Set specific Spark version to match what's expected in tests
           export SPARK_VERSION=3.2.1
           # Run integration tests with debug output
-          make LOG_LEVEL=DEBUG itest-yarn
+          make itest-yarn-debug
       - name: Collect logs
         if: success() || failure()
         run: |

From afcaaa38fd07b86fdc5633cd47337ad029f915d0 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Thu, 26 Jun 2025 00:22:08 -0700
Subject: [PATCH 31/43] Use enterprise-gateway-demo:dev-lresende

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 1c5b1e854..c4b813434 100644
--- a/Makefile
+++ b/Makefile
@@ -224,7 +224,7 @@ PREP_TIMEOUT?=60
 itest-yarn-prep:
 	@-docker rm -f itest-yarn >> /dev/null
 	@echo "Starting enterprise-gateway container (run \`docker logs itest-yarn\` to see container log)..."
-	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:$(TAG) --gateway
+	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:dev-lresende --gateway
 	@(r="1"; attempts=0; while [ "$$r" == "1" -a $$attempts -lt $(PREP_TIMEOUT) ]; do echo "Waiting for enterprise-gateway to start..."; sleep 2; ((attempts++)); docker logs itest-yarn |grep --regexp "Jupyter Enterprise Gateway .* is available at http"; r=$$?; done; if [ $$attempts -ge $(PREP_TIMEOUT) ]; then echo "Wait for startup timed out!"; exit 1; fi;)
 
 

From c31b6210ae1be49ffbf23b4d3fe3f599e4369902 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Thu, 26 Jun 2025 07:52:58 -0700
Subject: [PATCH 32/43] use official toree release

---
 etc/kernel-launchers/scala/toree-launcher/build.sbt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/etc/kernel-launchers/scala/toree-launcher/build.sbt b/etc/kernel-launchers/scala/toree-launcher/build.sbt
index 05f1c09ba..1174bc98c 100644
--- a/etc/kernel-launchers/scala/toree-launcher/build.sbt
+++ b/etc/kernel-launchers/scala/toree-launcher/build.sbt
@@ -14,4 +14,4 @@ resolvers += "Typesafe Repo" at "https://repo.typesafe.com/typesafe/releases/"
 resolvers += "Sonatype Maven Central Mirror" at "https://maven-central.storage-download.googleapis.com/maven2/"
 
 libraryDependencies += "com.typesafe.play" %% "play-json" % "2.7.4" // Apache v2
-libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating" from "https://repository.apache.org/content/repositories/orgapachetoree-1020/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar"
+libraryDependencies += "org.apache.toree" % "toree-assembly" % "0.5.0-incubating"

From 9bd5ccf89b7dd3d864e24270cb7676f917b5fd09 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Thu, 26 Jun 2025 07:59:56 -0700
Subject: [PATCH 33/43] Revert using enterprise-gateway-demo:dev-lresende

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index c4b813434..421c03c55 100644
--- a/Makefile
+++ b/Makefile
@@ -224,7 +224,7 @@ PREP_TIMEOUT?=60
 itest-yarn-prep:
 	@-docker rm -f itest-yarn >> /dev/null
 	@echo "Starting enterprise-gateway container (run \`docker logs itest-yarn\` to see container log)..."
-	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:dev-lresende --gateway
+	@-docker run -itd -p $(ITEST_YARN_PORT):$(ITEST_YARN_PORT) -p 8088:8088 -p 8042:8042 -h itest-yarn --name itest-yarn -v `pwd`/enterprise_gateway/itests:/tmp/byok elyra/enterprise-gateway-demo:dev --gateway
 	@(r="1"; attempts=0; while [ "$$r" == "1" -a $$attempts -lt $(PREP_TIMEOUT) ]; do echo "Waiting for enterprise-gateway to start..."; sleep 2; ((attempts++)); docker logs itest-yarn |grep --regexp "Jupyter Enterprise Gateway .* is available at http"; r=$$?; done; if [ $$attempts -ge $(PREP_TIMEOUT) ]; then echo "Wait for startup timed out!"; exit 1; fi;)
 
 

From f20967db9ab30cba72187a8aac7caf3cce8f6d3b Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Fri, 27 Jun 2025 21:28:18 -0700
Subject: [PATCH 34/43] Use maven to download

---
 etc/Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/etc/Makefile b/etc/Makefile
index 420aff25e..20ad46aac 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -87,7 +87,8 @@ kernel-launchers/scala/lib: $(TOREE_LAUNCHER_FILES)
 	-rm -rf kernel-launchers/scala/lib
 	mkdir -p kernel-launchers/scala/lib
 	@(cd kernel-launchers/scala/toree-launcher; sbt -Dversion=$(VERSION) -Dspark_version=$(SPARK_VERSION) package; cp target/scala-2.12/*.jar ../lib)
-	curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
+	# curl -L https://repository.apache.org/content/repositories/releases/org/apache/toree/toree-assembly/0.5.0-incubating/toree-assembly-0.5.0-incubating.jar --output ./kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
+	mvn org.apache.maven.plugins:maven-dependency-plugin:3.3.0:get -DrepoUrl=https://repository.apache.org/content/repositories/releases/ -Dartifact=org.apache.toree:toree-assembly:0.5.0-incubating -Ddest=./etc/kernel-launchers/scala/lib/toree-assembly-0.5.0-incubating.jar
 
 KERNEL_IMAGE_FILE:=../dist/jupyter_enterprise_gateway_kernel_image_files-$(VERSION).tar.gz
 kernel_image_files: ../build/kernel_image_files

From 8ecdd8439cb83e270652e5f3bcdec9c1a82e5578 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Fri, 27 Jun 2025 21:44:26 -0700
Subject: [PATCH 35/43] Update failing pi test to add imports

---
 enterprise_gateway/itests/test_python_kernel.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/enterprise_gateway/itests/test_python_kernel.py b/enterprise_gateway/itests/test_python_kernel.py
index 945920fd6..ed6328d98 100644
--- a/enterprise_gateway/itests/test_python_kernel.py
+++ b/enterprise_gateway/itests/test_python_kernel.py
@@ -124,10 +124,13 @@ def test_run_pi_example(self):
         pi_code.append("from operator import add\n")
         pi_code.append("partitions = 20\n")
         pi_code.append("n = 100000 * partitions\n")
+        # Define the function with explicit imports inside to avoid serialization issues
         pi_code.append("def f(_):\n")
+        pi_code.append("    from random import random\n")  # Include import inside function
         pi_code.append("    x = random() * 2 - 1\n")
         pi_code.append("    y = random() * 2 - 1\n")
         pi_code.append("    return 1 if x ** 2 + y ** 2 <= 1 else 0\n")
+        # Use a lambda function directly in map to avoid complex serialization
         pi_code.append("count = sc.parallelize(range(1, n + 1), partitions).map(f).reduce(add)\n")
         pi_code.append('print("Pi is roughly %f" % (4.0 * count / n))\n')
         result, has_error = self.kernel.execute(pi_code)

From 0ab28216beeda7f02e115ae10b414a9016376985 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sat, 28 Jun 2025 15:53:00 -0700
Subject: [PATCH 36/43] Remove with matrix.python-version

---
 .github/workflows/build.yml                     | 5 +++--
 enterprise_gateway/itests/test_python_kernel.py | 3 +++
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index c785a4a98..eca269841 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -21,8 +21,8 @@ jobs:
         with:
           clean: true
       - uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-        with:
-          python_version: "${{ matrix.python-version }}"
+#        with:
+#          python_version: "${{ matrix.python-version }}"
       - name: Display dependency info
         run: |
           python --version
@@ -33,6 +33,7 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install ".[test]"
+          pip install --upgrade cloudpickle
       - name: Build and install Jupyter Enterprise Gateway
         uses: nick-invision/retry@v3.0.0
         with:
diff --git a/enterprise_gateway/itests/test_python_kernel.py b/enterprise_gateway/itests/test_python_kernel.py
index ed6328d98..79c9d58ad 100644
--- a/enterprise_gateway/itests/test_python_kernel.py
+++ b/enterprise_gateway/itests/test_python_kernel.py
@@ -122,6 +122,7 @@ def test_run_pi_example(self):
         pi_code = []
         pi_code.append("from random import random\n")
         pi_code.append("from operator import add\n")
+        pi_code.append("from pyspark.cloudpickle import cloudpickle\n")
         pi_code.append("partitions = 20\n")
         pi_code.append("n = 100000 * partitions\n")
         # Define the function with explicit imports inside to avoid serialization issues
@@ -130,6 +131,8 @@ def test_run_pi_example(self):
         pi_code.append("    x = random() * 2 - 1\n")
         pi_code.append("    y = random() * 2 - 1\n")
         pi_code.append("    return 1 if x ** 2 + y ** 2 <= 1 else 0\n")
+        pi_code.append("pickled_f = cloudpickle.dumps(f)\n")
+        pi_code.append("print('Function f successfully pickled!')\n")
         # Use a lambda function directly in map to avoid complex serialization
         pi_code.append("count = sc.parallelize(range(1, n + 1), partitions).map(f).reduce(add)\n")
         pi_code.append('print("Pi is roughly %f" % (4.0 * count / n))\n')

From cc21ea12f0206b815f43f663707c79e7a1b3145a Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sat, 28 Jun 2025 20:10:53 -0700
Subject: [PATCH 37/43] Play with random

---
 enterprise_gateway/itests/test_python_kernel.py | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/enterprise_gateway/itests/test_python_kernel.py b/enterprise_gateway/itests/test_python_kernel.py
index 79c9d58ad..945920fd6 100644
--- a/enterprise_gateway/itests/test_python_kernel.py
+++ b/enterprise_gateway/itests/test_python_kernel.py
@@ -122,18 +122,12 @@ def test_run_pi_example(self):
         pi_code = []
         pi_code.append("from random import random\n")
         pi_code.append("from operator import add\n")
-        pi_code.append("from pyspark.cloudpickle import cloudpickle\n")
         pi_code.append("partitions = 20\n")
         pi_code.append("n = 100000 * partitions\n")
-        # Define the function with explicit imports inside to avoid serialization issues
         pi_code.append("def f(_):\n")
-        pi_code.append("    from random import random\n")  # Include import inside function
         pi_code.append("    x = random() * 2 - 1\n")
         pi_code.append("    y = random() * 2 - 1\n")
         pi_code.append("    return 1 if x ** 2 + y ** 2 <= 1 else 0\n")
-        pi_code.append("pickled_f = cloudpickle.dumps(f)\n")
-        pi_code.append("print('Function f successfully pickled!')\n")
-        # Use a lambda function directly in map to avoid complex serialization
         pi_code.append("count = sc.parallelize(range(1, n + 1), partitions).map(f).reduce(add)\n")
         pi_code.append('print("Pi is roughly %f" % (4.0 * count / n))\n')
         result, has_error = self.kernel.execute(pi_code)

From 4fbc8f1b78839acedb92daefd652f592f20fb1e5 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 01:49:20 -0700
Subject: [PATCH 38/43] uses python 3.9 for integration tests

---
 .github/workflows/build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index eca269841..9bd104bc6 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -33,7 +33,6 @@ jobs:
       - name: Install Python dependencies
         run: |
           pip install ".[test]"
-          pip install --upgrade cloudpickle
       - name: Build and install Jupyter Enterprise Gateway
         uses: nick-invision/retry@v3.0.0
         with:
@@ -59,6 +58,8 @@ jobs:
           export SPARK_VERSION=3.2.1
           # Run integration tests with debug output
           make itest-yarn-debug
+        with:
+          python-version: "3.9"
       - name: Collect logs
         if: success() || failure()
         run: |

From 08c381a43285c0391b162834ce66e51a7becf0b5 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 01:50:09 -0700
Subject: [PATCH 39/43] Fix demo-base docker build

Update ssh keygen
Uses python 3.10
---
 etc/docker/demo-base/Dockerfile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/etc/docker/demo-base/Dockerfile b/etc/docker/demo-base/Dockerfile
index 9b484c507..2edb15dd1 100644
--- a/etc/docker/demo-base/Dockerfile
+++ b/etc/docker/demo-base/Dockerfile
@@ -106,7 +106,8 @@ RUN conda install mamba -n base -c conda-forge && \
     'jupyter' \
     'r-devtools' \
     'r-stringr' \
-    'r-argparse' && \
+    'r-argparse' \
+    'python=3.10' && \
     mamba clean -y --all &&\
     fix-permissions $ANACONDA_HOME && \
     fix-permissions /home/$NB_USER
@@ -152,7 +153,7 @@ RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \
 USER root
 
 # SETUP PASSWORDLESS SSH
-RUN yes y | ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_dsa_key && \
+RUN yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_dsa_key && \
     yes y | ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key && \
     yes y | ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa && \
     cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys

From 8bfde2714712b412f4d2577bd9d8015c4a739adb Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 01:55:14 -0700
Subject: [PATCH 40/43] lint python tags

---
 .github/workflows/build.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9bd104bc6..2f8de62bc 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -58,8 +58,6 @@ jobs:
           export SPARK_VERSION=3.2.1
           # Run integration tests with debug output
           make itest-yarn-debug
-        with:
-          python-version: "3.9"
       - name: Collect logs
         if: success() || failure()
         run: |
@@ -93,7 +91,6 @@ jobs:
       - uses: jupyterlab/maintainer-tools/.github/actions/check-links@v1
         with:
           ignore_links: "http://my-gateway-server.com:8888"
-          python-version: '3.11'
 
   build_docs:
     runs-on: windows-latest

From ae253bfaf491ecb39c1298388e2b04cd7115e12d Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 02:05:41 -0700
Subject: [PATCH 41/43] uses nick-invision/retry@v3.0.0

---
 .github/workflows/build.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2f8de62bc..30405594a 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -51,13 +51,14 @@ jobs:
           command: |
             make test
       - name: Run integration tests
-        run: |
-          # Increase timeout for integration tests
-          export PREP_TIMEOUT=180
-          # Set specific Spark version to match what's expected in tests
-          export SPARK_VERSION=3.2.1
-          # Run integration tests with debug output
-          make itest-yarn-debug
+        uses: nick-invision/retry@v3.0.0
+        with:
+          timeout_minutes: 3
+          max_attempts: 1
+          command: |
+            export PREP_TIMEOUT=180
+            # Run integration tests with debug output
+            make itest-yarn-debug
       - name: Collect logs
         if: success() || failure()
         run: |

From 6fa1c2c16c96f962586ccb83c7dba2c704e6a0be Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 15:42:52 -0700
Subject: [PATCH 42/43] Use Spark 3.2.4

---
 Makefile                                   | 2 +-
 enterprise_gateway/itests/test_r_kernel.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 421c03c55..44e33fd63 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@ MULTIARCH_BUILD?=
 TARGET_ARCH?=undefined
 
 VERSION?=3.3.0.dev0
-SPARK_VERSION?=3.2.1
+SPARK_VERSION?=3.2.4
 
 ifeq (dev, $(findstring dev, $(VERSION)))
     TAG:=dev
diff --git a/enterprise_gateway/itests/test_r_kernel.py b/enterprise_gateway/itests/test_r_kernel.py
index b32cbde96..8d53b27a8 100644
--- a/enterprise_gateway/itests/test_r_kernel.py
+++ b/enterprise_gateway/itests/test_r_kernel.py
@@ -34,7 +34,7 @@ def test_restart(self):
         self.assertTrue(self.kernel.restart())
 
         error_result, has_error = self.kernel.execute("y = x + 1")
-        self.assertRegex(error_result, "Error in eval")
+        self.assertRegex(error_result, r"Error: object '(\w+)' not found")
         self.assertEqual(has_error, True)
 
     def test_interrupt(self):

From ecb09ce2e639aa32f7cce477db34ac5ec2bf09f5 Mon Sep 17 00:00:00 2001
From: Luciano Resende <lresende@apple.com>
Date: Sun, 29 Jun 2025 21:15:39 -0700
Subject: [PATCH 43/43] Fix dockerfile

---
 etc/docker/demo-base/Dockerfile               | 56 +++++++++----------
 etc/docker/demo-base/fix-permissions          |  4 +-
 etc/docker/enterprise-gateway-demo/Dockerfile |  4 +-
 etc/docker/kernel-py/Dockerfile               |  2 +-
 etc/docker/kernel-spark-r/Dockerfile          |  2 +-
 etc/docker/kernel-tf-gpu-py/Dockerfile        |  2 +-
 6 files changed, 35 insertions(+), 35 deletions(-)

diff --git a/etc/docker/demo-base/Dockerfile b/etc/docker/demo-base/Dockerfile
index 2edb15dd1..6d8ad0cf9 100644
--- a/etc/docker/demo-base/Dockerfile
+++ b/etc/docker/demo-base/Dockerfile
@@ -9,23 +9,23 @@ ARG NB_GID="100"
 
 USER root
 
-ENV HADOOP_HOME=/usr/hdp/current/hadoop \
-    ANACONDA_HOME=/opt/conda
-
-ENV SHELL=/bin/bash \
-    NB_USER=$NB_USER \
-    NB_UID=$NB_UID \
-    NB_GID=$NB_GID \
-    LC_ALL=en_US.UTF-8 \
-    LANG=en_US.UTF-8 \
-    LANGUAGE=en_US.UTF-8 \
-    JAVA_HOME=/usr/lib/jvm/java \
-    SPARK_HOME=/usr/hdp/current/spark2-client \
-    PYSPARK_PYTHON=$ANACONDA_HOME/bin/python \
-    HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
-
-ENV HOME=/home/$NB_USER \
-    PATH=$JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH
+ENV HADOOP_HOME /usr/hdp/current/hadoop
+ENV ANACONDA_HOME /opt/conda
+
+ENV SHELL /bin/bash \
+    NB_USER $NB_USER \
+    NB_UID $NB_UID \
+    NB_GID $NB_GID \
+    LC_ALL en_US.UTF-8 \
+    LANG en_US.UTF-8 \
+    LANGUAGE en_US.UTF-8 \
+    JAVA_HOME /usr/lib/jvm/java \
+    SPARK_HOME /usr/hdp/current/spark2-client \
+    PYSPARK_PYTHON $ANACONDA_HOME/bin/python \
+    HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
+
+ENV HOME /home/$NB_USER
+ENV PATH $JAVA_HOME/bin:$ANACONDA_HOME/bin:$HADOOP_HOME/bin:$SPARK_HOME/bin:$PATH
 
 ENV SPARK_VER $SPARK_VERSION
 ENV HADOOP_VER 3.3.1
@@ -56,7 +56,7 @@ RUN dpkg --purge --force-depends ca-certificates-java \
     && apt-add-repository 'deb http://security.debian.org/debian-security bullseye-security main' \
     && apt-add-repository 'deb http://deb.debian.org/debian/ sid main' \
     && apt-get update && apt-get install -yq --no-install-recommends  \
-    openjdk-8-jre-headless \
+    openjdk-8-jdk-headless \
     ca-certificates-java \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
@@ -71,14 +71,14 @@ ADD fix-permissions /usr/local/bin/fix-permissions
 # and make sure these dirs are writable by the `users` group.
 RUN groupadd wheel -g 11 && \
     echo "auth required pam_wheel.so use_uid" >> /etc/pam.d/su && \
-    useradd -m -s /bin/bash -N -u $NB_UID $NB_USER && \
+    useradd -m -s /bin/bash -N -u "$NB_UID" "$NB_USER" && \
     mkdir -p /usr/hdp/current && \
     mkdir -p /usr/local/share/jupyter && \
-    chown $NB_USER:$NB_GID $ANACONDA_HOME && \
+    chown "$NB_USER":"$NB_GID" "$ANACONDA_HOME" && \
     chmod g+w /etc/passwd && \
     chmod +x /usr/local/bin/fix-permissions && \
-    fix-permissions $HOME && \
-    fix-permissions $ANACONDA_HOME && \
+    fix-permissions "$HOME" && \
+    fix-permissions "$ANACONDA_HOME" && \
     fix-permissions /usr/hdp/current && \
     fix-permissions /usr/local/share/jupyter
 
@@ -90,8 +90,8 @@ RUN useradd -m -s /bin/bash -N -u 1111 elyra && \
 USER $NB_UID
 
 # Setup work directory for backward-compatibility
-RUN mkdir /home/$NB_USER/work && \
-    fix-permissions /home/$NB_USER
+RUN mkdir "/home/$NB_USER/work" && \
+    fix-permissions "/home/$NB_USER"
 
 # DOWNLOAD HADOOP AND SPARK
 RUN curl -sL https://archive.apache.org/dist/hadoop/common/hadoop-$HADOOP_VER/hadoop-$HADOOP_VER.tar.gz | tar -xz -C /usr/hdp/current
@@ -109,8 +109,8 @@ RUN conda install mamba -n base -c conda-forge && \
     'r-argparse' \
     'python=3.10' && \
     mamba clean -y --all &&\
-    fix-permissions $ANACONDA_HOME && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "$ANACONDA_HOME" && \
+    fix-permissions "/home/$NB_USER"
 
 USER $NB_UID
 
@@ -142,8 +142,8 @@ RUN cd /tmp && \
     pip install /tmp/toree-0.5.0.tar.gz && \
     jupyter toree install --spark_home=$SPARK_HOME --kernel_name="Spark $SPARK_VER" --interpreters=Scala && \
     rm -f /tmp/toree-0.5.0.tar.gz && \
-    fix-permissions $ANACONDA_HOME && \
-    fix-permissions /home/$NB_USER
+    fix-permissions "$ANACONDA_HOME" && \
+    fix-permissions "/home/$NB_USER"
 
 # SETUP PASSWORDLESS SSH FOR $NB_USER
 RUN ssh-keygen -q -N "" -t rsa -f /home/$NB_USER/.ssh/id_rsa && \
diff --git a/etc/docker/demo-base/fix-permissions b/etc/docker/demo-base/fix-permissions
index 659b2763e..485e5e911 100755
--- a/etc/docker/demo-base/fix-permissions
+++ b/etc/docker/demo-base/fix-permissions
@@ -20,10 +20,10 @@ set -e
 for d in "$@"; do
   find "$d" \
     ! \( \
-      -group $NB_GID \
+      -group "$NB_GID" \
       -a -perm -g+rwX  \
     \) \
-    -exec chgrp $NB_GID {} \; \
+    -exec chgrp "$NB_GID" {} \; \
     -exec chmod g+rwX {} \;
   # setuid,setgid *on directories only*
   find "$d" \
diff --git a/etc/docker/enterprise-gateway-demo/Dockerfile b/etc/docker/enterprise-gateway-demo/Dockerfile
index f8c2ea35c..01d0985ab 100644
--- a/etc/docker/enterprise-gateway-demo/Dockerfile
+++ b/etc/docker/enterprise-gateway-demo/Dockerfile
@@ -10,8 +10,8 @@ FROM $BASE_CONTAINER
 # use an ARG instruction without a value inside of a build stage:
 ARG SPARK_VERSION
 
-ENV NB_USER="jovyan"
-ENV SPARK_VER=${SPARK_VERSION}
+ENV NB_USER "jovyan"
+ENV SPARK_VER ${SPARK_VERSION}
 
 USER $NB_USER
 
diff --git a/etc/docker/kernel-py/Dockerfile b/etc/docker/kernel-py/Dockerfile
index e967509bb..54c688f33 100644
--- a/etc/docker/kernel-py/Dockerfile
+++ b/etc/docker/kernel-py/Dockerfile
@@ -2,7 +2,7 @@
 ARG BASE_CONTAINER=jupyter/scipy-notebook:2023-03-13
 FROM $BASE_CONTAINER
 
-ENV PATH=$PATH:$CONDA_DIR/bin
+ENV PATH $PATH:$CONDA_DIR/bin
 
 # Add debugger support
 RUN pip install --upgrade ipykernel
diff --git a/etc/docker/kernel-spark-r/Dockerfile b/etc/docker/kernel-spark-r/Dockerfile
index 5e92caeaa..8274cc511 100644
--- a/etc/docker/kernel-spark-r/Dockerfile
+++ b/etc/docker/kernel-spark-r/Dockerfile
@@ -10,7 +10,7 @@ USER root
 
 ENV SPARK_VER $SPARK_VERSION
 ENV SPARK_HOME /opt/spark
-ENV KERNEL_LANGUAGE=R
+ENV KERNEL_LANGUAGE R
 ENV R_LIBS_USER $R_LIBS_USER:${R_HOME}/library:${SPARK_HOME}/R/lib
 ENV PATH $PATH:$SPARK_HOME/bin
 
diff --git a/etc/docker/kernel-tf-gpu-py/Dockerfile b/etc/docker/kernel-tf-gpu-py/Dockerfile
index d6b6c5d27..a02b5117a 100644
--- a/etc/docker/kernel-tf-gpu-py/Dockerfile
+++ b/etc/docker/kernel-tf-gpu-py/Dockerfile
@@ -2,7 +2,7 @@
 ARG BASE_CONTAINER=tensorflow/tensorflow:2.9.1-gpu
 FROM $BASE_CONTAINER
 
-ENV DEBIAN_FRONTEND=noninteractive
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get update && apt-get install -yq \
     build-essential \