From a8089400646708461bf3ca775dde3a9025dcc6b8 Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Thu, 6 Feb 2025 20:13:07 +0100
Subject: [PATCH 1/3] Update integration script

---
 .github/workflows/armada.yml | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/armada.yml b/.github/workflows/armada.yml
index f45d3d50559c9..ea0f14d7a2d72 100644
--- a/.github/workflows/armada.yml
+++ b/.github/workflows/armada.yml
@@ -11,11 +11,37 @@ jobs:
       - uses: actions/checkout@v4
         with:
           path: spark-armada
+      - uses: actions/checkout@v4
+        with:
+          repository: richscott/armada
+          ref: rich/scala-build-container2
+          path: armada
       - uses: actions/checkout@v4
         with:
           repository: armadaproject/armada-operator
           path: armada-operator
+      - name: install sbt
+        env:
+          SBT_VERSION: 1.10.7
+          SCALA_VERSION: 2.13.15
+        run: |
+          curl -s -O https://downloads.lightbend.com/scala/${SCALA_VERSION}/scala-${SCALA_VERSION}.deb
+          sudo apt install -y ./scala-${SCALA_VERSION}.deb
+          curl -s -L -O https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz
+          tar -xzvf ./sbt-${SBT_VERSION}.tgz
+          echo "$PWD/sbt/bin" >> "${GITHUB_PATH}"
+          curl -s -L -O https://github.com/magefile/mage/releases/download/v1.15.0/mage_1.15.0_Linux-64bit.tar.gz
+          tar -xzvf ./mage_1.15.0_Linux-64bit.tar.gz mage
+          sudo apt install -y protobuf-compiler
       - run: |
+          cd armada
+          ../mage buildScala
+          cd ..
+
+          cd armada/client/scala/scala-armada-client
+          sbt -Dsbt.io.implicit.relative.glob.conversion=allow publishM2
+          cd ..
+
           cd spark-armada
           ./build/sbt package -Parmada -Pkubernetes
           ./bin/docker-image-tool.sh -t testing build
@@ -34,8 +60,7 @@ jobs:
           # sleep a bit, or we see: rpc error: code = PermissionDenied desc = could not find queue "test"
           sleep 60
 
-          ./bin/app/armadactl submit ../spark-armada/examples/spark-driver-job.yaml
-          ./bin/app/armadactl submit ../spark-armada/examples/spark-executor-job.yaml
+          ./bin/app/armadactl submit ../spark-armada/examples/spark-pi-driver.yaml
 
           # wait for the jobs to start
           sleep 60

From f0db7b89dfe77af5ee9585ed7d33a360c7afbae3 Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Fri, 7 Feb 2025 15:41:28 +0100
Subject: [PATCH 2/3] Skip mage and compile via docker directly

---
 .github/workflows/armada.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/armada.yml b/.github/workflows/armada.yml
index ea0f14d7a2d72..24512d6373b32 100644
--- a/.github/workflows/armada.yml
+++ b/.github/workflows/armada.yml
@@ -30,12 +30,10 @@ jobs:
           curl -s -L -O https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz
           tar -xzvf ./sbt-${SBT_VERSION}.tgz
           echo "$PWD/sbt/bin" >> "${GITHUB_PATH}"
-          curl -s -L -O https://github.com/magefile/mage/releases/download/v1.15.0/mage_1.15.0_Linux-64bit.tar.gz
-          tar -xzvf ./mage_1.15.0_Linux-64bit.tar.gz mage
-          sudo apt install -y protobuf-compiler
       - run: |
           cd armada
-          ../mage buildScala
+          docker build -t armada-scala-client-builder -f build/scala-client/Dockerfile .
+          docker run --rm -u $(id -u):$(id -u) -v "$PWD:/build" -w /build armada-scala-client-builder
           cd ..
 
           cd armada/client/scala/scala-armada-client

From 4a11b118e563d2ac6a0e54173dbe0cc225747bcd Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Fri, 7 Feb 2025 15:49:44 +0100
Subject: [PATCH 3/3] Restructure workflow

---
 .github/workflows/armada.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/armada.yml b/.github/workflows/armada.yml
index 24512d6373b32..ba7dba5ebbe14 100644
--- a/.github/workflows/armada.yml
+++ b/.github/workflows/armada.yml
@@ -20,7 +20,7 @@ jobs:
         with:
           repository: armadaproject/armada-operator
           path: armada-operator
-      - name: install sbt
+      - name: Install Scala and SBT
         env:
           SBT_VERSION: 1.10.7
           SCALA_VERSION: 2.13.15
@@ -30,16 +30,18 @@ jobs:
           curl -s -L -O https://github.com/sbt/sbt/releases/download/v${SBT_VERSION}/sbt-${SBT_VERSION}.tgz
           tar -xzvf ./sbt-${SBT_VERSION}.tgz
           echo "$PWD/sbt/bin" >> "${GITHUB_PATH}"
-      - run: |
+      - name: Build Armada Scala Client
+        run: |
           cd armada
           docker build -t armada-scala-client-builder -f build/scala-client/Dockerfile .
+          echo docker run --rm -u $(id -u):$(id -u) -v "$PWD:/build" -w /build armada-scala-client-builder
           docker run --rm -u $(id -u):$(id -u) -v "$PWD:/build" -w /build armada-scala-client-builder
           cd ..
 
           cd armada/client/scala/scala-armada-client
           sbt -Dsbt.io.implicit.relative.glob.conversion=allow publishM2
-          cd ..
-
+      - name: Build Spark Docker image
+        run: |
           cd spark-armada
           ./build/sbt package -Parmada -Pkubernetes
           ./bin/docker-image-tool.sh -t testing build
@@ -50,6 +52,8 @@ jobs:
           make kind-all
           ./bin/tooling/kind load image-archive ../spark_testing.tar --name armada
 
+      - name: Submit Spark app
+        run: |
           # sleep a bit, or we see: create queue request failed: rpc error: code = DeadlineExceeded
           sleep 60
 
@@ -63,7 +67,8 @@ jobs:
           # wait for the jobs to start
           sleep 60
 
-          # inspect jobs
+      - name: Inspect k8s pods
+        run: |
           kubectl get pods
           for pod in $(kubectl get pods | grep armada | cut -d " " -f 1)
           do