diff --git a/.github/workflows/release-binaries-all.yml b/.github/workflows/release-binaries-all.yml
index eef49b5e3625d..7e408497d1819 100644
--- a/.github/workflows/release-binaries-all.yml
+++ b/.github/workflows/release-binaries-all.yml
@@ -88,8 +88,6 @@ jobs:
         # We use ubuntu-22.04 rather than the latest version to make the built
         # binaries more portable (eg functional aginast older glibc).
         runs-on:
-          - ubuntu-22.04
-          - ubuntu-22.04-arm
           - macos-14
 
     uses: ./.github/workflows/release-binaries.yml
diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 1ffa088e99fa1..416619702f52b 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -202,9 +202,18 @@ jobs:
         fi
         echo "build-prefix=$build_prefix" >> $GITHUB_OUTPUT
 
+    - name: Download optimized compiler
+      shell: bash
+      run: |
+        curl -O -L https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.8/LLVM-20.1.8-macOS-ARM64.tar.xz
+        tar -xJf LLVM-20.1.8-macOS-ARM64.tar.xz
+
     - name: Configure
       id: build
       shell: bash
+      env:
+        CC: ${{ github.workspace }}/llvm-project/LLVM-20.1.8-macOS-ARM64/bin/clang
+        CXX: ${{ github.workspace }}/llvm-project/LLVM-20.1.8-macOS-ARM64/bin/clang++
       run: |
         # There were some issues on the ARM64 MacOS runners with trying to build x86 object,
         # so we need to set some extra cmake flags to disable this.