diff --git a/.github/workflows/bwc-test-workflow.yml b/.github/workflows/bwc-test-workflow.yml
index 7ec1f42cd..7f85cc424 100644
--- a/.github/workflows/bwc-test-workflow.yml
+++ b/.github/workflows/bwc-test-workflow.yml
@@ -10,20 +10,22 @@ on:
 
 jobs:
   Get-CI-Image-Tag:
-    uses: ./.github/workflows/get-ci-image-tag.yml
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
     with:
-      platform: centos7
-      usage: opensearch
+      product: opensearch
 
   build-linux:
     needs: Get-CI-Image-Tag
     strategy:
       matrix:
         java: [ 21 ]
+        os:
+          - ubuntu-24.04-arm  # arm64-preview
+          - ubuntu-24.04  # x64
     # Job name
-    name: Build and test Alerting
+    name: Backwards compatibility test Alerting with JDK ${{ matrix.java }} on ${{ matrix.os }}
     # This job runs on Linux
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
@@ -34,14 +36,18 @@ jobs:
     steps:
       - name: Run start commands
         run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
       # This step uses the checkout Github action: https://github.com/actions/checkout
       - name: Checkout Branch
         uses: actions/checkout@v4
+
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
       - name: Set Up JDK ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
+          distribution: temurin # Temurin is a distribution of adoptium
           java-version: ${{ matrix.java }}
+
       - name: Run Alerting Backwards Compatibility Tests
         run: |
           echo "Running backwards compatibility tests..."
diff --git a/alerting/build.gradle b/alerting/build.gradle
index e2e75c738..a4bd1b942 100644
--- a/alerting/build.gradle
+++ b/alerting/build.gradle
@@ -358,15 +358,17 @@ String baseName = "alertingBwcCluster"
             testDistribution = "ARCHIVE"
             versions = [bwcOpenSearchVersion, opensearch_version]
             numberOfNodes = 3
-            plugin(provider(new Callable<RegularFile>(){
+            plugin(provider(new Callable<RegularFile>() {
                 @Override
                 RegularFile call() throws Exception {
                     return new RegularFile() {
                         @Override
                         File getAsFile() {
-                            return configurations.zipArchive.asFileTree.matching {
-                                include '**/alerting*'
-                            }.singleFile
+                            return configurations.opensearchPlugin.resolvedConfiguration.resolvedArtifacts
+                                    .find { ResolvedArtifact f ->
+                                        f.name.startsWith("alerting")
+                                    }
+                                    .file
                         }
                     }
                 }