Create e2e scenario covering upgrade during training

Author: Srihari1192

.github/workflows/olm_tests.yaml

@@ -122,12 +122,12 @@ jobs:
           BUNDLE_PUSH_OPT: "--tls-verify=false"
           CATALOG_PUSH_OPT: "--tls-verify=false"
 
-      - name: Run OLM Upgrade e2e Ray cluster Up test
+      - name: Run OLM Upgrade e2e AppWrapper creation test
         run: |
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
           set -euo pipefail
-          go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
       - name: Update Operator to the built version
         run: |
@@ -158,12 +158,12 @@ jobs:
           SUBSCRIPTION_NAME: "codeflare-operator"
           SUBSCRIPTION_NAMESPACE: "openshift-operators"
 
-      - name: Run OLM Upgrade e2e Ray cluster Job submit test
+      - name: Run OLM Upgrade e2e Appwrapper Job status test to monitor training
         run: |
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
           set -euo pipefail
-          go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
       - name: Run e2e tests against built operator
         run: |

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/onsi/gomega v1.27.10
 	github.com/openshift/api v0.0.0-20230213134911-7ba313770556
 	github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069
+	github.com/project-codeflare/instascale v0.3.1
 	github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1
 	github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1
 	go.uber.org/zap v1.26.0