fix: exit on safe migration rollout failure

Author: andhreljaKern

k8s/test.sh

@@ -50,12 +50,13 @@ __safe_migration_rollout() {
     set -e
 
     if [ "$exitcode" != "0" ]; then
+        echo "::error::Alembic migration failure. See logs for details"
         kubectl logs deployment/$deploy -c $deploy-migrate
         if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
             downgrade_alembic_migrations
         fi
-        exit $exitcode
     fi
+    return $exitcode
 }
 
 upgrade_alembic_migrations() {
@@ -69,6 +70,9 @@ upgrade_alembic_migrations() {
                 ${REFINERY_DEPLOYMENT_NAME}-migrate=${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} \
                 ${REFINERY_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
             __safe_migration_rollout ${REFINERY_DEPLOYMENT_NAME}
+            if [ "$?" != "0" ]; then
+                exit 1
+            fi
             echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
             _REFINERY_ALEMBIC_VERSION=$(kubectl exec -i deployment/${REFINERY_DEPLOYMENT_NAME} -c ${REFINERY_DEPLOYMENT_NAME} -- alembic current)
             echo "::warning::upgraded $REFINERY_DEPLOYMENT_NAME alembic version: $_REFINERY_ALEMBIC_VERSION"
@@ -81,6 +85,9 @@ upgrade_alembic_migrations() {
             ${KUBERNETES_DEPLOYMENT_NAME}-migrate=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} \
             ${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
         __safe_migration_rollout ${KUBERNETES_DEPLOYMENT_NAME}
+        if [ "$?" != "0" ]; then
+            exit 1
+        fi
         echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
         _KUBERNETES_DEPLOYMENT_ALEMBIC_VERSION=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c ${KUBERNETES_DEPLOYMENT_NAME} -- alembic current)
         echo "::warning::upgraded $KUBERNETES_DEPLOYMENT_NAME alembic version: $_KUBERNETES_DEPLOYMENT_ALEMBIC_VERSION"
@@ -97,7 +104,7 @@ downgrade_alembic_migrations() {
             kubectl set image deployment/${REFINERY_DEPLOYMENT_NAME} \
                 ${REFINERY_DEPLOYMENT_NAME}-migrate=${REFINERY_POD_EXISTING_IMAGE} \
                 ${REFINERY_DEPLOYMENT_NAME}=${REFINERY_POD_EXISTING_IMAGE}
-            __safe_migration_rollout ${REFINERY_DEPLOYMENT_NAME}
+            kubectl rollout status deployment ${REFINERY_DEPLOYMENT_NAME}
             echo "::warning::using ${REFINERY_POD_EXISTING_IMAGE}"
         fi
     else
@@ -106,7 +113,7 @@ downgrade_alembic_migrations() {
         kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} \
             ${KUBERNETES_DEPLOYMENT_NAME}-migrate=${KUBERNETES_POD_EXISTING_IMAGE} \
             ${KUBERNETES_DEPLOYMENT_NAME}=${KUBERNETES_POD_EXISTING_IMAGE}
-        __safe_migration_rollout ${KUBERNETES_DEPLOYMENT_NAME}
+        kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
         echo "::warning::using ${KUBERNETES_POD_EXISTING_IMAGE}"
     fi
     echo "::endgroup::"
@@ -118,7 +125,10 @@ fi
 
 echo "::group::Set test image: ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
 kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
-kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
+__safe_migration_rollout ${KUBERNETES_DEPLOYMENT_NAME}
+if [ "$?" != "0" ]; then
+    exit 1
+fi
 echo "::notice::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
 echo "::endgroup::"