Skip to content

Commit 5bd84aa

Browse files
andhreljaKernandhreljaKern
committed
perf: improve test script robustness
1 parent 1a57df2 commit 5bd84aa

File tree

1 file changed

+46
-21
lines changed

1 file changed

+46
-21
lines changed

‎k8s/test.sh‎

Lines changed: 46 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,25 @@ REFINERY_POD_EXISTING_IMAGE=$(kubectl get pod --output json \
4040

4141
REFINERY_IMAGE_TAG_EXISTS=$(az acr repository show --name ${AZURE_CONTAINER_REGISTRY} --image ${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 2> /dev/null || true)
4242

43+
__safe_migration_rollout() {
44+
deploy=$1
4345

44-
if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
46+
set +e
47+
exitcode=0
48+
kubectl rollout status deployment $deploy --timeout 2m 2> /dev/null
49+
exitcode=$?
50+
set -e
51+
52+
if [ "$exitcode" != "0" ]; then
53+
kubectl logs deployment/$deploy -c $deploy-migrate
54+
if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
55+
downgrade_alembic_migrations
56+
fi
57+
exit $exitcode
58+
fi
59+
}
60+
61+
upgrade_alembic_migrations() {
4562
echo "::group::Upgrade alembic migrations for test"
4663
if [ $KUBERNETES_DEPLOYMENT_NAME != "refinery-gateway" ] && [ $KUBERNETES_DEPLOYMENT_NAME != "gates-gateway" ] && [ $KUBERNETES_DEPLOYMENT_NAME != "hosted-inference-api" ]; then
4764
if [ -n "$REFINERY_IMAGE_TAG_EXISTS" ]; then
@@ -51,7 +68,7 @@ if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
5168
kubectl set image deployment/${REFINERY_DEPLOYMENT_NAME} \
5269
${REFINERY_DEPLOYMENT_NAME}-migrate=${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} \
5370
${REFINERY_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
54-
kubectl rollout status deployment ${REFINERY_DEPLOYMENT_NAME}
71+
__safe_migration_rollout ${REFINERY_DEPLOYMENT_NAME}
5572
echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${REFINERY_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
5673
_REFINERY_ALEMBIC_VERSION=$(kubectl exec -i deployment/${REFINERY_DEPLOYMENT_NAME} -c ${REFINERY_DEPLOYMENT_NAME} -- alembic current)
5774
echo "::warning::upgraded $REFINERY_DEPLOYMENT_NAME alembic version: $_REFINERY_ALEMBIC_VERSION"
@@ -63,29 +80,15 @@ if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
6380
kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} \
6481
${KUBERNETES_DEPLOYMENT_NAME}-migrate=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} \
6582
${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
66-
kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
83+
__safe_migration_rollout ${KUBERNETES_DEPLOYMENT_NAME}
6784
echo "::warning::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
6885
_KUBERNETES_DEPLOYMENT_ALEMBIC_VERSION=$(kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c ${KUBERNETES_DEPLOYMENT_NAME} -- alembic current)
6986
echo "::warning::upgraded $KUBERNETES_DEPLOYMENT_NAME alembic version: $_KUBERNETES_DEPLOYMENT_ALEMBIC_VERSION"
7087
fi
7188
echo "::endgroup::"
72-
fi
73-
74-
echo "::group::Set test image: ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
75-
kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
76-
kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
77-
echo "::notice::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
78-
echo "::endgroup::"
79-
80-
echo "::group::Running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
81-
set +e
82-
exitcode=0
83-
kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- ''$TEST_CMD''
84-
exitcode=$?
85-
set -e
86-
echo "::endgroup::"
89+
}
8790

88-
if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
91+
downgrade_alembic_migrations() {
8992
echo "::group::Downgrade alembic migrations"
9093
if [ $KUBERNETES_DEPLOYMENT_NAME != "refinery-gateway" ] && [ $KUBERNETES_DEPLOYMENT_NAME != "gates-gateway" ] && [ $KUBERNETES_DEPLOYMENT_NAME != "hosted-inference-api" ]; then
9194
if [ -n "$REFINERY_IMAGE_TAG_EXISTS" ]; then
@@ -94,7 +97,7 @@ if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
9497
kubectl set image deployment/${REFINERY_DEPLOYMENT_NAME} \
9598
${REFINERY_DEPLOYMENT_NAME}-migrate=${REFINERY_POD_EXISTING_IMAGE} \
9699
${REFINERY_DEPLOYMENT_NAME}=${REFINERY_POD_EXISTING_IMAGE}
97-
kubectl rollout status deployment ${REFINERY_DEPLOYMENT_NAME}
100+
__safe_migration_rollout ${REFINERY_DEPLOYMENT_NAME}
98101
echo "::warning::using ${REFINERY_POD_EXISTING_IMAGE}"
99102
fi
100103
else
@@ -103,10 +106,32 @@ if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
103106
kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} \
104107
${KUBERNETES_DEPLOYMENT_NAME}-migrate=${KUBERNETES_POD_EXISTING_IMAGE} \
105108
${KUBERNETES_DEPLOYMENT_NAME}=${KUBERNETES_POD_EXISTING_IMAGE}
106-
kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
109+
__safe_migration_rollout ${KUBERNETES_DEPLOYMENT_NAME}
107110
echo "::warning::using ${KUBERNETES_POD_EXISTING_IMAGE}"
108111
fi
109112
echo "::endgroup::"
113+
}
114+
115+
if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
116+
upgrade_alembic_migrations
117+
fi
118+
119+
echo "::group::Set test image: ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
120+
kubectl set image deployment/${KUBERNETES_DEPLOYMENT_NAME} ${KUBERNETES_DEPLOYMENT_NAME}=${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG} 1> /dev/null
121+
kubectl rollout status deployment ${KUBERNETES_DEPLOYMENT_NAME}
122+
echo "::notice::using ${AZURE_CONTAINER_REGISTRY}/${KUBERNETES_DEPLOYMENT_NAME}:${TEST_IMAGE_TAG}"
123+
echo "::endgroup::"
124+
125+
echo "::group::Running test command: kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- '$TEST_CMD'"
126+
set +e
127+
exitcode=0
128+
kubectl exec -i deployment/${KUBERNETES_DEPLOYMENT_NAME} -c $KUBERNETES_DEPLOYMENT_NAME -- ''$TEST_CMD''
129+
exitcode=$?
130+
set -e
131+
echo "::endgroup::"
132+
133+
if [ "$ENABLE_ALEMBIC_MIGRATIONS" = "true" ]; then
134+
downgrade_alembic_migrations
110135
fi
111136

112137
echo "::group::Revert test image: ${KUBERNETES_POD_EXISTING_IMAGE}"

0 commit comments

Comments
 (0)