add retry to gcloud run deploy (#2796)

* add retry to gcloud run deploy

* add retries to policy binding

Author: mdunker

quests/develop-apis-apigee/rest-backend/deploy.sh

@@ -18,8 +18,13 @@ export SVCACCT_EMAIL="${SVCACCT_NAME}@${GOOGLE_PROJECT_ID}.iam.gserviceaccount.c
 
 # deploy service
 # NOTE: in a production environment, you would not use max-instances=1
-echo "*** deploy ${SERVICE_NAME} service to ${CLOUDRUN_REGION} with service account ${SVCACCT_EMAIL} ***"
-gcloud run deploy ${SERVICE_NAME} \
+echo "*** deploy ${SERVICE_NAME} service to ${CLOUDRUN_REGION} with service account ${SVCACCT_EMAIL} (with retries) ***"
+
+retries=5
+delay=15
+count=0
+
+until gcloud run deploy ${SERVICE_NAME} \
   --platform=managed \
   --max-instances=1 \
   --region=${CLOUDRUN_REGION} \
@@ -29,3 +34,13 @@ gcloud run deploy ${SERVICE_NAME} \
   --project=${GOOGLE_PROJECT_ID} \
   --quiet \
   --source .
+do
+  count=$((count+1))
+  if [[ ${count} -ge ${retries} ]]; then
+    echo "Deployment failed after ${retries} attempts."
+    exit 1
+  fi
+  echo "Deployment failed, likely due to IAM propagation delay. Retrying in ${delay}s... (${count}/${retries})"
+  sleep ${delay}
+done
+

quests/develop-apis-apigee/rest-backend/init-service.sh

@@ -20,11 +20,25 @@ gcloud iam service-accounts create ${SVCACCT_NAME} \
 
 # add permission to access Firestore
 echo "*** adding role ${SVCACCT_ROLE} for Firestore access ***"
-gcloud projects add-iam-policy-binding ${GOOGLE_PROJECT_ID} \
+
+retries=5
+delay=15
+count=0
+
+until gcloud projects add-iam-policy-binding ${GOOGLE_PROJECT_ID} \
   --member="serviceAccount:${SVCACCT_EMAIL}" \
   --role=${SVCACCT_ROLE}
+do
+  count=$((count+1))
+  if [[ ${count} -ge ${retries} ]]; then
+    echo "Policy binding failed after ${retries} attempts."
+    exit 1
+  fi
+  echo "Policy binding failed, likely due to IAM propagation delay. Retrying in ${delay}s... (${count}/${retries})"
+  sleep ${delay}
+done
 
-# add permission to access Cloud Run
+# add permission to access Cloud Run, second binding should work the first time
 echo "*** adding role ${SVCACCT_ROLE2} for Cloud Run access ***"
 gcloud projects add-iam-policy-binding ${GOOGLE_PROJECT_ID} \
   --member="serviceAccount:${SVCACCT_EMAIL}" \