improved restart logic and error handling for grp config

Author: Barkha Choithani

charts/templates/statefulset.yaml

@@ -166,38 +166,52 @@ spec:
                           echo "${TIMESTAMP} $@" > /proc/$pid/fd/1
                       fi
                     }
-                    log "Info: [poststart] Begin Poststart Hook Execution"
+
+                    log "Info: [poststart] Poststart Hook Execution"
+                    restart_check () {
+                      local retry_count LAST_START timestamp=$1
+                      LAST_START=$(curl -s --anyauth --user "${MARKLOGIC_ADMIN_USERNAME}":"${MARKLOGIC_ADMIN_PASSWORD}" "http://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/timestamp")
+                      for ((retry_count = 0; retry_count < 5; retry_count = retry_count + 1)); do
+                        if [[ "${timestamp}" == "${LAST_START}" ]] || [[ -z "${LAST_START}" ]]; then
+                            sleep 10
+                            LAST_START=$(curl -s --anyauth --user "${MARKLOGIC_ADMIN_USERNAME}":"${MARKLOGIC_ADMIN_PASSWORD}" "http://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/timestamp")
+                        else
+                            log "Info: [poststart] MarkLogic host: ${MARKLOGIC_BOOTSTRAP_HOST} has restarted."
+                            return 0
+                        fi
+                      done
+                      log "Error: [poststart] Failed to restart MarkLogic host: ${MARKLOGIC_BOOTSTRAP_HOST}"
+                      exit 1
+                    }
+
                     if [[ $POD_NAME != *-0 ]]; then
                       log "Info: [poststart] Skipping group configuration."
                     else
+                      log "Info: [poststart] Begin group configuration."
                       while [ ! -f /var/opt/MarkLogic/ready ]; do
-                        log "[poststart] wait for marklogic server to be ready"
+                        log "[poststart] waiting for MarkLogic server to be ready"
                         sleep 5s
                       done
 
-                      # Fetch timestamp before updating group config
-                      TIMESTAMP=`curl --anyauth http://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/timestamp --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}`
-
                       GROUP_CFG_TEMPLATE='{"group-name":"%s", "xdqp-ssl-enabled":"%s"}'
                       GROUP_CFG=$(printf "$GROUP_CFG_TEMPLATE" "$MARKLOGIC_GROUP" "$XDQP_SSL_ENABLED") 
                       log "Info: [poststart] Updating group configuration: ${GROUP_CFG}"
-                      GROUP_RESP_CODE=`curl --retry 5 --retry-max-time 60 -o /dev/null -w "%{http_code}" --anyauth -m 20 -s -X PUT -H "Content-type: application/json" -d "${GROUP_CFG}" http://${MARKLOGIC_BOOTSTRAP_HOST}:8002/manage/v2/groups/Default/properties --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}`
-                      
-                      # check if MarkLogic has restarted
-                      if [[ ${GROUP_RESP_CODE} -eq 202 ]]; then
-                        log "Info: [poststart] Waiting for MarkLogic to restart."
-                        LAST_START=`curl --anyauth http://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/timestamp --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}`
-                        for i in $(seq 1 5); do
-                            if [ ${TIMESTAMP} == ${LAST_START} ] || [ -z ${LAST_START} ]; then
-                                sleep 10
-                                LAST_START=`curl --anyauth http://${MARKLOGIC_BOOTSTRAP_HOST}:8001/admin/v1/timestamp --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}`
-                            else
-                                log "Info: [poststart] MarkLogic has restarted."
-                            fi
-                        done
+                      GROUP_RESP_CODE=`curl --retry 5 --retry-max-time 60 -o /tmp/restart_payload.xml -w "%{http_code}" --anyauth -m 20 -s -X PUT -H "Content-type: application/json" -d "${GROUP_CFG}" http://${MARKLOGIC_BOOTSTRAP_HOST}:8002/manage/v2/groups/Default/properties --user ${MARKLOGIC_ADMIN_USERNAME}:${MARKLOGIC_ADMIN_PASSWORD}`
+                     
+                      # check whether MarkLogic has restarted
+                      if [[ ${GROUP_RESP_CODE} -eq 202 ]] || [[ ${GROUP_RESP_CODE} -eq 204 ]]; then
+                        log "Info: [poststart] Successfully configured properties for $MARKLOGIC_GROUP group on the MarkLogic cluster."
+                        TIMESTAMP=$(< /tmp/restart_payload.xml grep "last-startup" | sed 's%^.*<last-startup.*>\(.*\)</last-startup>.*$%\1%')
+                        rm  -f /tmp/restart_payload.xml
+                        if [[ -n ${TIMESTAMP} ]]; then
+                          restart_check ${TIMESTAMP}
+                        fi
+                      else
+                        log "Error: [poststart] Failed to configure properties for $MARKLOGIC_GROUP group.
+                        log "Error: [poststart] Expected response code 202 or 204, got "${GROUP_RESP_CODE}"
+                        exit 1
                       fi
                     fi
-                    log "Info: [poststart] Poststart Hook Execution Completed"
             {{- end }}
             preStop:
               exec: