fix: handle curl exit code 000 in validate-kubelet-credentials.sh (#6201)

Co-authored-by: Cameron Meissner <cameronmeissner@gmail.com>

Author: UtheMan

parts/linux/cloud-init/artifacts/cse_config.sh

@@ -662,7 +662,8 @@ EOF
         logs_to_events "AKS.CSE.ensureKubelet.installCredentialProvider" installCredentialProvider
     fi
 
-    systemctlEnableAndStart kubelet 30 || exit $ERR_KUBELET_START_FAIL
+    # 4-minute timeout to give enough time to all of kubelet's ExecStartPre scripts before hitting their own respective timeouts
+    systemctlEnableAndStart kubelet 240 || exit $ERR_KUBELET_START_FAIL
 }
 
 ensureSnapshotUpdate() {

parts/linux/cloud-init/artifacts/validate-kubelet-credentials.sh

@@ -60,6 +60,8 @@ function validateBootstrapKubeconfig {
         exit 0
     fi
 
+    echo "will check credential validity against apiserver url: $apiserver_url"
+
     local retry_count=0
     while true; do
         code=$(curl -sL \
@@ -71,16 +73,34 @@ function validateBootstrapKubeconfig {
             --cacert "$cacert" \
             "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s")
 
+        curl_code=$?
+
         if [ $code -ge 200 ] && [ $code -lt 400 ]; then
             echo "(retry=$retry_count) received valid HTTP status code from apiserver: $code"
             break
         fi
 
+        if [ $code -eq 000 ]; then
+            echo "(retry=$retry_count) curl response code is $code, curl exited with code: $curl_code"
+            echo "retrying once more to get a more detailed error response..."
+
+            curl -L \
+                -m $RETRY_TIMEOUT_SECONDS \
+                -H "Accept: application/json, */*" \
+                -H "Authorization: Bearer ${bootstrap_token//\"/}" \
+                --cacert "$cacert" \
+                "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s"
+
+            echo "proceeding to start kubelet..."
+            exit 0
+        fi
+
         echo "(retry=$retry_count) received invalid HTTP status code from apiserver: $code"
 
         retry_count=$(( $retry_count + 1 ))
         if [ $retry_count -eq $MAX_RETRIES ]; then
             echo "unable to validate bootstrap credentials after $retry_count attempts"
+            echo "proceeding to start kubelet..."
             exit 0
         fi
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/CustomData

@@ -57,6 +57,8 @@ function validateBootstrapKubeconfig {
         exit 0
     fi
 
+    echo "will check credential validity against apiserver url: $apiserver_url"
+
     local retry_count=0
     while true; do
         code=$(curl -sL \
@@ -68,16 +70,34 @@ function validateBootstrapKubeconfig {
             --cacert "$cacert" \
             "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s")
 
+        curl_code=$?
+
         if [ $code -ge 200 ] && [ $code -lt 400 ]; then
             echo "(retry=$retry_count) received valid HTTP status code from apiserver: $code"
             break
         fi
 
+        if [ $code -eq 000 ]; then
+            echo "(retry=$retry_count) curl response code is $code, curl exited with code: $curl_code"
+            echo "retrying once more to get a more detailed error response..."
+
+            curl -L \
+                -m $RETRY_TIMEOUT_SECONDS \
+                -H "Accept: application/json, */*" \
+                -H "Authorization: Bearer ${bootstrap_token//\"/}" \
+                --cacert "$cacert" \
+                "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s"
+
+            echo "proceeding to start kubelet..."
+            exit 0
+        fi
+
         echo "(retry=$retry_count) received invalid HTTP status code from apiserver: $code"
 
         retry_count=$(( $retry_count + 1 ))
         if [ $retry_count -eq $MAX_RETRIES ]; then
             echo "unable to validate bootstrap credentials after $retry_count attempts"
+            echo "proceeding to start kubelet..."
             exit 0
         fi
 

pkg/agent/testdata/AKSUbuntu1604+Containerd/line315.sh

@@ -627,7 +627,7 @@ EOF
         logs_to_events "AKS.CSE.ensureKubelet.installCredentialProvider" installCredentialProvider
     fi
 
-    systemctlEnableAndStart kubelet 30 || exit $ERR_KUBELET_START_FAIL
+    systemctlEnableAndStart kubelet 240 || exit $ERR_KUBELET_START_FAIL
 }
 
 ensureSnapshotUpdate() {

pkg/agent/testdata/AKSUbuntu1604+Containerd/line70.sh

@@ -57,6 +57,8 @@ function validateBootstrapKubeconfig {
         exit 0
     fi
 
+    echo "will check credential validity against apiserver url: $apiserver_url"
+
     local retry_count=0
     while true; do
         code=$(curl -sL \
@@ -68,16 +70,34 @@ function validateBootstrapKubeconfig {
             --cacert "$cacert" \
             "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s")
 
+        curl_code=$?
+
         if [ $code -ge 200 ] && [ $code -lt 400 ]; then
             echo "(retry=$retry_count) received valid HTTP status code from apiserver: $code"
             break
         fi
 
+        if [ $code -eq 000 ]; then
+            echo "(retry=$retry_count) curl response code is $code, curl exited with code: $curl_code"
+            echo "retrying once more to get a more detailed error response..."
+
+            curl -L \
+                -m $RETRY_TIMEOUT_SECONDS \
+                -H "Accept: application/json, */*" \
+                -H "Authorization: Bearer ${bootstrap_token//\"/}" \
+                --cacert "$cacert" \
+                "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s"
+
+            echo "proceeding to start kubelet..."
+            exit 0
+        fi
+
         echo "(retry=$retry_count) received invalid HTTP status code from apiserver: $code"
 
         retry_count=$(( $retry_count + 1 ))
         if [ $retry_count -eq $MAX_RETRIES ]; then
             echo "unable to validate bootstrap credentials after $retry_count attempts"
+            echo "proceeding to start kubelet..."
             exit 0
         fi
 

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/CustomData

@@ -627,7 +627,7 @@ EOF
         logs_to_events "AKS.CSE.ensureKubelet.installCredentialProvider" installCredentialProvider
     fi
 
-    systemctlEnableAndStart kubelet 30 || exit $ERR_KUBELET_START_FAIL
+    systemctlEnableAndStart kubelet 240 || exit $ERR_KUBELET_START_FAIL
 }
 
 ensureSnapshotUpdate() {

pkg/agent/testdata/AKSUbuntu1604+CustomKubeletConfig+CustomLinuxOSConfig/line315.sh

@@ -57,6 +57,8 @@ function validateBootstrapKubeconfig {
         exit 0
     fi
 
+    echo "will check credential validity against apiserver url: $apiserver_url"
+
     local retry_count=0
     while true; do
         code=$(curl -sL \
@@ -68,16 +70,34 @@ function validateBootstrapKubeconfig {
             --cacert "$cacert" \
             "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s")
 
+        curl_code=$?
+
         if [ $code -ge 200 ] && [ $code -lt 400 ]; then
             echo "(retry=$retry_count) received valid HTTP status code from apiserver: $code"
             break
         fi
 
+        if [ $code -eq 000 ]; then
+            echo "(retry=$retry_count) curl response code is $code, curl exited with code: $curl_code"
+            echo "retrying once more to get a more detailed error response..."
+
+            curl -L \
+                -m $RETRY_TIMEOUT_SECONDS \
+                -H "Accept: application/json, */*" \
+                -H "Authorization: Bearer ${bootstrap_token//\"/}" \
+                --cacert "$cacert" \
+                "${apiserver_url}/version?timeout=${RETRY_TIMEOUT_SECONDS}s"
+
+            echo "proceeding to start kubelet..."
+            exit 0
+        fi
+
         echo "(retry=$retry_count) received invalid HTTP status code from apiserver: $code"
 
         retry_count=$(( $retry_count + 1 ))
         if [ $retry_count -eq $MAX_RETRIES ]; then
             echo "unable to validate bootstrap credentials after $retry_count attempts"
+            echo "proceeding to start kubelet..."
             exit 0
         fi