Merge pull request #5663 from nojnhuh/apiserver-logs

Collect kube-apiserver logs from node

Author: k8s-ci-robot

hack/create-dev-cluster.sh

@@ -66,7 +66,7 @@ capz::util::generate_ssh_key
 echo "================ DOCKER BUILD ==============="
 PULL_POLICY=IfNotPresent make modules docker-build
 
-setup() {
+create_cluster() {
     echo "================ MAKE CLEAN ==============="
     make clean
 
@@ -75,17 +75,14 @@ setup() {
 
     echo "================ INSTALL TOOLS ==============="
     make install-tools
-}
 
-create_cluster() {
     echo "================ CREATE CLUSTER ==============="
     make create-cluster
 }
 
 retries=$CLUSTER_CREATE_ATTEMPTS
-while ((retries > 0)); do
-    setup
-    create_cluster && break
-    sleep 5
-    ((retries --))
+until create_cluster; do
+  if ((--retries == 0)); then
+    exit 1
+  fi
 done

test/e2e/azure_logcollector.go

@@ -308,6 +308,12 @@ func linuxLogs(execToPathFn func(outputFileName string, command string, args ...
 			"cni.log",
 			"cat", "/var/log/calico/cni/cni.log",
 		),
+		// If kube-apiserver fails to come up, its logs aren't accessible via `kubectl logs`.
+		// Grab them from the node instead.
+		execToPathFn(
+			"kube-apiserver.log",
+			crictlPodLogsCmd("kube-apiserver"),
+		),
 	}
 }
 
@@ -524,3 +530,13 @@ func writeBootLog(bootDiagnostics armcompute.RetrieveBootDiagnosticsDataResult,
 
 	return nil
 }
+
+func crictlPodLogsCmd(podNamePattern string) string {
+	//nolint: dupword // this is bash, not english
+	return `sudo crictl pods --name "` + podNamePattern + `" -o json | jq -c '.items[]' | while read -r pod; do
+    sudo crictl ps -a --pod $(jq -r .id <<< $pod) -o json | jq -c '.containers[]' | while read -r ctr; do
+        echo "========= Pod $(jq -r .metadata.name <<< $pod), container $(jq -r .metadata.name <<< $ctr) ========="
+        sudo crictl logs "$(jq -r .id <<< $ctr)" 2>&1
+    done
+done`
+}