Merge pull request #1238 from consideRatio/pr/full_namespace_report-update

ci: sync full_namespace_report from z2jh

Author: consideRatio

.github/workflows/test.yml

@@ -160,9 +160,10 @@ jobs:
       - name: Kubernetes namespace report
         if: ${{ always() }}
         run: |
-          # Display debugging information
+          # Display debugging information and always provide the logs from
+          # certain important k8s deployments.
           . ci/common
-          full_namespace_report
+          full_namespace_report deploy/binder deploy/hub deploy/proxy
       - name: Upload coverage stats
         uses: codecov/codecov-action@v1
         if: ${{ always() }}

ci/common

@@ -22,34 +22,128 @@ await_binderhub() {
     kubectl rollout status --watch --timeout 300s deployment/binder
 }
 
-full_namespace_report() {
-    # list config (secret,configmap)
-    kubectl get secret,cm
-    # list networking (service,ingress)
-    kubectl get svc,ing
+full_namespace_report () {
+    # This was copied from z2jh 2021-01-06. Work to make it a dedicated GitHub
+    # action and avoid a duplicated code base is planned. / @consideRatio
+    # ------------------------------------------------------------------------
+    #
+    # Purpose:
+    # - To chart agnostically print relevant information of the resources in a
+    #   namespace.
+    #
+    # Arguments:
+    # - Accepts a sequence of arguments such as "deploy/hub" "deploy/proxy". It
+    #   will do `kubectl logs --all-containers <arg>` on them.
+    #
+    # Relevant references:
+    # - ContainerStatus ref: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#containerstatus-v1-core
+
+    # printf formatting: a bold bold colored topic with a divider.
+    yellow=33
+    red=31
+    divider='--------------------------------------------------------------------------------'
+    # GitHub Workflows resets formatting after \n, so its reapplied.
+    export format="\n\033[${yellow};1m%s\n\033[${yellow};1m${divider}\033[0m\n"
+    export format_error="\n\033[${red};1m%s\n\033[${red};1m${divider}\033[0m\n"
+
+    printf "$format" "# Full namespace report"
+    printf "$format" "## Resource overview"
     # list workloads (deployment,statefulset,daemonset,pod)
+    printf "$format" "### \$ kubectl get deploy,sts,ds,pod"
     kubectl get deploy,sts,ds,pod
+    # list config (secret,configmap)
+    printf "$format" "### \$ kubectl get secret,cm"
+    kubectl get secret,cm
+    # list networking (service,ingress,networkpolicy)
+    printf "$format" "### \$ kubectl get svc,ing,netpol"
+    kubectl get svc,ing,netpol
+    # list rbac (serviceaccount,role,rolebinding)
+    printf "$format" "### \$ kubectl get sa,role,rolebinding"
+    kubectl get sa,role,rolebinding
+
+    # Check if any container of any pod has a non ready status
+    PODS_WITH_NON_READY_CONTAINER=$(
+        kubectl get pods -o json \
+            | jq -r '
+                .items[]
+                | select(
+                    any(.status.initContainerStatuses[]?; .ready == false)
+                    or
+                    any(.status.containerStatuses[]?; .ready == false)
+                )
+                | .metadata.name
+        '
+    )
+    if [ -n "$PODS_WITH_NON_READY_CONTAINER" ]; then
+        printf "$format_error" "## Pods with non-ready container(s) detected!"
+        echo "$PODS_WITH_NON_READY_CONTAINER" | xargs --max-args=1 echo -
 
-    # if any pod has any non-ready -> show its containers' logs
-    kubectl get pods -o json \
-    | jq '
-        .items[]
-        | select(
-            any(.status.containerStatuses[]?; .ready == false)
-        )
-        | .metadata.name' \
-    | xargs --max-args 1 --no-run-if-empty \
-    sh -c 'printf "\nPod with non-ready container detected\n - Logs of $0:\n"; kubectl logs --all-containers $0'
+        for var in $PODS_WITH_NON_READY_CONTAINER; do
+            printf "$format_error" "### \$ kubectl describe pod/$var"
+            kubectl describe pod/$var
+            printf "$format_error" "### \$ kubectl logs --all-containers pod/$var"
+            kubectl logs --all-containers pod/$var || echo  # a newline on failure for consistency with non-failure
+        done
+
+    fi
+
+    # Check if any container of any pod has a restartCount > 0. Then, we inspect
+    # their logs with --previous. We also add --follow and --ignore-errors in
+    # order to ensure we get the information from all containers, and combined
+    # with --previous it will exit and not get stuck.
+    #
+    # ref: https://github.com/kubernetes/kubernetes/issues/97530
+    PODS_WITH_RESTARTED_CONTAINERS=$(
+        kubectl get pods -o json \
+            | jq -r '
+                .items[]
+                | select(
+                    any(.status.initContainerStatuses[]?; .restartCount > 0)
+                    or
+                    any(.status.containerStatuses[]?; .restartCount > 0)
+                )
+                | .metadata.name
+        '
+    )
+    if [ -n "$PODS_WITH_RESTARTED_CONTAINERS" ]; then
+        printf "$format_error" "## Pods with restarted containers detected!"
+        echo "$PODS_WITH_RESTARTED_CONTAINERS" | xargs --max-args=1 echo -
+
+        for var in $PODS_WITH_RESTARTED_CONTAINERS; do
+            printf "$format_error" "### \$ kubectl describe pod/$var"
+            kubectl describe pod/$var
+            printf "$format_error" "### \$ kubectl logs --previous --all-containers --follow --ignore-errors pod/$var"
+            kubectl logs --previous --all-containers --follow --ignore-errors pod/$var
+        done
+    fi
 
     # if any pods that should be scheduled by the user-scheduler are pending ->
     # show user-scheduler's logs
-    (
+    PENDING_USER_PODS=$(
         kubectl get pods -l "component in (user-placeholder,singleuser-server)" -o json \
-        | jq -r '
-            .items[]
-            | select(.status.phase == "Pending")
-            | .metadata.name
+            | jq -r '
+                .items[]
+                | select(.status.phase == "Pending")
+                | .metadata.name
         '
-    ) | xargs --max-args 1 --no-run-if-empty --max-lines \
-    sh -c 'printf "\nPending user pod detected ($0)\n - Logs of deploy/user-scheduler:\n"; kubectl logs --all-containers deploy/user-scheduler'
+    )
+    if [ -n "$PENDING_USER_PODS" ]; then
+        printf "$format_error" "## Pending pods detected!"
+        echo "$PENDING_USER_PODS" | xargs --max-args=1 echo -
+
+        printf "$format_error" "### \$ kubectl logs --all-containers deploy/user-scheduler"
+        kubectl logs --all-containers deploy/user-scheduler || echo  # a newline on failure for consistency with non-failure
+    fi
+
+    # show container logs of all important workloads passed to the function,
+    # "deploy/hub" and "deploy/proxy" for example.
+    if [ "$#" -gt 0 ]; then
+        printf "$format" "## Important workload's logs"
+        echo "$@" | xargs --max-args=1 echo -
+
+        for var in "$@"; do
+            printf "$format" "### \$ kubectl logs --all-containers $var"
+            kubectl logs --all-containers $var || echo  # a newline on failure for consistency with non-failure
+        done
+    fi
 }