ci: Add check for cilium error logs to cilium nightly pipeline (#3592)

* ci: create log check template

* ci: add log check to cilium nightly

* chore: clean up spacing.

* ci: move log check, address nit

Author: jpayne3506

.pipelines/cni/cilium/nightly-release-test.yml

@@ -144,12 +144,37 @@ stages:
           service: true
 
       - job: logs
-        displayName: "Failure Logs"
+        displayName: "Nightly Logs"
         dependsOn:
           - cilium_nightly
           - cni_linux
-        condition: failed()
+        condition: always()
         steps:
+          - template: ../../templates/cilium-cli.yaml
+
+          - task: AzureCLI@2
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              scriptType: "bash"
+              addSpnToEnvironment: true
+              inlineScript: |
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+
+                set -e
+                echo "Run Cilium Connectivity Tests"
+                cilium status
+                cilium connectivity test --test check-log-errors --log-check-levels error
+              name: "ciliumConnectivityTests"
+              displayName: "Run Cilium Connectivity Tests"
+              condition: always()
+
+          - template: ../../templates/log-check-template.yaml # Operator Check
+            parameters:
+              clusterName: $(clusterName)-$(commitID)
+              podLabel: "name=cilium-operator"
+              logGrep: "level=error"
+
           - template: ../../templates/log-template.yaml
             parameters:
               clusterName: ciliumnightly-$(commitID)

.pipelines/templates/log-check-template.yaml

@@ -0,0 +1,23 @@
+parameters:
+  clusterName: ""
+  podLabel: ""
+  logGrep: ""
+
+steps:
+  - task: AzureCLI@2
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}
+
+        if ! [ -z "$(kubectl logs -n kube-system -l ${{ parameters.podLabel }} --tail=-1 | grep ${{ parameters.logGrep }})" ]; then
+          kubectl logs -n kube-system -l ${{ parameters.podLabel }} --tail=-1 | grep ${{ parameters.logGrep }}
+          echo "Logs found with ${{ parameters.logGrep }}"
+          exit 1
+        fi
+    # Leaving "name:" blank as this template could be called multiple times in a single job with the same parameters.
+    displayName: "Check pod with ${{ parameters.podLabel }} label for ${{ parameters.logGrep }}"
+    condition: always()