output kube-system logs from workload clusters

devigned · devigned · commit ae02111be7c2 · 2021-01-19T15:47:41.000-05:00
diff --git a/templates/test/cluster-template-prow-ci-version.yaml b/templates/test/cluster-template-prow-ci-version.yaml
@@ -63,6 +63,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-ipv6.yaml b/templates/test/cluster-template-prow-ipv6.yaml
@@ -84,6 +84,7 @@ spec:
           cluster-cidr: 2001:1234:5678:9a40::/58
           cluster-name: ${CLUSTER_NAME}
           configure-cloud-routes: "true"
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-machine-pool-ci-version.yaml b/templates/test/cluster-template-prow-machine-pool-ci-version.yaml
@@ -63,6 +63,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-machine-pool-windows.yaml b/templates/test/cluster-template-prow-machine-pool-windows.yaml
@@ -64,6 +64,7 @@ spec:
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
           configure-cloud-routes: "false"
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-machine-pool.yaml b/templates/test/cluster-template-prow-machine-pool.yaml
@@ -63,6 +63,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-multi-tenancy.yaml b/templates/test/cluster-template-prow-multi-tenancy.yaml
@@ -68,6 +68,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-nvidia-gpu.yaml b/templates/test/cluster-template-prow-nvidia-gpu.yaml
@@ -63,6 +63,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-private.yaml b/templates/test/cluster-template-prow-private.yaml
@@ -78,6 +78,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow-windows.yaml b/templates/test/cluster-template-prow-windows.yaml
@@ -64,6 +64,7 @@ spec:
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
           configure-cloud-routes: "false"
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/cluster-template-prow.yaml b/templates/test/cluster-template-prow.yaml
@@ -63,6 +63,7 @@ spec:
           cloud-config: /etc/kubernetes/azure.json
           cloud-provider: azure
           cluster-name: ${CLUSTER_NAME}
+          v: "4"
         extraVolumes:
         - hostPath: /etc/kubernetes/azure.json
           mountPath: /etc/kubernetes/azure.json
diff --git a/templates/test/patches/controller-manager.yaml b/templates/test/patches/controller-manager.yaml
@@ -0,0 +1,10 @@
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha3
+metadata:
+  name: "${CLUSTER_NAME}-control-plane"
+spec:
+  kubeadmConfigSpec:
+    clusterConfiguration:
+      controllerManager:
+        extraArgs:
+          v: "4"
diff --git a/templates/test/prow-ipv6/kustomization.yaml b/templates/test/prow-ipv6/kustomization.yaml
@@ -6,5 +6,6 @@ resources:
   - cni-resource-set.yaml
 patchesStrategicMerge:
   - ../patches/tags.yaml
+  - ../patches/controller-manager.yaml
   - patches/cluster-cni.yaml
 
diff --git a/templates/test/prow-machine-pool-ci-version/kustomization.yaml b/templates/test/prow-machine-pool-ci-version/kustomization.yaml
@@ -5,4 +5,5 @@ resources:
   - ../prow-machine-pool
 patchesStrategicMerge:
   - ../patches/control-plane-image-ci-version.yaml
+  - ../patches/controller-manager.yaml
   - patches/machine-pool-ci-version.yaml
diff --git a/templates/test/prow-machine-pool-windows/kustomization.yaml b/templates/test/prow-machine-pool-windows/kustomization.yaml
@@ -7,4 +7,5 @@ resources:
 patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cni-resource-set.yaml
+  - ../patches/controller-manager.yaml
 
diff --git a/templates/test/prow-machine-pool/kustomization.yaml b/templates/test/prow-machine-pool/kustomization.yaml
@@ -7,3 +7,4 @@ resources:
 patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cni-resource-set.yaml
+  - ../patches/controller-manager.yaml
diff --git a/templates/test/prow-multi-tenancy/kustomization.yaml b/templates/test/prow-multi-tenancy/kustomization.yaml
@@ -6,4 +6,5 @@ resources:
   - cni-resource-set.yaml
 patchesStrategicMerge:
   - ../patches/tags.yaml
+  - ../patches/controller-manager.yaml
   - patches/cluster-cni.yaml
diff --git a/templates/test/prow-nvidia-gpu/kustomization.yaml b/templates/test/prow-nvidia-gpu/kustomization.yaml
@@ -7,6 +7,7 @@ resources:
 patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cni-resource-set.yaml
+  - ../patches/controller-manager.yaml
 patchesJson6902:
 - path: patches/node-storage-type.yaml
   target:
diff --git a/templates/test/prow-private/kustomization.yaml b/templates/test/prow-private/kustomization.yaml
@@ -6,6 +6,7 @@ resources:
   - cni-resource-set.yaml
 patchesStrategicMerge:
   - ../patches/tags.yaml
+  - ../patches/controller-manager.yaml
   - patches/cluster-cni.yaml
   - patches/custom-vnet.yaml
 
diff --git a/templates/test/prow-windows/kustomization.yaml b/templates/test/prow-windows/kustomization.yaml
@@ -7,3 +7,4 @@ resources:
 patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/cni-resource-set.yaml
+  - ../patches/controller-manager.yaml
diff --git a/templates/test/prow/kustomization.yaml b/templates/test/prow/kustomization.yaml
@@ -9,3 +9,4 @@ patchesStrategicMerge:
   - ../patches/tags.yaml
   - ../patches/mhc.yaml
   - ../patches/cni-resource-set.yaml
+  - ../patches/controller-manager.yaml
diff --git a/test/e2e/azure_lb.go b/test/e2e/azure_lb.go
@@ -27,15 +27,15 @@ import (
 	. "github.com/onsi/gomega"
 	k8snet "k8s.io/utils/net"
 
-	deploymentBuilder "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/deployment"
-	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/job"
-	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/node"
-	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/windows"
-
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
 	"sigs.k8s.io/cluster-api/test/framework"
+
+	deploymentBuilder "sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/deployment"
+	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/job"
+	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/node"
+	"sigs.k8s.io/cluster-api-provider-azure/test/e2e/kubernetes/windows"
 )
 
 // AzureLBSpecInput is the input for AzureLBSpec.
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -19,10 +19,13 @@ limitations under the License.
 package e2e
 
 import (
+	"bufio"
 	"context"
 	"flag"
 	"fmt"
+	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -32,14 +35,17 @@ import (
 	"github.com/onsi/ginkgo/config"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 	capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework/bootstrap"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1alpha3"
 )
 
 // Test suite flags
@@ -80,6 +86,71 @@ var (
 	useCIArtifacts bool
 )
 
+type (
+	AzureClusterProxy struct {
+		framework.ClusterProxy
+	}
+)
+
+func NewAzureClusterProxy(name string, kubeconfigPath string, scheme *runtime.Scheme, options ...framework.Option) *AzureClusterProxy {
+	return &AzureClusterProxy{
+		ClusterProxy: framework.NewClusterProxy(name, kubeconfigPath, scheme, options...),
+	}
+}
+
+func (acp *AzureClusterProxy) CollectWorkloadClusterLogs(ctx context.Context, namespace, name, outputPath string) {
+	const (
+		kubesystem = "kube-system"
+	)
+
+	Byf("Dumping workload cluster %s/%s logs", namespace, name)
+	acp.ClusterProxy.CollectWorkloadClusterLogs(ctx, namespace, name, outputPath)
+
+	Byf("Dumping workload cluster %s/%s kube-system pod logs", namespace, name)
+	aboveMachinesPath := strings.Replace(outputPath, "/machines", "", 1)
+	workload := acp.GetWorkloadCluster(ctx, namespace, name)
+	pods := &corev1.PodList{}
+	Expect(workload.GetClient().List(ctx, pods, client.InNamespace(kubesystem))).To(Succeed())
+
+	for _, pod := range pods.Items {
+		for _, container := range pod.Spec.Containers {
+			// Watch each container's logs in a goroutine so we can stream them all concurrently.
+			go func(pod corev1.Pod, container corev1.Container) {
+				defer GinkgoRecover()
+
+				Byf("Creating log watcher for controller %s/%s, container %s", kubesystem, pod.Name, container.Name)
+				logFile := path.Join(aboveMachinesPath, kubesystem, pod.Name, container.Name+".log")
+				Expect(os.MkdirAll(filepath.Dir(logFile), 0755)).To(Succeed())
+
+				f, err := os.OpenFile(logFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+				Expect(err).NotTo(HaveOccurred())
+				defer f.Close()
+
+				opts := &corev1.PodLogOptions{
+					Container: container.Name,
+					Follow:    true,
+				}
+
+				podLogs, err := workload.GetClientSet().CoreV1().Pods(kubesystem).GetLogs(pod.Name, opts).Stream()
+				if err != nil {
+					// Failing to stream logs should not cause the test to fail
+					Byf("Error starting logs stream for pod %s/%s, container %s: %v", kubesystem, pod.Name, container.Name, err)
+					return
+				}
+				defer podLogs.Close()
+
+				out := bufio.NewWriter(f)
+				defer out.Flush()
+				_, err = out.ReadFrom(podLogs)
+				if err != nil && err != io.ErrUnexpectedEOF {
+					// Failing to stream logs should not cause the test to fail
+					Byf("Got error while streaming logs for pod %s/%s, container %s: %v", kubesystem, pod.Name, container.Name, err)
+				}
+			}(pod, container)
+		}
+	}
+}
+
 func init() {
 	flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file")
 	flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "", "folder where e2e test artifact should be stored")
@@ -140,7 +211,7 @@ var _ = SynchronizedBeforeSuite(func() []byte {
 	kubeconfigPath := parts[3]
 
 	e2eConfig = loadE2EConfig(configPath)
-	bootstrapClusterProxy = framework.NewClusterProxy("bootstrap", kubeconfigPath, initScheme(),
+	bootstrapClusterProxy = NewAzureClusterProxy("bootstrap", kubeconfigPath, initScheme(),
 		framework.WithMachineLogCollector(AzureLogCollector{}))
 })
 
@@ -231,7 +302,7 @@ func setupBootstrapCluster(config *clusterctl.E2EConfig, scheme *runtime.Scheme,
 		Expect(kubeconfigPath).To(BeAnExistingFile(), "Failed to get the kubeconfig file for the bootstrap cluster")
 	}
 
-	clusterProxy := framework.NewClusterProxy("bootstrap", kubeconfigPath, scheme)
+	clusterProxy := NewAzureClusterProxy("bootstrap", kubeconfigPath, scheme)
 	Expect(clusterProxy).ToNot(BeNil(), "Failed to get a bootstrap cluster proxy")
 
 	return clusterProvider, clusterProxy
