Add support for specifying logging configuration for kubelet (#1173)

* chore: add container logs maxsize with tests

* Add support for configuring logging configuration for kubelet

Signed-off-by: Waleed Malik <[email protected]>

* Refactored code

Signed-off-by: Waleed Malik <[email protected]>

* Use const for ContainerLogMaxSize default value

Signed-off-by: Waleed Malik <[email protected]>

* Use const for ContainerLogMaxSize default value

Signed-off-by: Waleed Malik <[email protected]>

Co-authored-by: Waleed Malik <[email protected]>

Author: nerdeveloper

pkg/apis/cluster/common/consts.go

@@ -135,9 +135,11 @@ const (
 )
 
 const (
-	SystemReservedKubeletConfig = "SystemReserved"
-	KubeReservedKubeletConfig   = "KubeReserved"
-	EvictionHardKubeletConfig   = "EvictionHard"
+	SystemReservedKubeletConfig       = "SystemReserved"
+	KubeReservedKubeletConfig         = "KubeReserved"
+	EvictionHardKubeletConfig         = "EvictionHard"
+	ContainerLogMaxSizeKubeletConfig  = "ContainerLogMaxSize"
+	ContainerLogMaxFilesKubeletConfig = "ContainerLogMaxFiles"
 )
 
 const (

pkg/containerruntime/containerruntime.go

@@ -78,12 +78,14 @@ func get(containerRuntimeName string, opts ...Opt) Config {
 }
 
 type Config struct {
-	Docker              *Docker               `json:",omitempty"`
-	Containerd          *Containerd           `json:",omitempty"`
-	InsecureRegistries  []string              `json:",omitempty"`
-	RegistryMirrors     map[string][]string   `json:",omitempty"`
-	RegistryCredentials map[string]AuthConfig `json:",omitempty"`
-	SandboxImage        string                `json:",omitempty"`
+	Docker               *Docker               `json:",omitempty"`
+	Containerd           *Containerd           `json:",omitempty"`
+	InsecureRegistries   []string              `json:",omitempty"`
+	RegistryMirrors      map[string][]string   `json:",omitempty"`
+	RegistryCredentials  map[string]AuthConfig `json:",omitempty"`
+	SandboxImage         string                `json:",omitempty"`
+	ContainerLogMaxFiles string                `json:",omitempty"`
+	ContainerLogMaxSize  string                `json:",omitempty"`
 }
 
 func (cfg Config) String() string {
@@ -99,8 +101,10 @@ func (cfg Config) String() string {
 
 func (cfg Config) Engine(kubeletVersion *semver.Version) Engine {
 	docker := &Docker{
-		insecureRegistries: cfg.InsecureRegistries,
-		registryMirrors:    cfg.RegistryMirrors["docker.io"],
+		insecureRegistries:   cfg.InsecureRegistries,
+		registryMirrors:      cfg.RegistryMirrors["docker.io"],
+		containerLogMaxFiles: cfg.ContainerLogMaxFiles,
+		containerLogMaxSize:  cfg.ContainerLogMaxSize,
 	}
 
 	containerd := &Containerd{

pkg/containerruntime/docker.go

@@ -31,12 +31,14 @@ const (
 )
 
 type Docker struct {
-	insecureRegistries []string
-	registryMirrors    []string
+	insecureRegistries   []string
+	registryMirrors      []string
+	containerLogMaxFiles string
+	containerLogMaxSize  string
 }
 
 func (eng *Docker) Config() (string, error) {
-	return helper.DockerConfig(eng.insecureRegistries, eng.registryMirrors)
+	return helper.DockerConfig(eng.insecureRegistries, eng.registryMirrors, eng.containerLogMaxFiles, eng.containerLogMaxSize)
 }
 
 func (eng *Docker) ConfigFileName() string {

pkg/controller/machine/machine_controller.go

@@ -717,7 +717,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 
 			// grab kubelet general options from the annotations
 			kubeletFlags := common.GetKubeletFlags(machine.GetAnnotations())
-			KubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations())
+			kubeletConfigs := common.GetKubeletConfigs(machine.GetAnnotations())
 
 			// look up for ExternalCloudProvider feature, with fallback to command-line input
 			externalCloudProvider := r.nodeSettings.ExternalCloudProvider
@@ -733,6 +733,14 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 			crRuntime := r.nodeSettings.ContainerRuntime
 			crRuntime.RegistryCredentials = registryCredentials
 
+			if val, ok := kubeletConfigs[common.ContainerLogMaxSizeKubeletConfig]; ok {
+				crRuntime.ContainerLogMaxSize = val
+			}
+
+			if val, ok := kubeletConfigs[common.ContainerLogMaxFilesKubeletConfig]; ok {
+				crRuntime.ContainerLogMaxFiles = val
+			}
+
 			req := plugin.UserDataRequest{
 				MachineSpec:              machine.Spec,
 				Kubeconfig:               kubeconfig,
@@ -743,7 +751,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 				PauseImage:               r.nodeSettings.PauseImage,
 				KubeletCloudProviderName: kubeletCloudProviderName,
 				KubeletFeatureGates:      kubeletFeatureGates,
-				KubeletConfigs:           KubeletConfigs,
+				KubeletConfigs:           kubeletConfigs,
 				NoProxy:                  r.nodeSettings.NoProxy,
 				HTTPProxy:                r.nodeSettings.HTTPProxy,
 				ContainerRuntime:         crRuntime,
@@ -760,8 +768,9 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 					return nil, fmt.Errorf("failed to find machine's MachineDployment: %v", err)
 				}
 
-				cloudConfigSecretName := fmt.Sprintf("%s-%s",
+				cloudConfigSecretName := fmt.Sprintf("%s-%s-%s",
 					referencedMachineDeployment,
+					machine.Namespace,
 					provisioningSuffix)
 
 				// It is important to check if the secret holding cloud-config exists

pkg/userdata/amzn2/provider.go

@@ -265,7 +265,7 @@ write_files:
 
 - path: "/etc/kubernetes/kubelet.conf"
   content: |
-{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs | indent 4 }}
+{{ kubeletConfiguration "cluster.local" .DNSIPs .KubeletFeatureGates .KubeletConfigs .ContainerRuntimeName | indent 4 }}
 
 - path: "/etc/kubernetes/pki/ca.crt"
   content: |

pkg/userdata/amzn2/testdata/containerd-kubelet-v1.20-aws.yaml

@@ -299,6 +299,7 @@ write_files:
         cacheUnauthorizedTTL: 0s
     cgroupDriver: systemd
     clusterDomain: cluster.local
+    containerLogMaxSize: 100Mi
     cpuManagerReconcilePeriod: 0s
     evictionHard:
       imagefs.available: 15%

pkg/userdata/amzn2/testdata/kubelet-v1.20-aws.yaml

@@ -296,6 +296,7 @@ write_files:
         cacheUnauthorizedTTL: 0s
     cgroupDriver: systemd
     clusterDomain: cluster.local
+    containerLogMaxSize: 100Mi
     cpuManagerReconcilePeriod: 0s
     evictionHard:
       imagefs.available: 15%
@@ -397,7 +398,7 @@ write_files:
 - path: /etc/docker/daemon.json
   permissions: "0644"
   content: |
-    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}}
+    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}}
 
 - path: /etc/systemd/system/kubelet-healthcheck.service
   permissions: "0644"

pkg/userdata/amzn2/testdata/kubelet-v1.21-aws-external.yaml

@@ -296,6 +296,7 @@ write_files:
         cacheUnauthorizedTTL: 0s
     cgroupDriver: systemd
     clusterDomain: cluster.local
+    containerLogMaxSize: 100Mi
     cpuManagerReconcilePeriod: 0s
     evictionHard:
       imagefs.available: 15%
@@ -397,7 +398,7 @@ write_files:
 - path: /etc/docker/daemon.json
   permissions: "0644"
   content: |
-    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}}
+    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}}
 
 - path: /etc/systemd/system/kubelet-healthcheck.service
   permissions: "0644"

pkg/userdata/amzn2/testdata/kubelet-v1.21-aws.yaml

@@ -296,6 +296,7 @@ write_files:
         cacheUnauthorizedTTL: 0s
     cgroupDriver: systemd
     clusterDomain: cluster.local
+    containerLogMaxSize: 100Mi
     cpuManagerReconcilePeriod: 0s
     evictionHard:
       imagefs.available: 15%
@@ -397,7 +398,7 @@ write_files:
 - path: /etc/docker/daemon.json
   permissions: "0644"
   content: |
-    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"}}
+    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"}}
 
 - path: /etc/systemd/system/kubelet-healthcheck.service
   permissions: "0644"

pkg/userdata/amzn2/testdata/kubelet-v1.21-vsphere-mirrors.yaml

@@ -313,6 +313,7 @@ write_files:
         cacheUnauthorizedTTL: 0s
     cgroupDriver: systemd
     clusterDomain: cluster.local
+    containerLogMaxSize: 100Mi
     cpuManagerReconcilePeriod: 0s
     evictionHard:
       imagefs.available: 15%
@@ -414,7 +415,7 @@ write_files:
 - path: /etc/docker/daemon.json
   permissions: "0644"
   content: |
-    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"10m"},"registry-mirrors":["https://registry.docker-cn.com"]}
+    {"exec-opts":["native.cgroupdriver=systemd"],"storage-driver":"overlay2","log-driver":"json-file","log-opts":{"max-file":"5","max-size":"100m"},"registry-mirrors":["https://registry.docker-cn.com"]}
 
 - path: /etc/systemd/system/kubelet-healthcheck.service
   permissions: "0644"