New -node-kubelet-repository flag (default: quay.io/poseidon/kubelet) (#801)

on Flatcar Linux:
* -node-hyperkube-image is used only for kubelet version < 1.18
* replace RKT / kubelet-wrapper usage with docker

on CoreOS ContainerLinux:
* no changes

Signed-off-by: Artiom Diomin <[email protected]>

Author: kron4eg

.gitignore

@@ -1,5 +1,5 @@
 .idea
-machine-controller*
+/machine-controller*
 .terraform
 terraform.tfstate
 terraform.tfstate.backup

Makefile

@@ -31,7 +31,7 @@ LDFLAGS ?= -ldflags '-s -w'
 
 IMAGE_TAG = \
 		$(shell echo $$(git rev-parse HEAD && if [[ -n $$(git status --porcelain) ]]; then echo '-dirty'; fi)|tr -d ' ')
-IMAGE_NAME = $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG)
+IMAGE_NAME ?= $(REGISTRY)/$(REGISTRY_NAMESPACE)/machine-controller:$(IMAGE_TAG)
 
 OS = centos coreos ubuntu sles rhel flatcar
 USERDATA_BIN = $(patsubst %, machine-controller-userdata-%, $(OS))

cmd/machine-controller/main.go

@@ -82,6 +82,7 @@ var (
 	nodeRegistryMirrors    string
 	nodePauseImage         string
 	nodeHyperkubeImage     string
+	nodeKubeletRepository  string
 )
 
 const (
@@ -165,7 +166,8 @@ func main() {
 	flag.StringVar(&nodeInsecureRegistries, "node-insecure-registries", "", "Comma separated list of registries which should be configured as insecure on the container runtime")
 	flag.StringVar(&nodeRegistryMirrors, "node-registry-mirrors", "", "Comma separated list of Docker image mirrors")
 	flag.StringVar(&nodePauseImage, "node-pause-image", "", "Image for the pause container including tag. If not set, the kubelet default will be used: https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/")
-	flag.StringVar(&nodeHyperkubeImage, "node-hyperkube-image", "k8s.gcr.io/hyperkube-amd64", "Image for the hyperkube container excluding tag.")
+	flag.StringVar(&nodeHyperkubeImage, "node-hyperkube-image", "k8s.gcr.io/hyperkube-amd64", "Image for the hyperkube container excluding tag. Only has effect on CoreOS Container Linux and Flatcar Linux, and for kubernetes < 1.18.")
+	flag.StringVar(&nodeKubeletRepository, "node-kubelet-repository", "quay.io/poseidon/kubelet", "Repository for the kubelet container. Only has effect on Flatcar Linux, and for kubernetes >= 1.18.")
 	flag.BoolVar(&nodeCSRApprover, "node-csr-approver", false, "Enable NodeCSRApprover controller to automatically approve node serving certificate requests.")
 
 	flag.Parse()
@@ -198,13 +200,23 @@ func main() {
 	if err := clusterv1alpha1.AddToScheme(scheme.Scheme); err != nil {
 		klog.Fatalf("failed to add clusterv1alpha1 api to scheme: %v", err)
 	}
+
 	// Check if the hyperkube image has a tag set
 	hyperkubeImageRef, err := reference.Parse(nodeHyperkubeImage)
 	if err != nil {
-		klog.Fatalf("failed to parse --node-hyperkube-image %s: %v", nodeHyperkubeImage, err)
+		klog.Fatalf("failed to parse -node-hyperkube-image %s: %v", nodeHyperkubeImage, err)
 	}
 	if _, ok := hyperkubeImageRef.(reference.NamedTagged); ok {
-		klog.Fatalf("--node-hyperkube-image must not contain a tag. The tag will be dynamically set for each Machine.")
+		klog.Fatalf("-node-hyperkube-image must not contain a tag. The tag will be dynamically set for each Machine.")
+	}
+
+	// Check if the kubelet image has a tag set
+	kubeletRepoRef, err := reference.Parse(nodeKubeletRepository)
+	if err != nil {
+		klog.Fatalf("failed to parse -node-hyperkube-image %s: %v", nodeHyperkubeImage, err)
+	}
+	if _, ok := kubeletRepoRef.(reference.NamedTagged); ok {
+		klog.Fatalf("-node-kubelet-image must not contain a tag. The tag will be dynamically set for each Machine.")
 	}
 
 	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
@@ -246,11 +258,12 @@ func main() {
 		skipEvictionAfter:     skipEvictionAfter,
 		nodeCSRApprover:       nodeCSRApprover,
 		node: machinecontroller.NodeSettings{
-			ClusterDNSIPs:  clusterDNSIPs,
-			HTTPProxy:      nodeHTTPProxy,
-			NoProxy:        nodeNoProxy,
-			HyperkubeImage: nodeHyperkubeImage,
-			PauseImage:     nodePauseImage,
+			ClusterDNSIPs:     clusterDNSIPs,
+			HTTPProxy:         nodeHTTPProxy,
+			NoProxy:           nodeNoProxy,
+			HyperkubeImage:    nodeHyperkubeImage,
+			KubeletRepository: nodeKubeletRepository,
+			PauseImage:        nodePauseImage,
 		},
 	}
 	if parsedJoinClusterTimeout != nil {

docs/network-restrictions.md

@@ -27,16 +27,35 @@ If that image won't be accessible from the node, a custom image can be specified
 -node-pause-image="192.168.1.1:5000/kubernetes/pause:3.1"
 ```
 
-For ContainerLinux nodes the [hyperkube](https://github.com/kubernetes/kubernetes/tree/master/cluster/images/hyperkube) image must be accessible as well.
-This is due to the usage of the [kubelet-wrapper](https://github.com/coreos/coreos-kubernetes/blob/master/Documentation/kubelet-wrapper.md).
 
-By default the image `k8s.gcr.io/hyperkube-amd64` will be used.
-If that image won't be accessible from the node, a custom image can be specified on the machine-controller:
+## Kubelet images
+
+### CoreOS ContainerLinux
+For ContainerLinux nodes, the [hyperkube][1] image must be accessible as well. This is due to the usage of the
+[kubelet-wrapper][2].
+
+By default the image `k8s.gcr.io/hyperkube-amd64` will be used. If that image won't be accessible from the node, a
+custom image can be specified on the machine-controller:
+```bash
+# Do not set a tag. The tag depends on the used Kubernetes version of a machine.
+# Example:
+# A Node using v1.14.2 would use 192.168.1.1:5000/kubernetes/hyperkube-amd64:v1.14.2
+-node-hyperkube-image="192.168.1.1:5000/kubernetes/hyperkube-amd64"
+```
+
+### Flatcar Linux
+For Flatcar Linux nodes, the [hyperkube][1] or [kubelet][3] image must be accessible as well. This is due to the fact
+that kubelet is running as a docker container. For kubelet version `< 1.18` hyperkube will be used, otherwise `kubelet`
+image.
+
+By default the image `quay.io/poseidon/kubelet` will be used. If that image won't be accessible from the node, a custom
+image can be specified on the machine-controller:
 ```bash
 # Do not set a tag. The tag depends on the used Kubernetes version of a machine.
 # Example:
 # A Node using v1.14.2 would use 192.168.1.1:5000/kubernetes/hyperkube-amd64:v1.14.2
 -node-hyperkube-image="192.168.1.1:5000/kubernetes/hyperkube-amd64"
+-node-kubelet-image="192.168.1.1:5000/my-custom/kubelet-amd64"
 ```
 
 # Insecure registries
@@ -45,3 +64,7 @@ If nodes require access to insecure registries, all registries must be specified
 ```bash
 --node-insecure-registries="192.168.1.1:5000,10.0.0.1:5000"
 ```
+
+[1]: https://console.cloud.google.com/gcr/images/google-containers/GLOBAL/hyperkube
+[2]: https://github.com/coreos/coreos-kubernetes/blob/master/Documentation/kubelet-wrapper.md
+[3]: https://quay.io/poseidon/kubelet

pkg/apis/plugin/plugin.go

@@ -52,6 +52,7 @@ type UserDataRequest struct {
 	RegistryMirrors       []string
 	PauseImage            string
 	HyperkubeImage        string
+	KubeletRepository     string
 }
 
 // UserDataResponse contains the responded user data.

pkg/controller/machine/machine.go

@@ -124,8 +124,10 @@ type NodeSettings struct {
 	RegistryMirrors []string
 	// Translates to --pod-infra-container-image on the kubelet. If not set, the kubelet will default it.
 	PauseImage string
-	// The hyperkube image to use. Currently only Container Linux uses it.
+	// The hyperkube image to use. Currently only Container Linux and Flatcar Linux uses it.
 	HyperkubeImage string
+	// The kubelet repository to use. Currently only Flatcar Linux uses it.
+	KubeletRepository string
 }
 
 type KubeconfigProvider interface {
@@ -666,6 +668,7 @@ func (r *Reconciler) ensureInstanceExistsForMachine(
 				RegistryMirrors:       r.nodeSettings.RegistryMirrors,
 				PauseImage:            r.nodeSettings.PauseImage,
 				HyperkubeImage:        r.nodeSettings.HyperkubeImage,
+				KubeletRepository:     r.nodeSettings.KubeletRepository,
 				NoProxy:               r.nodeSettings.NoProxy,
 				HTTPProxy:             r.nodeSettings.HTTPProxy,
 			}

pkg/userdata/flatcar/provider.go

@@ -23,7 +23,6 @@ package flatcar
 import (
 	"bytes"
 	"fmt"
-	"strings"
 	"text/template"
 
 	"github.com/Masterminds/semver"
@@ -33,12 +32,15 @@ import (
 	userdatahelper "github.com/kubermatic/machine-controller/pkg/userdata/helper"
 )
 
+const (
+	lessThen118Check = "< 1.18"
+)
+
 // Provider is a pkg/userdata/plugin.Provider implementation.
 type Provider struct{}
 
 // UserData renders user-data template to string.
 func (p Provider) UserData(req plugin.UserDataRequest) (string, error) {
-
 	tmpl, err := template.New("user-data").Funcs(userdatahelper.TxtFuncMap()).Parse(userDataTemplate)
 	if err != nil {
 		return "", fmt.Errorf("failed to parse user-data template: %v", err)
@@ -73,37 +75,40 @@ func (p Provider) UserData(req plugin.UserDataRequest) (string, error) {
 		return "", fmt.Errorf("error extracting cacert: %v", err)
 	}
 
-	// We need to reconfigure rkt to allow insecure registries in case the hyperkube image comes from an insecure registry
-	var insecureHyperkubeImage bool
-	for _, registry := range req.InsecureRegistries {
-		if strings.Contains(req.HyperkubeImage, registry) {
-			insecureHyperkubeImage = true
-		}
-	}
-
 	if flatcarConfig.DisableAutoUpdate {
 		flatcarConfig.DisableLocksmithD = true
 		flatcarConfig.DisableUpdateEngine = true
 	}
 
+	kubeletImage := req.KubeletRepository
+	lessThen118, err := semver.NewConstraint(lessThen118Check)
+	if err != nil {
+		return "", err
+	}
+
+	if lessThen118.Check(kubeletVersion) {
+		kubeletImage = req.HyperkubeImage
+	}
+	kubeletImage = kubeletImage + ":v" + kubeletVersion.String()
+
 	data := struct {
 		plugin.UserDataRequest
-		ProviderSpec           *providerconfigtypes.Config
-		FlatcarConfig          *Config
-		Kubeconfig             string
-		KubernetesCACert       string
-		KubeletVersion         string
-		InsecureHyperkubeImage bool
-		NodeIPScript           string
+		ProviderSpec     *providerconfigtypes.Config
+		FlatcarConfig    *Config
+		Kubeconfig       string
+		KubernetesCACert string
+		KubeletImage     string
+		KubeletVersion   string
+		NodeIPScript     string
 	}{
-		UserDataRequest:        req,
-		ProviderSpec:           pconfig,
-		FlatcarConfig:          flatcarConfig,
-		Kubeconfig:             kubeconfigString,
-		KubernetesCACert:       kubernetesCACert,
-		KubeletVersion:         kubeletVersion.String(),
-		InsecureHyperkubeImage: insecureHyperkubeImage,
-		NodeIPScript:           userdatahelper.SetupNodeIPEnvScript(),
+		UserDataRequest:  req,
+		ProviderSpec:     pconfig,
+		FlatcarConfig:    flatcarConfig,
+		Kubeconfig:       kubeconfigString,
+		KubernetesCACert: kubernetesCACert,
+		KubeletImage:     kubeletImage,
+		KubeletVersion:   kubeletVersion.String(),
+		NodeIPScript:     userdatahelper.SetupNodeIPEnvScript(),
 	}
 	b := &bytes.Buffer{}
 	err = tmpl.Execute(b, data)
@@ -164,7 +169,7 @@ systemd:
             Environment=ALL_PROXY={{ .HTTPProxy }}
 {{- end }}
 
-    - name: download-healthcheck-script.service
+    - name: download-script.service
       enabled: true
       contents: |
         [Unit]
@@ -183,8 +188,8 @@ systemd:
       - name: 40-docker.conf
         contents: |
           [Unit]
-          Requires=download-healthcheck-script.service
-          After=download-healthcheck-script.service
+          Requires=download-script.service
+          After=download-script.service
       contents: |
 {{ containerRuntimeHealthCheckSystemdUnit | indent 10 }}
 
@@ -194,8 +199,8 @@ systemd:
       - name: 40-docker.conf
         contents: |
           [Unit]
-          Requires=download-healthcheck-script.service
-          After=download-healthcheck-script.service
+          Requires=download-script.service
+          After=download-script.service
       contents: |
 {{ kubeletHealthCheckSystemdUnit | indent 10 }}
 
@@ -227,38 +232,42 @@ systemd:
         MemoryAccounting=true
         EnvironmentFile=-/etc/environment
         EnvironmentFile=/etc/kubernetes/nodeip.conf
-{{- if .HTTPProxy }}
-        Environment=KUBELET_IMAGE=docker://{{ .HyperkubeImage }}:v{{ .KubeletVersion }}
-{{- else }}
-        Environment=KUBELET_IMAGE=docker://k8s.gcr.io/hyperkube-amd64:v{{ .KubeletVersion }}
-{{- end }}
-        Environment="RKT_RUN_ARGS=--uuid-file-save=/var/cache/kubelet-pod.uuid \
-          --inherit-env \
-          --insecure-options=image{{if .InsecureHyperkubeImage }},http{{ end }} \
-          --volume=resolv,kind=host,source=/etc/resolv.conf \
-          --mount volume=resolv,target=/etc/resolv.conf \
-          --volume cni-bin,kind=host,source=/opt/cni/bin \
-          --mount volume=cni-bin,target=/opt/cni/bin \
-          --volume cni-conf,kind=host,source=/etc/cni/net.d \
-          --mount volume=cni-conf,target=/etc/cni/net.d \
-          --volume etc-kubernetes,kind=host,source=/etc/kubernetes \
-          --mount volume=etc-kubernetes,target=/etc/kubernetes \
-          --volume var-log,kind=host,source=/var/log \
-          --mount volume=var-log,target=/var/log \
-          --volume var-lib-calico,kind=host,source=/var/lib/calico \
-          --mount volume=var-lib-calico,target=/var/lib/calico"
+        Environment=PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:/opt/bin
         ExecStartPre=/bin/bash /opt/bin/setup_net_env.sh
         ExecStartPre=/bin/mkdir -p /var/lib/calico
         ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
         ExecStartPre=/bin/mkdir -p /etc/cni/net.d
         ExecStartPre=/bin/mkdir -p /opt/cni/bin
-        ExecStartPre=-/usr/bin/rkt rm --uuid-file=/var/cache/kubelet-pod.uuid
-        ExecStartPre=-/bin/rm -rf /var/lib/rkt/cas/tmp/
         ExecStartPre=/bin/bash /opt/load-kernel-modules.sh
-        ExecStart=/usr/lib/flatcar/kubelet-wrapper \
-{{ if semverCompare ">=1.17.0" .KubeletVersion }}{{ print "          kubelet \\\n" }}{{ end -}}
+        ExecStartPre=/bin/sh -c '/usr/bin/env > /tmp/environment'
+        ExecStart=/usr/bin/docker run --name %n \
+          --rm --tty --restart no \
+          --network host \
+          --pid host \
+          --env-file /tmp/environment \
+          --privileged \
+          --cgroup-parent system.slice \
+          --entrypoint kubelet \
+          -v /dev:/dev \
+          -v /etc/cni/net.d:/etc/cni/net.d \
+          -v /etc/kubernetes:/etc/kubernetes \
+          -v /etc/machine-id:/etc/machine-id:ro \
+          -v /etc/os-release:/etc/os-release:ro \
+          -v /etc/resolv.conf:/etc/resolv.conf:ro \
+          -v /lib/modules:/lib/modules \
+          -v /mnt:/mnt:rshared \
+          -v /opt/cni/bin:/opt/cni/bin:ro \
+          -v /run:/run \
+          -v /sys:/sys \
+          -v /usr/sbin/iscsiadm:/usr/sbin/iscsiadm \
+          -v /var/lib/calico:/var/lib/calico:ro \
+          -v /var/lib/cni:/var/lib/cni \
+          -v /var/lib/docker:/var/lib/docker \
+          -v /var/lib/kubelet:/var/lib/kubelet:rshared \
+          -v /var/log/pods:/var/log/pods \
+          {{ .KubeletImage }} \
 {{ kubeletFlags .KubeletVersion .CloudProviderName .MachineSpec.Name .DNSIPs .ExternalCloudProvider .PauseImage .MachineSpec.Taints | indent 10 }}
-        ExecStop=-/usr/bin/rkt stop --uuid-file=/var/cache/kubelet-pod.uuid
+        ExecStop=-/usr/bin/docker stop %n
         Restart=always
         RestartSec=10
         [Install]

pkg/userdata/flatcar/provider_test.go

@@ -109,6 +109,7 @@ type userDataTestCase struct {
 	registryMirrors       []string
 	pauseImage            string
 	hyperkubeImage        string
+	kubeletImage          string
 }
 
 // TestUserDataGeneration runs the data generation for different
@@ -145,6 +146,39 @@ func TestUserDataGeneration(t *testing.T) {
 			osConfig: &Config{
 				DisableAutoUpdate: true,
 			},
+			hyperkubeImage: "for-kubernetes-less-then-1.18/hyperkubeImage",
+			kubeletImage:   "for-kubernetes-more-then-1.18/kubeletImage",
+		},
+		{
+			name: "v1.18.0",
+			providerSpec: &providerconfigtypes.Config{
+				CloudProvider: "vsphere",
+				SSHPublicKeys: []string{"ssh-rsa AAABBB", "ssh-rsa CCCDDD"},
+				Network: &providerconfigtypes.NetworkConfig{
+					CIDR:    "192.168.81.4/24",
+					Gateway: "192.168.81.1",
+					DNS: providerconfigtypes.DNSConfig{
+						Servers: []string{"8.8.8.8"},
+					},
+				},
+			},
+			spec: clusterv1alpha1.MachineSpec{
+				ObjectMeta: metav1.ObjectMeta{Name: "node1"},
+				Versions: clusterv1alpha1.MachineVersionInfo{
+					Kubelet: "v1.18.0",
+				},
+			},
+			ccProvider: &fakeCloudConfigProvider{
+				name:   "vsphere",
+				config: "{vsphere-config:true}",
+				err:    nil,
+			},
+			DNSIPs: []net.IP{net.ParseIP("10.10.10.10")},
+			osConfig: &Config{
+				DisableAutoUpdate: true,
+			},
+			hyperkubeImage: "for-kubernetes-less-then-1.18/hyperkubeImage",
+			kubeletImage:   "for-kubernetes-more-then-1.18/kubeletImage",
 		},
 	}
 
@@ -188,6 +222,7 @@ func TestUserDataGeneration(t *testing.T) {
 				RegistryMirrors:       test.registryMirrors,
 				PauseImage:            test.pauseImage,
 				HyperkubeImage:        test.hyperkubeImage,
+				KubeletRepository:     test.kubeletImage,
 			}
 
 			s, err := provider.UserData(req)