kubeadm errors now ignorable via v1beta2 config files

Specifically, IgnorePreflightErrors in {Init,Join}Configuration's NodeRegistrationOptions can be used to achieve this.
See also: https://docs.google.com/document/d/1XnP67oO1i9VcDIpw42IzptnJsc5OQM-HTf8cVcjCR2w/edit

Author: marccarre

cmd/kubeadm/app/apis/kubeadm/fuzzer/fuzzer.go

@@ -32,6 +32,7 @@ func Funcs(codecs runtimeserializer.CodecFactory) []interface{} {
 		fuzzClusterConfiguration,
 		fuzzComponentConfigs,
 		fuzzDNS,
+		fuzzNodeRegistration,
 		fuzzLocalEtcd,
 		fuzzNetworking,
 		fuzzJoinConfiguration,
@@ -87,6 +88,13 @@ func fuzzInitConfiguration(obj *kubeadm.InitConfiguration, c fuzz.Continue) {
 	obj.CertificateKey = ""
 }
 
+func fuzzNodeRegistration(obj *kubeadm.NodeRegistrationOptions, c fuzz.Continue) {
+	c.FuzzNoCustom(obj)
+
+	// Pinning values for fields that get defaults if fuzz value is empty string or nil (thus making the round trip test fail)
+	obj.IgnorePreflightErrors = nil
+}
+
 func fuzzClusterConfiguration(obj *kubeadm.ClusterConfiguration, c fuzz.Continue) {
 	c.FuzzNoCustom(obj)
 

cmd/kubeadm/app/apis/kubeadm/types.go

@@ -229,6 +229,9 @@ type NodeRegistrationOptions struct {
 	// kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap
 	// Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on.
 	KubeletExtraArgs map[string]string
+
+	// IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered.
+	IgnorePreflightErrors []string
 }
 
 // Networking contains elements describing cluster's networking configuration.

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion.go

@@ -45,3 +45,15 @@ func Convert_kubeadm_JoinControlPlane_To_v1beta1_JoinControlPlane(in *kubeadm.Jo
 
 	return nil
 }
+
+func Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in *kubeadm.NodeRegistrationOptions, out *NodeRegistrationOptions, s conversion.Scope) error {
+	if err := autoConvert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(in, out, s); err != nil {
+		return err
+	}
+
+	if len(in.IgnorePreflightErrors) > 0 {
+		return errors.New("ignorePreflightErrors field is not supported by v1beta1 config format")
+	}
+
+	return nil
+}

cmd/kubeadm/app/apis/kubeadm/v1beta1/conversion_test.go

@@ -37,6 +37,14 @@ func TestInternalToVersionedInitConfigurationConversion(t *testing.T) {
 			},
 			expectedError: true,
 		},
+		"ignorePreflightErrors set causes an error": {
+			in: kubeadm.InitConfiguration{
+				NodeRegistration: kubeadm.NodeRegistrationOptions{
+					IgnorePreflightErrors: []string{"SomeUndesirableError"},
+				},
+			},
+			expectedError: true,
+		},
 	}
 	for name, tc := range testcases {
 		t.Run(name, func(t *testing.T) {
@@ -51,6 +59,66 @@ func TestInternalToVersionedInitConfigurationConversion(t *testing.T) {
 	}
 }
 
+func TestInternalToVersionedJoinConfigurationConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.JoinConfiguration
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.JoinConfiguration{},
+			expectedError: false,
+		},
+		"ignorePreflightErrors set causes an error": {
+			in: kubeadm.JoinConfiguration{
+				NodeRegistration: kubeadm.NodeRegistrationOptions{
+					IgnorePreflightErrors: []string{"SomeUndesirableError"},
+				},
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &JoinConfiguration{}
+			err := Convert_kubeadm_JoinConfiguration_To_v1beta1_JoinConfiguration(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
+func TestInternalToVersionedNodeRegistrationOptionsConversion(t *testing.T) {
+	testcases := map[string]struct {
+		in            kubeadm.NodeRegistrationOptions
+		expectedError bool
+	}{
+		"conversion succeeds": {
+			in:            kubeadm.NodeRegistrationOptions{},
+			expectedError: false,
+		},
+		"ignorePreflightErrors set causes an error": {
+			in: kubeadm.NodeRegistrationOptions{
+				IgnorePreflightErrors: []string{"SomeUndesirableError"},
+			},
+			expectedError: true,
+		},
+	}
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			versioned := &NodeRegistrationOptions{}
+			err := Convert_kubeadm_NodeRegistrationOptions_To_v1beta1_NodeRegistrationOptions(&tc.in, versioned, nil)
+			if err == nil && tc.expectedError {
+				t.Error("unexpected success")
+			} else if err != nil && !tc.expectedError {
+				t.Errorf("unexpected error: %v", err)
+			}
+		})
+	}
+}
+
 func TestInternalToVersionedJoinControlPlaneConversion(t *testing.T) {
 	testcases := map[string]struct {
 		in            kubeadm.JoinControlPlane

cmd/kubeadm/app/apis/kubeadm/v1beta2/types.go

@@ -215,6 +215,9 @@ type NodeRegistrationOptions struct {
 	// kubeadm writes at runtime for the kubelet to source. This overrides the generic base-level configuration in the kubelet-config-1.X ConfigMap
 	// Flags have higher priority when parsing. These values are local and specific to the node kubeadm is executing on.
 	KubeletExtraArgs map[string]string `json:"kubeletExtraArgs,omitempty"`
+
+	// IgnorePreflightErrors provides a slice of pre-flight errors to be ignored when the current node is registered.
+	IgnorePreflightErrors []string `json:"ignorePreflightErrors,omitempty"`
 }
 
 // Networking contains elements describing cluster's networking configuration

cmd/kubeadm/app/apis/kubeadm/validation/validation.go

@@ -463,12 +463,25 @@ func ValidateAPIEndpoint(c *kubeadm.APIEndpoint, fldPath *field.Path) field.Erro
 	return allErrs
 }
 
-// ValidateIgnorePreflightErrors validates duplicates in ignore-preflight-errors flag.
-func ValidateIgnorePreflightErrors(ignorePreflightErrors []string) (sets.String, error) {
+// ValidateIgnorePreflightErrors validates duplicates in:
+// - ignore-preflight-errors flag and
+// - ignorePreflightErrors field in {Init,Join}Configuration files.
+func ValidateIgnorePreflightErrors(ignorePreflightErrorsFromCLI, ignorePreflightErrorsFromConfigFile []string) (sets.String, error) {
 	ignoreErrors := sets.NewString()
 	allErrs := field.ErrorList{}
 
-	for _, item := range ignorePreflightErrors {
+	for _, item := range ignorePreflightErrorsFromConfigFile {
+		ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive
+	}
+
+	if ignoreErrors.Has("all") {
+		// "all" is forbidden in config files. Administrators should use an
+		// explicit list of errors they want to ignore, as it can be risky to
+		// mask all errors in such a way. Hence, we return an error:
+		allErrs = append(allErrs, field.Invalid(field.NewPath("ignorePreflightErrors"), "all", "'all' cannot be used in configuration file"))
+	}
+
+	for _, item := range ignorePreflightErrorsFromCLI {
 		ignoreErrors.Insert(strings.ToLower(item)) // parameters are case insensitive
 	}
 

cmd/kubeadm/app/apis/kubeadm/validation/validation_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/spf13/pflag"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 	kubeadmapiv1beta2 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta2"
@@ -703,26 +704,81 @@ func TestValidateFeatureGates(t *testing.T) {
 
 func TestValidateIgnorePreflightErrors(t *testing.T) {
 	var tests = []struct {
-		ignorePreflightErrors []string
-		expectedLen           int
-		expectedError         bool
+		ignorePreflightErrorsFromCLI        []string
+		ignorePreflightErrorsFromConfigFile []string
+		expectedSet                         sets.String
+		expectedError                       bool
 	}{
-		{[]string{}, 0, false},                             // empty list
-		{[]string{"check1", "check2"}, 2, false},           // non-duplicate
-		{[]string{"check1", "check2", "check1"}, 2, false}, // duplicates
-		{[]string{"check1", "check2", "all"}, 3, true},     // non-duplicate, but 'all' present together wth individual checks
-		{[]string{"all"}, 1, false},                        // skip all checks by using new flag
-		{[]string{"all"}, 1, false},                        // skip all checks by using both old and new flags at the same time
+		{ // empty lists in CLI and config file
+			[]string{},
+			[]string{},
+			sets.NewString(),
+			false,
+		},
+		{ // empty list in CLI only
+			[]string{},
+			[]string{"a"},
+			sets.NewString("a"),
+			false,
+		},
+		{ // empty list in config file only
+			[]string{"a"},
+			[]string{},
+			sets.NewString("a"),
+			false,
+		},
+		{ // no duplicates, no overlap
+			[]string{"a", "b"},
+			[]string{"c", "d"},
+			sets.NewString("a", "b", "c", "d"),
+			false,
+		},
+		{ // some duplicates, with some overlapping duplicates
+			[]string{"a", "b", "a"},
+			[]string{"c", "b"},
+			sets.NewString("a", "b", "c"),
+			false,
+		},
+		{ // non-duplicate, but 'all' present together with individual checks in CLI
+			[]string{"a", "b", "all"},
+			[]string{},
+			sets.NewString(),
+			true,
+		},
+		{ // empty list in CLI, but 'all' present in config file, which is forbidden
+			[]string{},
+			[]string{"all"},
+			sets.NewString(),
+			true,
+		},
+		{ // non-duplicate, but 'all' present in config file, which is forbidden
+			[]string{"a", "b"},
+			[]string{"all"},
+			sets.NewString(),
+			true,
+		},
+		{ // non-duplicate, but 'all' present in CLI, while values are in config file, which is forbidden
+			[]string{"all"},
+			[]string{"a", "b"},
+			sets.NewString(),
+			true,
+		},
+		{ // skip all checks
+			[]string{"all"},
+			[]string{},
+			sets.NewString("all"),
+			false,
+		},
 	}
 	for _, rt := range tests {
-		result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrors)
+		result, err := ValidateIgnorePreflightErrors(rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile)
 		switch {
 		case err != nil && !rt.expectedError:
-			t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s), error: %v", rt.ignorePreflightErrors, err)
+			t.Errorf("ValidateIgnorePreflightErrors: unexpected error for input (%s, %s), error: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, err)
 		case err == nil && rt.expectedError:
-			t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s) but got: %v", rt.ignorePreflightErrors, result)
-		case result.Len() != rt.expectedLen:
-			t.Errorf("ValidateIgnorePreflightErrors: expected Len = %d for input (%s) but got: %v, %v", rt.expectedLen, rt.ignorePreflightErrors, result.Len(), result)
+			t.Errorf("ValidateIgnorePreflightErrors: expected error for input (%s, %s) but got: %v", rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result)
+		case err == nil && !result.Equal(rt.expectedSet):
+			t.Errorf("ValidateIgnorePreflightErrors: expected (%v) for input (%s, %s) but got: %v", rt.expectedSet, rt.ignorePreflightErrorsFromCLI, rt.ignorePreflightErrorsFromConfigFile, result)
 		}
 	}
 }

cmd/kubeadm/app/cmd/init.go

@@ -296,11 +296,6 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		return nil, err
 	}
 
-	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors)
-	if err != nil {
-		return nil, err
-	}
-
 	if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil {
 		return nil, err
 	}
@@ -316,6 +311,13 @@ func newInitData(cmd *cobra.Command, args []string, options *initOptions, out io
 		return nil, err
 	}
 
+	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(options.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
+	if err != nil {
+		return nil, err
+	}
+	// Also set the union of pre-flight errors to InitConfiguration, to provide a consistent view of the runtime configuration:
+	cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()
+
 	// override node name and CRI socket from the command line options
 	if options.externalcfg.NodeRegistration.Name != "" {
 		cfg.NodeRegistration.Name = options.externalcfg.NodeRegistration.Name

cmd/kubeadm/app/cmd/init_test.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/kubernetes/cmd/kubeadm/app/cmd/options"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 )
@@ -38,6 +39,9 @@ bootstrapTokens:
 nodeRegistration:
   criSocket: /run/containerd/containerd.sock
   name: someName
+  ignorePreflightErrors:
+    - c
+    - d
 ---
 apiVersion: kubeadm.k8s.io/v1beta2
 kind: ClusterConfiguration
@@ -129,6 +133,30 @@ func TestNewInitData(t *testing.T) {
 			},
 			expectError: true,
 		},
+
+		// Pre-flight errors:
+		{
+			name: "pre-flights errors from CLI args only",
+			flags: map[string]string{
+				options.IgnorePreflightErrors: "a,b",
+			},
+			validate: expectedInitIgnorePreflightErrors("a", "b"),
+		},
+		{
+			name: "pre-flights errors from InitConfiguration only",
+			flags: map[string]string{
+				options.CfgPath: configFilePath,
+			},
+			validate: expectedInitIgnorePreflightErrors("c", "d"),
+		},
+		{
+			name: "pre-flights errors from both CLI args and InitConfiguration",
+			flags: map[string]string{
+				options.CfgPath:               configFilePath,
+				options.IgnorePreflightErrors: "a,b",
+			},
+			validate: expectedInitIgnorePreflightErrors("a", "b", "c", "d"),
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -157,3 +185,15 @@ func TestNewInitData(t *testing.T) {
 		})
 	}
 }
+
+func expectedInitIgnorePreflightErrors(expectedItems ...string) func(t *testing.T, data *initData) {
+	expected := sets.NewString(expectedItems...)
+	return func(t *testing.T, data *initData) {
+		if !expected.Equal(data.ignorePreflightErrors) {
+			t.Errorf("Invalid ignore preflight errors. Expected: %v. Actual: %v", expected.List(), data.ignorePreflightErrors.List())
+		}
+		if !expected.HasAll(data.cfg.NodeRegistration.IgnorePreflightErrors...) {
+			t.Errorf("Invalid ignore preflight errors in InitConfiguration. Expected: %v. Actual: %v", expected.List(), data.cfg.NodeRegistration.IgnorePreflightErrors)
+		}
+	}
+}

cmd/kubeadm/app/cmd/join.go

@@ -349,12 +349,7 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 		}
 	}
 
-	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = validation.ValidateMixedArguments(cmd.Flags()); err != nil {
+	if err := validation.ValidateMixedArguments(cmd.Flags()); err != nil {
 		return nil, err
 	}
 
@@ -383,6 +378,13 @@ func newJoinData(cmd *cobra.Command, args []string, opt *joinOptions, out io.Wri
 		return nil, err
 	}
 
+	ignorePreflightErrorsSet, err := validation.ValidateIgnorePreflightErrors(opt.ignorePreflightErrors, cfg.NodeRegistration.IgnorePreflightErrors)
+	if err != nil {
+		return nil, err
+	}
+	// Also set the union of pre-flight errors to JoinConfiguration, to provide a consistent view of the runtime configuration:
+	cfg.NodeRegistration.IgnorePreflightErrors = ignorePreflightErrorsSet.List()
+
 	// override node name and CRI socket from the command line opt
 	if opt.externalcfg.NodeRegistration.Name != "" {
 		cfg.NodeRegistration.Name = opt.externalcfg.NodeRegistration.Name