Add a event to PV when mount fails because of fs mismatch

Filesystem mismatch is a special event. This could indicate
either user has asked for incorrect filesystem or there is a error
from which mount operation can not recover on retry.

Co-Authored-By: Jordan Liggitt <[email protected]>

Author: gnufied

pkg/kubelet/events/event.go

@@ -72,6 +72,7 @@ const (
 	SandboxChanged                       = "SandboxChanged"
 	FailedCreatePodSandBox               = "FailedCreatePodSandBox"
 	FailedStatusPodSandBox               = "FailedPodSandBoxStatus"
+	FailedMountOnFilesystemMismatch      = "FailedMountOnFilesystemMismatch"
 )
 
 // Image manager event reason list

pkg/volume/local/local.go

@@ -342,7 +342,7 @@ func (dm *deviceMounter) mountLocalBlockDevice(spec *volume.Spec, devicePath str
 		if rmErr := os.Remove(deviceMountPath); rmErr != nil {
 			klog.Warningf("local: failed to remove %s: %v", deviceMountPath, rmErr)
 		}
-		return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %v", devicePath, deviceMountPath, fstype, err)
+		return fmt.Errorf("local: failed to mount device %s at %s (fstype: %s), error %w", devicePath, deviceMountPath, fstype, err)
 	}
 	klog.V(3).Infof("local: successfully mount device %s at %s (fstype: %s)", devicePath, deviceMountPath, fstype)
 	return nil

pkg/volume/util/operationexecutor/operation_generator.go

@@ -581,6 +581,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 				devicePath,
 				deviceMountPath)
 			if err != nil {
+				og.checkForFailedMount(volumeToMount, err)
 				og.markDeviceErrorState(volumeToMount, devicePath, deviceMountPath, err, actualStateOfWorld)
 				// On failure, return error. Caller will log and retry.
 				return volumeToMount.GenerateError("MountVolume.MountDevice failed", err)
@@ -635,6 +636,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 			VolumeMountState:    VolumeMounted,
 		}
 		if mountErr != nil {
+			og.checkForFailedMount(volumeToMount, mountErr)
 			og.markVolumeErrorState(volumeToMount, markOpts, mountErr, actualStateOfWorld)
 			// On failure, return error. Caller will log and retry.
 			return volumeToMount.GenerateError("MountVolume.SetUp failed", mountErr)
@@ -684,6 +686,18 @@ func (og *operationGenerator) GenerateMountVolumeFunc(
 	}
 }
 
+func (og *operationGenerator) checkForFailedMount(volumeToMount VolumeToMount, mountError error) {
+	pv := volumeToMount.VolumeSpec.PersistentVolume
+	if pv == nil {
+		return
+	}
+
+	if volumetypes.IsFilesystemMismatchError(mountError) {
+		simpleMsg, _ := volumeToMount.GenerateMsg("MountVolume failed", mountError.Error())
+		og.recorder.Eventf(pv, v1.EventTypeWarning, kevents.FailedMountOnFilesystemMismatch, simpleMsg)
+	}
+}
+
 func (og *operationGenerator) markDeviceErrorState(volumeToMount VolumeToMount, devicePath, deviceMountPath string, mountError error, actualStateOfWorld ActualStateOfWorldMounterUpdater) {
 	if volumetypes.IsOperationFinishedError(mountError) &&
 		actualStateOfWorld.GetDeviceMountState(volumeToMount.VolumeName) == DeviceMountUncertain {

pkg/volume/util/types/BUILD

@@ -3,6 +3,7 @@ package(default_visibility = ["//visibility:public"])
 load(
     "@io_bazel_rules_go//go:def.bzl",
     "go_library",
+    "go_test",
 )
 
 go_library(
@@ -12,6 +13,7 @@ go_library(
     deps = [
         "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
         "//staging/src/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
+        "//vendor/k8s.io/utils/mount:go_default_library",
     ],
 )
 
@@ -27,3 +29,10 @@ filegroup(
     srcs = [":package-srcs"],
     tags = ["automanaged"],
 )
+
+go_test(
+    name = "go_default_test",
+    srcs = ["types_test.go"],
+    embed = [":go_default_library"],
+    deps = ["//vendor/k8s.io/utils/mount:go_default_library"],
+)

pkg/volume/util/types/types.go

@@ -18,8 +18,11 @@ limitations under the License.
 package types
 
 import (
+	"errors"
+
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/utils/mount"
 )
 
 // UniquePodName defines the type to key pods off of
@@ -93,6 +96,16 @@ func IsOperationFinishedError(err error) bool {
 	return true
 }
 
+// IsFilesystemMismatchError checks if mount failed because requested filesystem
+// on PVC and actual filesystem on disk did not match
+func IsFilesystemMismatchError(err error) bool {
+	mountError := &mount.MountError{}
+	if errors.As(err, &mountError) && mountError.Type == mount.FilesystemMismatch {
+		return true
+	}
+	return false
+}
+
 // IsUncertainProgressError checks if given error is of type that indicates
 // operation might be in-progress in background.
 func IsUncertainProgressError(err error) bool {

pkg/volume/util/types/types_test.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package types
+
+import (
+	"fmt"
+	"testing"
+
+	"k8s.io/utils/mount"
+)
+
+func TestIsFilesystemMismatchError(t *testing.T) {
+	tests := []struct {
+		mountError  error
+		expectError bool
+	}{
+		{
+			mount.NewMountError(mount.FilesystemMismatch, "filesystem mismatch"),
+			true,
+		},
+		{
+			mount.NewMountError(mount.FormatFailed, "filesystem mismatch"),
+			false,
+		},
+		{
+			fmt.Errorf("mount failed %w", mount.NewMountError(mount.FilesystemMismatch, "filesystem mismatch")),
+			true,
+		},
+	}
+	for _, test := range tests {
+		ok := IsFilesystemMismatchError(test.mountError)
+		if ok != test.expectError {
+			t.Errorf("expected filesystem mismatch to be %v but got %v", test.expectError, ok)
+		}
+	}
+}