Merge pull request #35 from hughdanliu/startup-taint

Add support for node startup taints

Author: k8s-ci-robot

charts/aws-fsx-openzfs-csi-driver/templates/clusterrole-csi-node.yaml

@@ -8,4 +8,4 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
-    verbs: ["get"]
+    verbs: ["get", "patch"]

charts/aws-fsx-openzfs-csi-driver/templates/node-daemonset.yaml

@@ -43,6 +43,8 @@ spec:
         {{- with .Values.node.tolerations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+        - key: "fsx.openzfs.csi.aws.com/agent-not-ready"
+          operator: "Exists"
         {{- end }}
       {{- with .Values.node.securityContext }}
       securityContext:

charts/aws-fsx-openzfs-csi-driver/values.yaml

@@ -3,7 +3,6 @@
 # Declare variables to be passed into your templates.
 
 image:
-  # TODO: replace this temporary private ECR and regenerate the manifests when we are ready to release
   repository: public.ecr.aws/fsx-csi-driver/aws-fsx-openzfs-csi-driver
   # Overrides the image tag whose default is v{{ .Chart.AppVersion }}
   tag: v0.1.0

deploy/kubernetes/base/clusterrole-csi-node.yaml

@@ -9,4 +9,4 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["nodes"]
-    verbs: ["get"]
+    verbs: ["get", "patch"]

docs/install.md

@@ -79,6 +79,21 @@ Additionally, the driver node tolerates all taints.
 If you do not wish to deploy the driver node on all nodes, please set Helm `Value.node.tolerateAllTaints` to false before deployment. 
 Add policies to `Value.node.tolerations` to configure customized toleration for nodes.
 
+### Configure node startup taint
+There are potential race conditions on node startup (especially when a node is first joining the cluster) 
+where pods/processes that rely on the FSx for OpenZFS CSI Driver can act on a node before the FSx for OpenZFS CSI Driver is able to start up and become fully ready. 
+To combat this, the FSx for OpenZFS CSI Driver contains a feature to automatically remove a taint from the node on startup. 
+Users can taint their nodes when they join the cluster and/or on startup.  
+This will prevent other pods from running and/or being scheduled on the node prior to the FSx for OpenZFS CSI Driver becoming ready.
+
+This feature is activated by default. Cluster administrators should apply the taint `fsx.openzfs.csi.aws.com/agent-not-ready:NoExecute` to their nodes:
+```shell
+kubectl taint nodes $NODE_NAME fsx.openzfs.csi.aws.com/agent-not-ready:NoExecute
+```
+Note that any effect will work, but `NoExecute` is recommended. 
+
+For example, EKS Managed Node Groups [support automatically tainting nodes](https://docs.aws.amazon.com/eks/latest/userguide/node-taints-managed-node-groups.html).
+
 ### Deploy driver
 You may deploy the FSx for OpenZFS CSI driver via Kustomize or Helm
 

hack/update-gomock

@@ -22,3 +22,7 @@ mockgen -package=mocks -destination=./pkg/cloud/mocks/mock_metadata.go --build_f
 
 mockgen -package=mocks -destination=./pkg/cloud/mocks/mock_fsx.go --build_flags=--mod=mod ${IMPORT_PATH}/pkg/cloud FSx
 mockgen -package=mocks -destination=./pkg/driver/mocks/mock_cloud.go --build_flags=--mod=mod ${IMPORT_PATH}/pkg/cloud Cloud
+
+# Reflection-based mocking for external dependencies
+mockgen -package=mocks -destination=./pkg/driver/mocks/mock_k8s_client.go --build_flags=--mod=mod -mock_names='Interface=MockKubernetesClient' k8s.io/client-go/kubernetes Interface
+mockgen -package=mocks -destination=./pkg/driver/mocks/mock_k8s_corev1.go --build_flags=--mod=mod k8s.io/client-go/kubernetes/typed/core/v1 CoreV1Interface,NodeInterface

pkg/driver/constants.go

@@ -6,3 +6,9 @@ package driver
 const (
 	DefaultCSIEndpoint = "unix://tmp/csi.sock"
 )
+
+// constants for node k8s API use
+const (
+	// AgentNotReadyNodeTaintKey contains the key of taints to be removed on driver startup
+	AgentNotReadyNodeTaintKey = "fsx.openzfs.csi.aws.com/agent-not-ready"
+)