diff --git a/docs/book/src/topics/external-cloud-provider-with-ebs-csi-driver.md b/docs/book/src/topics/external-cloud-provider-with-ebs-csi-driver.md index 817e1ceed7..2717949df9 100644 --- a/docs/book/src/topics/external-cloud-provider-with-ebs-csi-driver.md +++ b/docs/book/src/topics/external-cloud-provider-with-ebs-csi-driver.md @@ -27,15 +27,15 @@ For clusters that will use external CCM, `cloud-provider: external` flag needs t External CCM and EBS CSI driver can be installed manually or using ClusterResourceSets (CRS) onto the CAPA workload cluster. -To install them with CRS, create a CRS resource on the management cluster with labels, for example `csi: external` and `ccm: external` labels. +To install them with CRS, create a CRS resource on the management cluster with labels, for example `csi: external` and `ccm: external` labels. Then, when creating `Cluster` objects for workload clusters that should have this CSR applied, create them with matching labels `csi: external` and `ccm: external` for CSI and CCM, respectively. -Manifests for installing the AWS CCM and the AWS EBS CSI driver are available from their respective -GitHub repositories (see [here for the AWS CCM](https://github.com/kubernetes/cloud-provider-aws) and +Manifests for installing the AWS CCM and the AWS EBS CSI driver are available from their respective +GitHub repositories (see [here for the AWS CCM](https://github.com/kubernetes/cloud-provider-aws) and [here for the AWS EBS CSI driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver)). -An example of a workload cluster manifest with labels assigned for matching to a CRS can be found -[here](https://github.com/kubernetes-sigs/cluster-api-provider-aws/tree/main/templates/cluster-template-external-cloud-provider.yaml). +An example of a workload cluster manifest with labels assigned for matching to a CRS can be found +[here](https://github.com/kubernetes-sigs/cluster-api-provider-aws/tree/main/templates/cluster-template.yaml). ### Verifying dynamically provisioned volumes with CSI driver Once you have the cluster with external CCM and CSI controller running successfully, you can test the CSI driver functioning with following steps after switching to workload cluster: @@ -113,14 +113,14 @@ spec: 3. Once you apply the above manifest, the EBS volumes will be created and attached to the worker nodes. >**IMPORTANT WARNING:** The CRDs from the AWS EBS CSI driver and AWS external cloud provider gives issue while installing the respective controllers on the AWS Cluster, it doesn't allow statefulsets to create the volume on existing EC2 instance. -> We need the CSI controller deployment and CCM pinned to the control plane which has right permissions to create, attach +> We need the CSI controller deployment and CCM pinned to the control plane which has right permissions to create, attach > and mount the volumes to EC2 instances. To achieve this, you should add the node affinity rules to the CSI driver controller deployment and CCM DaemonSet manifests. > ```yaml > tolerations: > - key: node-role.kubernetes.io/master > effect: NoSchedule > - effect: NoSchedule -> key: node-role.kubernetes.io/control-plane +> key: node-role.kubernetes.io/control-plane > affinity: > nodeAffinity: > requiredDuringSchedulingIgnoredDuringExecution: @@ -132,14 +132,14 @@ spec: > - key: node-role.kubernetes.io/master > operator: Exists >``` - + ## Validated upgrade paths for existing clusters From Kubernetes 1.23 onwards, `CSIMigrationAWS` flag is enabled by default, which requires the installation of [external CSI driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver), unless `CSIMigrationAWS` is disabled by the user. For installing external CSI/CCM in the upgraded cluster, CRS can be used, see the section above for details. -CCM and CSI do not need to be migrated to use external plugins at the same time, +CCM and CSI do not need to be migrated to use external plugins at the same time, external CSI drivers works with in-tree CCM (Warning: using in-tree CSI with external CCM does not work). **Following 3 upgrade paths are validated:** diff --git a/templates/cluster-template-external-cloud-provider.yaml b/templates/cluster-template-external-cloud-provider.yaml deleted file mode 100644 index beca123294..0000000000 --- a/templates/cluster-template-external-cloud-provider.yaml +++ /dev/null @@ -1,992 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - labels: - ccm: external - cni: ${CLUSTER_NAME}-crs-0 - csi: external - name: ${CLUSTER_NAME} -spec: - clusterNetwork: - pods: - cidrBlocks: - - 192.168.0.0/16 - controlPlaneRef: - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSCluster - name: ${CLUSTER_NAME} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSCluster -metadata: - name: ${CLUSTER_NAME} -spec: - network: - vpc: - availabilityZoneUsageLimit: 1 - region: ${AWS_REGION} - sshKeyName: ${AWS_SSH_KEY_NAME} ---- -apiVersion: controlplane.cluster.x-k8s.io/v1beta1 -kind: KubeadmControlPlane -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - kubeadmConfigSpec: - clusterConfiguration: - apiServer: - extraArgs: - cloud-provider: external - controllerManager: - extraArgs: - cloud-provider: external - initConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - name: '{{ ds.meta_data.local_hostname }}' - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - name: '{{ ds.meta_data.local_hostname }}' - machineTemplate: - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-control-plane - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-control-plane -spec: - template: - spec: - iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_MACHINE_TYPE} - sshKeyName: ${AWS_SSH_KEY_NAME} ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: MachineDeployment -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - clusterName: ${CLUSTER_NAME} - replicas: ${WORKER_MACHINE_COUNT} - selector: - matchLabels: null - template: - spec: - bootstrap: - configRef: - apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 - kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 - clusterName: ${CLUSTER_NAME} - infrastructureRef: - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-md-0 - version: ${KUBERNETES_VERSION} ---- -apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 -kind: AWSMachineTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_MACHINE_TYPE} - sshKeyName: ${AWS_SSH_KEY_NAME} ---- -apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 -kind: KubeadmConfigTemplate -metadata: - name: ${CLUSTER_NAME}-md-0 -spec: - template: - spec: - joinConfiguration: - nodeRegistration: - kubeletExtraArgs: - cloud-provider: external - name: '{{ ds.meta_data.local_hostname }}' ---- -apiVersion: v1 -data: ${CNI_RESOURCES} -kind: ConfigMap -metadata: - name: cni-${CLUSTER_NAME}-crs-0 ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: ${CLUSTER_NAME}-crs-0 -spec: - clusterSelector: - matchLabels: - cni: ${CLUSTER_NAME}-crs-0 - resources: - - kind: ConfigMap - name: cni-${CLUSTER_NAME}-crs-0 - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-ccm -spec: - clusterSelector: - matchLabels: - ccm: external - resources: - - kind: ConfigMap - name: cloud-controller-manager-addon - strategy: ApplyOnce ---- -apiVersion: addons.cluster.x-k8s.io/v1beta1 -kind: ClusterResourceSet -metadata: - name: crs-csi -spec: - clusterSelector: - matchLabels: - csi: external - resources: - - kind: ConfigMap - name: aws-ebs-csi-driver-addon - strategy: ApplyOnce ---- -apiVersion: v1 -data: - aws-ccm-external.yaml: | - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - name: aws-cloud-controller-manager - namespace: kube-system - labels: - k8s-app: aws-cloud-controller-manager - spec: - selector: - matchLabels: - k8s-app: aws-cloud-controller-manager - updateStrategy: - type: RollingUpdate - template: - metadata: - labels: - k8s-app: aws-cloud-controller-manager - spec: - nodeSelector: - node-role.kubernetes.io/master: "" - tolerations: - - key: node.cloudprovider.kubernetes.io/uninitialized - value: "true" - effect: NoSchedule - - key: node-role.kubernetes.io/master - effect: NoSchedule - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - serviceAccountName: cloud-controller-manager - containers: - - name: aws-cloud-controller-manager - image: gcr.io/k8s-staging-provider-aws/cloud-controller-manager:v1.20.0-alpha.0 - args: - - --v=2 - resources: - requests: - cpu: 200m - hostNetwork: true - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: RoleBinding - metadata: - name: cloud-controller-manager:apiserver-authentication-reader - namespace: kube-system - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: extension-apiserver-authentication-reader - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - name: system:cloud-controller-manager - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - create - - patch - - update - - apiGroups: - - "" - resources: - - nodes - verbs: - - '*' - - apiGroups: - - "" - resources: - - nodes/status - verbs: - - patch - - apiGroups: - - "" - resources: - - services - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - services/status - verbs: - - list - - patch - - update - - watch - - apiGroups: - - "" - resources: - - serviceaccounts - verbs: - - create - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - update - - watch - - apiGroups: - - "" - resources: - - configmaps - verbs: - - list - - watch - - apiGroups: - - "" - resources: - - endpoints - verbs: - - create - - get - - list - - watch - - update - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - create - - get - - list - - watch - - update - --- - kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: system:cloud-controller-manager - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: system:cloud-controller-manager - subjects: - - apiGroup: "" - kind: ServiceAccount - name: cloud-controller-manager - namespace: kube-system -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: cloud-controller-manager-addon ---- -apiVersion: v1 -data: - aws-ebs-csi-external.yaml: |- - apiVersion: v1 - kind: Secret - metadata: - name: aws-secret - namespace: kube-system - stringData: - key_id: "" - access_key: "" - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-controller-sa - namespace: kube-system - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-node-sa - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-external-attacher-role - rules: - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - csi.storage.k8s.io - resources: - - csinodeinfos - verbs: - - get - - list - - watch - - apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - - update - - patch - - apiGroups: - - storage.k8s.io - resources: - - volumeattachments/status - verbs: - - patch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-external-provisioner-role - rules: - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - create - - delete - - apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - update - - apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch - - apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshots - verbs: - - get - - list - - apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - get - - list - - apiGroups: - - storage.k8s.io - resources: - - csinodes - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - watch - - list - - delete - - update - - create - - apiGroups: - - storage.k8s.io - resources: - - volumeattachments - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-external-resizer-role - rules: - - apiGroups: - - "" - resources: - - persistentvolumes - verbs: - - get - - list - - watch - - update - - patch - - apiGroups: - - "" - resources: - - persistentvolumeclaims - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - persistentvolumeclaims/status - verbs: - - update - - patch - - apiGroups: - - storage.k8s.io - resources: - - storageclasses - verbs: - - get - - list - - watch - - apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRole - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-external-snapshotter-role - rules: - - apiGroups: - - "" - resources: - - events - verbs: - - list - - watch - - create - - update - - patch - - apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotclasses - verbs: - - get - - list - - watch - - apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents - verbs: - - create - - get - - list - - watch - - update - - delete - - apiGroups: - - snapshot.storage.k8s.io - resources: - - volumesnapshotcontents/status - verbs: - - update - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-attacher-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ebs-external-attacher-role - subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-provisioner-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ebs-external-provisioner-role - subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-resizer-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ebs-external-resizer-role - subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system - --- - apiVersion: rbac.authorization.k8s.io/v1 - kind: ClusterRoleBinding - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-snapshotter-binding - roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: ebs-external-snapshotter-role - subjects: - - kind: ServiceAccount - name: ebs-csi-controller-sa - namespace: kube-system - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-controller - namespace: kube-system - spec: - replicas: 2 - selector: - matchLabels: - app: ebs-csi-controller - app.kubernetes.io/name: aws-ebs-csi-driver - template: - metadata: - labels: - app: ebs-csi-controller - app.kubernetes.io/name: aws-ebs-csi-driver - spec: - containers: - - args: - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=2 - env: - - name: CSI_ENDPOINT - value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: AWS_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - key: key_id - name: aws-secret - optional: true - - name: AWS_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - key: access_key - name: aws-secret - optional: true - image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.2.0 - imagePullPolicy: IfNotPresent - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 3 - name: ebs-plugin - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - readinessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 3 - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --v=2 - - --feature-gates=Topology=true - - --extra-create-metadata - - --leader-election=true - - --default-fstype=ext4 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8.io/sig-storage/csi-provisioner:v2.1.1 - name: csi-provisioner - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --v=2 - - --leader-election=true - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8.io/sig-storage/csi-attacher:v3.1.0 - name: csi-attacher - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --leader-election=true - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8.io/sig-storage/csi-snapshotter:v3.0.3 - name: csi-snapshotter - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=$(ADDRESS) - - --v=2 - env: - - name: ADDRESS - value: /var/lib/csi/sockets/pluginproxy/csi.sock - image: registry.k8.io/sig-storage/csi-resizer:v1.0.0 - imagePullPolicy: Always - name: csi-resizer - volumeMounts: - - mountPath: /var/lib/csi/sockets/pluginproxy/ - name: socket-dir - - args: - - --csi-address=/csi/csi.sock - image: registry.k8.io/sig-storage/livenessprobe:v2.2.0 - name: liveness-probe - volumeMounts: - - mountPath: /csi - name: socket-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-cluster-critical - serviceAccountName: ebs-csi-controller-sa - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - tolerationSeconds: 300 - - key: node-role.kubernetes.io/master - effect: NoSchedule - - effect: NoSchedule - key: node-role.kubernetes.io/control-plane - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - volumes: - - emptyDir: {} - name: socket-dir - --- - apiVersion: policy/v1 - kind: PodDisruptionBudget - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-controller - namespace: kube-system - spec: - maxUnavailable: 1 - selector: - matchLabels: - app: ebs-csi-controller - app.kubernetes.io/name: aws-ebs-csi-driver - --- - apiVersion: apps/v1 - kind: DaemonSet - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs-csi-node - namespace: kube-system - spec: - selector: - matchLabels: - app: ebs-csi-node - app.kubernetes.io/name: aws-ebs-csi-driver - template: - metadata: - labels: - app: ebs-csi-node - app.kubernetes.io/name: aws-ebs-csi-driver - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: eks.amazonaws.com/compute-type - operator: NotIn - values: - - fargate - containers: - - args: - - node - - --endpoint=$(CSI_ENDPOINT) - - --logtostderr - - --v=2 - env: - - name: CSI_ENDPOINT - value: unix:/csi/csi.sock - - name: CSI_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - image: registry.k8.io/provider-aws/aws-ebs-csi-driver:v1.2.0 - livenessProbe: - failureThreshold: 5 - httpGet: - path: /healthz - port: healthz - initialDelaySeconds: 10 - periodSeconds: 10 - timeoutSeconds: 3 - name: ebs-plugin - ports: - - containerPort: 9808 - name: healthz - protocol: TCP - securityContext: - privileged: true - volumeMounts: - - mountPath: /var/lib/kubelet - mountPropagation: Bidirectional - name: kubelet-dir - - mountPath: /csi - name: plugin-dir - - mountPath: /dev - name: device-dir - - args: - - --csi-address=$(ADDRESS) - - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) - - --v=2 - env: - - name: ADDRESS - value: /csi/csi.sock - - name: DRIVER_REG_SOCK_PATH - value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock - image: registry.k8.io/sig-storage/csi-node-driver-registrar:v2.1.0 - name: node-driver-registrar - volumeMounts: - - mountPath: /csi - name: plugin-dir - - mountPath: /registration - name: registration-dir - - args: - - --csi-address=/csi/csi.sock - image: registry.k8.io/sig-storage/livenessprobe:v2.2.0 - name: liveness-probe - volumeMounts: - - mountPath: /csi - name: plugin-dir - nodeSelector: - kubernetes.io/os: linux - priorityClassName: system-node-critical - serviceAccountName: ebs-csi-node-sa - tolerations: - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - operator: Exists - tolerationSeconds: 300 - volumes: - - hostPath: - path: /var/lib/kubelet - type: Directory - name: kubelet-dir - - hostPath: - path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ - type: DirectoryOrCreate - name: plugin-dir - - hostPath: - path: /var/lib/kubelet/plugins_registry/ - type: Directory - name: registration-dir - - hostPath: - path: /dev - type: Directory - name: device-dir - updateStrategy: - rollingUpdate: - maxUnavailable: 10% - type: RollingUpdate - --- - apiVersion: storage.k8s.io/v1 - kind: CSIDriver - metadata: - labels: - app.kubernetes.io/name: aws-ebs-csi-driver - name: ebs.csi.aws.com - spec: - attachRequired: true - podInfoOnMount: false -kind: ConfigMap -metadata: - annotations: - note: generated - labels: - type: generated - name: aws-ebs-csi-driver-addon diff --git a/templates/cluster-template-flatcar.yaml b/templates/cluster-template-flatcar.yaml index 058e9d48c6..1a59976720 100644 --- a/templates/cluster-template-flatcar.yaml +++ b/templates/cluster-template-flatcar.yaml @@ -871,7 +871,7 @@ data: - emptyDir: {} name: socket-dir --- - apiVersion: policy/v1beta1 + apiVersion: policy/v1 kind: PodDisruptionBudget metadata: labels: diff --git a/templates/cluster-template-machinepool.yaml b/templates/cluster-template-machinepool.yaml index 3b6aeacb17..964770777d 100644 --- a/templates/cluster-template-machinepool.yaml +++ b/templates/cluster-template-machinepool.yaml @@ -1,17 +1,23 @@ ---- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: + labels: + ccm: external + csi: external name: "${CLUSTER_NAME}" spec: + clusterNetwork: + pods: + cidrBlocks: + - 192.168.0.0/16 + controlPlaneRef: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + name: "${CLUSTER_NAME}-control-plane" infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster name: "${CLUSTER_NAME}" - controlPlaneRef: - kind: KubeadmControlPlane - apiVersion: controlplane.cluster.x-k8s.io/v1beta1 - name: "${CLUSTER_NAME}-control-plane" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster @@ -21,55 +27,54 @@ spec: region: "${AWS_REGION}" sshKeyName: "${AWS_SSH_KEY_NAME}" --- -kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane metadata: name: "${CLUSTER_NAME}-control-plane" spec: - replicas: ${CONTROL_PLANE_MACHINE_COUNT} - machineTemplate: - infrastructureRef: - kind: AWSMachineTemplate - apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 - name: "${CLUSTER_NAME}-control-plane" kubeadmConfigSpec: - initConfiguration: - nodeRegistration: - name: '{{ ds.meta_data.local_hostname }}' - kubeletExtraArgs: - cloud-provider: aws clusterConfiguration: apiServer: extraArgs: - cloud-provider: aws + cloud-provider: external controllerManager: extraArgs: - cloud-provider: aws - joinConfiguration: + cloud-provider: external + initConfiguration: nodeRegistration: + kubeletExtraArgs: + cloud-provider: external name: '{{ ds.meta_data.local_hostname }}' + joinConfiguration: + nodeRegistration: kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external + name: '{{ ds.meta_data.local_hostname }}' + machineTemplate: + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 + kind: AWSMachineTemplate + name: "${CLUSTER_NAME}-control-plane" + replicas: ${CONTROL_PLANE_MACHINE_COUNT} version: "${KUBERNETES_VERSION}" --- -kind: AWSMachineTemplate apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 +kind: AWSMachineTemplate metadata: name: "${CLUSTER_NAME}-control-plane" spec: template: spec: - instanceType: "${AWS_CONTROL_PLANE_MACHINE_TYPE}" iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" + instanceType: "${AWS_CONTROL_PLANE_MACHINE_TYPE}" sshKeyName: "${AWS_SSH_KEY_NAME}" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachinePool metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default + name: "${CLUSTER_NAME}-mp-0" spec: - clusterName: ${CLUSTER_NAME} + clusterName: "${CLUSTER_NAME}" replicas: ${WORKER_MACHINE_COUNT} template: spec: @@ -77,40 +82,38 @@ spec: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig - name: ${CLUSTER_NAME}-mp-0 + name: "${CLUSTER_NAME}-mp-0" clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool - name: ${CLUSTER_NAME}-mp-0 + name: "${CLUSTER_NAME}-mp-0" version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachinePool metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default + name: "${CLUSTER_NAME}-mp-0" spec: - minSize: 1 - maxSize: 10 - availabilityZones: - - "${AWS_AVAILABILITY_ZONE}" awsLaunchTemplate: iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io instanceType: "${AWS_NODE_MACHINE_TYPE}" sshKeyName: "${AWS_SSH_KEY_NAME}" + availabilityZones: + - "${AWS_AVAILABILITY_ZONE}" + minSize: 1 + maxSize: 10 --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfig metadata: - name: ${CLUSTER_NAME}-mp-0 - namespace: default + name: "${CLUSTER_NAME}-mp-0" spec: joinConfiguration: nodeRegistration: - name: '{{ ds.meta_data.local_hostname }}' kubeletExtraArgs: - cloud-provider: aws + cloud-provider: external + name: '{{ ds.meta_data.local_hostname }}' --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineHealthCheck @@ -128,4 +131,896 @@ spec: timeout: 300s - type: Ready status: "False" - timeout: 300s \ No newline at end of file + timeout: 300s +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-ccm +spec: + clusterSelector: + matchLabels: + ccm: external + resources: + - kind: ConfigMap + name: cloud-controller-manager-addon + strategy: ApplyOnce +--- +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + name: crs-csi +spec: + clusterSelector: + matchLabels: + csi: external + resources: + - kind: ConfigMap + name: aws-ebs-csi-driver-addon + strategy: ApplyOnce +--- +apiVersion: v1 +data: + aws-ccm-external.yaml: | + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + name: aws-cloud-controller-manager + namespace: kube-system + labels: + k8s-app: aws-cloud-controller-manager + spec: + selector: + matchLabels: + k8s-app: aws-cloud-controller-manager + updateStrategy: + type: RollingUpdate + template: + metadata: + labels: + k8s-app: aws-cloud-controller-manager + spec: + nodeSelector: + node-role.kubernetes.io/control-plane: "" + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + serviceAccountName: cloud-controller-manager + containers: + - name: aws-cloud-controller-manager + image: registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.3 + args: + - --v=2 + - --cloud-provider=aws + - --use-service-account-credentials=true + - --configure-cloud-routes=false + resources: + requests: + cpu: 200m + hostNetwork: true + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: cloud-controller-manager:apiserver-authentication-reader + namespace: kube-system + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: extension-apiserver-authentication-reader + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: system:cloud-controller-manager + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: + - "" + resources: + - nodes + verbs: + - '*' + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch + - apiGroups: + - "" + resources: + - services + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - services/status + verbs: + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - serviceaccounts + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - update + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - endpoints + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - watch + - update + - apiGroups: + - "" + resources: + - serviceaccounts/token + verbs: + - create + --- + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: system:cloud-controller-manager + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:cloud-controller-manager + subjects: + - apiGroup: "" + kind: ServiceAccount + name: cloud-controller-manager + namespace: kube-system +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: cloud-controller-manager-addon +--- +apiVersion: v1 +data: + aws-ebs-csi-external.yaml: |- + apiVersion: v1 + kind: Secret + metadata: + name: aws-secret + namespace: kube-system + stringData: + key_id: "" + access_key: "" + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-attacher-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - csi.storage.k8s.io + resources: + - csinodeinfos + verbs: + - get + - list + - watch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments/status + verbs: + - patch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-provisioner-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshots + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - csinodes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - watch + - list + - delete + - update + - create + - apiGroups: + - storage.k8s.io + resources: + - volumeattachments + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-resizer-role + rules: + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - persistentvolumeclaims/status + verbs: + - update + - patch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-external-snapshotter-role + rules: + - apiGroups: + - "" + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotclasses + verbs: + - get + - list + - watch + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents + verbs: + - create + - get + - list + - watch + - update + - delete + - apiGroups: + - snapshot.storage.k8s.io + resources: + - volumesnapshotcontents/status + verbs: + - update + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-attacher-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-attacher-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-provisioner-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-provisioner-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-resizer-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-resizer-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-snapshotter-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-external-snapshotter-role + subjects: + - kind: ServiceAccount + name: ebs-csi-controller-sa + namespace: kube-system + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + replicas: 2 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + containers: + - args: + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/csi/sockets/pluginproxy/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: key_id + name: aws-secret + optional: true + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: access_key + name: aws-secret + optional: true + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --feature-gates=Topology=true + - --extra-create-metadata + - --leader-election=true + - --default-fstype=ext4 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.2 + name: csi-provisioner + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.2 + name: csi-attacher + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --leader-election=true + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.2 + name: csi-snapshotter + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=$(ADDRESS) + - --v=2 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.2 + imagePullPolicy: Always + name: csi-resizer + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: socket-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + serviceAccountName: ebs-csi-controller-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + - key: node-role.kubernetes.io/master + effect: NoSchedule + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists + volumes: + - emptyDir: {} + name: socket-dir + --- + apiVersion: policy/v1 + kind: PodDisruptionBudget + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-controller + namespace: kube-system + spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ebs-csi-controller + app.kubernetes.io/name: aws-ebs-csi-driver + --- + apiVersion: apps/v1 + kind: DaemonSet + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + namespace: kube-system + spec: + selector: + matchLabels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + template: + metadata: + labels: + app: ebs-csi-node + app.kubernetes.io/name: aws-ebs-csi-driver + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + containers: + - args: + - node + - --endpoint=$(CSI_ENDPOINT) + - --logtostderr + - --v=2 + env: + - name: CSI_ENDPOINT + value: unix:/csi/csi.sock + - name: CSI_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: registry.k8s.io/provider-aws/aws-ebs-csi-driver:v1.25.0 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + name: ebs-plugin + ports: + - containerPort: 9808 + name: healthz + protocol: TCP + securityContext: + privileged: true + volumeMounts: + - mountPath: /var/lib/kubelet + mountPropagation: Bidirectional + name: kubelet-dir + - mountPath: /csi + name: plugin-dir + - mountPath: /dev + name: device-dir + - args: + - --csi-address=$(ADDRESS) + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --v=2 + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ebs.csi.aws.com/csi.sock + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.2 + name: node-driver-registrar + volumeMounts: + - mountPath: /csi + name: plugin-dir + - mountPath: /registration + name: registration-dir + - args: + - --csi-address=/csi/csi.sock + image: registry.k8s.io/sig-storage/livenessprobe:v2.11.0 + name: liveness-probe + volumeMounts: + - mountPath: /csi + name: plugin-dir + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-node-critical + serviceAccountName: ebs-csi-node-sa + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + tolerationSeconds: 300 + volumes: + - hostPath: + path: /var/lib/kubelet + type: Directory + name: kubelet-dir + - hostPath: + path: /var/lib/kubelet/plugins/ebs.csi.aws.com/ + type: DirectoryOrCreate + name: plugin-dir + - hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: Directory + name: registration-dir + - hostPath: + path: /dev + type: Directory + name: device-dir + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + type: RollingUpdate + --- + apiVersion: storage.k8s.io/v1 + kind: CSIDriver + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs.csi.aws.com + spec: + attachRequired: true + podInfoOnMount: false +kind: ConfigMap +metadata: + annotations: + note: generated + labels: + type: generated + name: aws-ebs-csi-driver-addon diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index d2ed5b83f7..48d7699a1d 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -4,7 +4,7 @@ metadata: labels: ccm: external csi: external - name: ${CLUSTER_NAME} + name: "${CLUSTER_NAME}" spec: clusterNetwork: pods: @@ -13,24 +13,24 @@ spec: controlPlaneRef: apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane - name: ${CLUSTER_NAME}-control-plane + name: "${CLUSTER_NAME}-control-plane" infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster - name: ${CLUSTER_NAME} + name: "${CLUSTER_NAME}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSCluster metadata: - name: ${CLUSTER_NAME} + name: "${CLUSTER_NAME}" spec: - region: ${AWS_REGION} - sshKeyName: ${AWS_SSH_KEY_NAME} + region: "${AWS_REGION}" + sshKeyName: "${AWS_SSH_KEY_NAME}" --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane metadata: - name: ${CLUSTER_NAME}-control-plane + name: "${CLUSTER_NAME}-control-plane" spec: kubeadmConfigSpec: clusterConfiguration: @@ -54,27 +54,27 @@ spec: infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-control-plane + name: "${CLUSTER_NAME}-control-plane" replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION} + version: "${KUBERNETES_VERSION}" --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: ${CLUSTER_NAME}-control-plane + name: "${CLUSTER_NAME}-control-plane" spec: template: spec: - iamInstanceProfile: control-plane.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_CONTROL_PLANE_MACHINE_TYPE} - sshKeyName: ${AWS_SSH_KEY_NAME} + iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" + instanceType: "${AWS_CONTROL_PLANE_MACHINE_TYPE}" + sshKeyName: "${AWS_SSH_KEY_NAME}" --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment metadata: - name: ${CLUSTER_NAME}-md-0 + name: "${CLUSTER_NAME}-md-0" spec: - clusterName: ${CLUSTER_NAME} + clusterName: "${CLUSTER_NAME}" replicas: ${WORKER_MACHINE_COUNT} selector: matchLabels: null @@ -84,29 +84,29 @@ spec: configRef: apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate - name: ${CLUSTER_NAME}-md-0 + name: "${CLUSTER_NAME}-md-0" clusterName: ${CLUSTER_NAME} infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate - name: ${CLUSTER_NAME}-md-0 + name: "${CLUSTER_NAME}-md-0" version: ${KUBERNETES_VERSION} --- apiVersion: infrastructure.cluster.x-k8s.io/v1beta2 kind: AWSMachineTemplate metadata: - name: ${CLUSTER_NAME}-md-0 + name: "${CLUSTER_NAME}-md-0" spec: template: spec: iamInstanceProfile: nodes.cluster-api-provider-aws.sigs.k8s.io - instanceType: ${AWS_NODE_MACHINE_TYPE} - sshKeyName: ${AWS_SSH_KEY_NAME} + instanceType: "${AWS_NODE_MACHINE_TYPE}" + sshKeyName: "${AWS_SSH_KEY_NAME}" --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate metadata: - name: ${CLUSTER_NAME}-md-0 + name: "${CLUSTER_NAME}-md-0" spec: template: spec: @@ -263,9 +263,6 @@ data: - serviceaccounts verbs: - create - - get - - list - - watch - apiGroups: - "" resources: @@ -275,6 +272,13 @@ data: - list - update - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch - apiGroups: - "" resources: @@ -403,6 +407,27 @@ data: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - patch + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole metadata: labels: app.kubernetes.io/name: aws-ebs-csi-driver @@ -664,6 +689,21 @@ data: name: ebs-csi-controller-sa namespace: kube-system --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + labels: + app.kubernetes.io/name: aws-ebs-csi-driver + name: ebs-csi-node-binding + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ebs-csi-node + subjects: + - kind: ServiceAccount + name: ebs-csi-node-sa + namespace: kube-system + --- apiVersion: apps/v1 kind: Deployment metadata: