Implement automatic topology awareness for Manila share provisioning (#2255)

Author: kayrus

docs/manila-csi-plugin/using-manila-csi-plugin.md

@@ -50,6 +50,7 @@ Parameter | Required | Description
 `type` | _yes_ | Manila [share type](https://wiki.openstack.org/wiki/Manila/Concepts#share_type)
 `shareNetworkID` | _no_ | Manila [share network ID](https://wiki.openstack.org/wiki/Manila/Concepts#share_network)
 `availability` | _no_ | Manila availability zone of the provisioned share. If none is provided, the default Manila zone will be used. Note that this parameter is opaque to the CO and does not influence placement of workloads that will consume this share, meaning they may be scheduled onto any node of the cluster. If the specified Manila AZ is not equally accessible from all compute nodes of the cluster, use [Topology-aware dynamic provisioning](#topology-aware-dynamic-provisioning).
+`autoTopology` | _no_ | When set to "true" and the `availability` parameter is empty, the Manila CSI controller will map the Manila availability zone to the target compute node availability zone.
 `appendShareMetadata` | _no_ | Append user-defined metadata to the provisioned share. If not empty, this field must be a string with a valid JSON object. The object must consist of key-value pairs of type string. Example: `"{..., \"key\": \"value\"}"`.
 `cephfs-mounter` | _no_ | Relevant for CephFS Manila shares. Specifies which mounting method to use with the CSI CephFS driver. Available options are `kernel` and `fuse`, defaults to `fuse`. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information.
 `cephfs-kernelMountOptions` | _no_ | Relevant for CephFS Manila shares. Specifies mount options for CephFS kernel client. See [CSI CephFS docs](https://github.com/ceph/ceph-csi/blob/csi-v1.0/docs/deploy-cephfs.md#configuration) for further information.
@@ -130,6 +131,35 @@ Storage AZ does not influence
           Shares in zone-a are accessible only from nodes in nova-1 and nova-2.
 ```
 
+In cases when the Manila availability zone must correspond to the Nova
+availability zone, you can set the `autoTopology: "true"` along with the
+`volumeBindingMode: WaitForFirstConsumer` and omit the `availability`
+parameter. By doing so, the share will be provisioned in the target compute
+node availability zone.
+
+```
+                          Auto topology-aware storage class example:
+
+
+           Both Compute and Storage AZs influence the placement of workloads.
+
+        +-----------+                                         +---------------+
+        | Manila AZ |                                         |  Compute AZs  |
+        |   zone-1  |    apiVersion: storage.k8s.io/v1        |     zone-1    |
+        |   zone-2  |    kind: StorageClass                   |     zone-2    |
+        +-----------+    metadata:                            +---------------+
+              |            name: nfs-gold                             |
+              |          provisioner: nfs.manila.csi.openstack.org    |
+              |          parameters:                                  |
+              +---------+  autoTopology: "true"  +--------------------+
+                           ...
+                         volumeBindingMode: WaitForFirstConsumer
+                           ...
+
+Shares for workloads in zone-1 will be created in zone-1 and accessible only from nodes in zone-1.
+Shares for workloads in zone-2 will be created in zone-2 and accessible only from nodes in zone-2.
+```
+
 [Enabling topology awareness in Kubernetes](#enabling-topology-awareness)
 
 ### Runtime configuration file

examples/manila-csi-plugin/nfs/auto-topology-aware/pod.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: new-nfs-share-pod
+spec:
+  containers:
+    - name: web-server
+      image: nginx
+      imagePullPolicy: IfNotPresent
+      volumeMounts:
+        - name: mypvc
+          mountPath: /var/lib/www
+  nodeSelector:
+    topology.kubernetes.io/zone: zone-1
+  volumes:
+    - name: mypvc
+      persistentVolumeClaim:
+        claimName: new-nfs-share-pvc
+        readOnly: false

examples/manila-csi-plugin/nfs/auto-topology-aware/pvc.yaml

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: new-nfs-share-pvc
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-manila-nfs

examples/manila-csi-plugin/nfs/auto-topology-aware/storageclass.yaml

@@ -0,0 +1,29 @@
+# Topology constraints example:
+#
+# Let's have two Manila AZs: zone-{1..2}
+# Let's have six Nova AZs: zone-{1..6}
+#
+# Manila zone-1 is accessible from nodes in zone-1 only
+# Manila zone-2 is accessible from nodes in zone-2 only
+#
+# We're provisioning into zone-1
+# availability parameter and allowedTopologies are empty, therefore the dynamic
+# share provisioning with automatic availability zone selection takes place.
+# The "volumeBindingMode" must be set to "WaitForFirstConsumer".
+
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-manila-nfs
+provisioner: nfs.manila.csi.openstack.org
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true
+parameters:
+  type: default
+  autoTopology: "true"
+  csi.storage.k8s.io/provisioner-secret-name: csi-manila-secrets
+  csi.storage.k8s.io/provisioner-secret-namespace: default
+  csi.storage.k8s.io/node-stage-secret-name: csi-manila-secrets
+  csi.storage.k8s.io/node-stage-secret-namespace: default
+  csi.storage.k8s.io/node-publish-secret-name: csi-manila-secrets
+  csi.storage.k8s.io/node-publish-secret-namespace: default

examples/manila-csi-plugin/nfs/dynamic-provisioning/pvc.yaml

@@ -9,4 +9,3 @@ spec:
     requests:
       storage: 1Gi
   storageClassName: csi-manila-nfs
-

examples/manila-csi-plugin/nfs/topology-aware/storageclass.yaml

@@ -24,6 +24,7 @@ parameters:
   csi.storage.k8s.io/node-stage-secret-namespace: default
   csi.storage.k8s.io/node-publish-secret-name: csi-manila-secrets
   csi.storage.k8s.io/node-publish-secret-namespace: default
+allowVolumeExpansion: true
 allowedTopologies:
   - matchLabelExpressions:
     - key: topology.manila.csi.openstack.org/zone

pkg/csi/cinder/controllerserver.go

@@ -69,16 +69,13 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 	// Volume Type
 	volType := req.GetParameters()["type"]
 
-	var volAvailability string
-
 	// First check if volAvailability is already specified, if not get preferred from Topology
 	// Required, incase vol AZ is different from node AZ
-	volAvailability = req.GetParameters()["availability"]
-
-	if len(volAvailability) == 0 {
+	volAvailability := req.GetParameters()["availability"]
+	if volAvailability == "" {
 		// Check from Topology
 		if req.GetAccessibilityRequirements() != nil {
-			volAvailability = getAZFromTopology(req.GetAccessibilityRequirements())
+			volAvailability = util.GetAZFromTopology(topologyKey, req.GetAccessibilityRequirements())
 		}
 	}
 
@@ -651,23 +648,6 @@ func (cs *controllerServer) ControllerExpandVolume(ctx context.Context, req *csi
 	}, nil
 }
 
-func getAZFromTopology(requirement *csi.TopologyRequirement) string {
-	for _, topology := range requirement.GetPreferred() {
-		zone, exists := topology.GetSegments()[topologyKey]
-		if exists {
-			return zone
-		}
-	}
-
-	for _, topology := range requirement.GetRequisite() {
-		zone, exists := topology.GetSegments()[topologyKey]
-		if exists {
-			return zone
-		}
-	}
-	return ""
-}
-
 func getCreateVolumeResponse(vol *volumes.Volume, ignoreVolumeAZ bool, accessibleTopologyReq *csi.TopologyRequirement) *csi.CreateVolumeResponse {
 
 	var volsrc *csi.VolumeContentSource

pkg/csi/manila/controllerserver.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila/capabilities"
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila/options"
 	"k8s.io/cloud-provider-openstack/pkg/csi/manila/shareadapters"
+	"k8s.io/cloud-provider-openstack/pkg/util"
 	clouderrors "k8s.io/cloud-provider-openstack/pkg/util/errors"
 	"k8s.io/klog/v2"
 )
@@ -54,7 +55,7 @@ var (
 	}
 )
 
-func getVolumeCreator(source *csi.VolumeContentSource, shareOpts *options.ControllerVolumeContext, compatOpts *options.CompatibilityOptions) (volumeCreator, error) {
+func getVolumeCreator(source *csi.VolumeContentSource) (volumeCreator, error) {
 	if source == nil {
 		return &blankVolume{}, nil
 	}
@@ -135,9 +136,25 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 
 	sizeInGiB := bytesToGiB(requestedSize)
 
+	var accessibleTopology []*csi.Topology
+	accessibleTopologyReq := req.GetAccessibilityRequirements()
+	if cs.d.withTopology && accessibleTopologyReq != nil {
+		// All requisite/preferred topologies are considered valid. Nodes from those zones are required to be able to reach the storage.
+		// The operator is responsible for making sure that provided topology keys are valid and present on the nodes of the cluster.
+		accessibleTopology = accessibleTopologyReq.GetPreferred()
+
+		// When "autoTopology" is enabled and "availability" is empty, obtain the AZ from the target node.
+		if shareOpts.AvailabilityZone == "" && strings.EqualFold(shareOpts.AutoTopology, "true") {
+			shareOpts.AvailabilityZone = util.GetAZFromTopology(topologyKey, accessibleTopologyReq)
+			accessibleTopology = []*csi.Topology{{
+				Segments: map[string]string{topologyKey: shareOpts.AvailabilityZone},
+			}}
+		}
+	}
+
 	// Retrieve an existing share or create a new one
 
-	volCreator, err := getVolumeCreator(req.GetVolumeContentSource(), shareOpts, cs.d.compatOpts)
+	volCreator, err := getVolumeCreator(req.GetVolumeContentSource())
 	if err != nil {
 		return nil, err
 	}
@@ -147,7 +164,8 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, err
 	}
 
-	if err = verifyVolumeCompatibility(sizeInGiB, req, share, shareOpts, cs.d.compatOpts, shareTypeCaps); err != nil {
+	err = verifyVolumeCompatibility(sizeInGiB, req, share, shareOpts, cs.d.compatOpts, shareTypeCaps)
+	if err != nil {
 		return nil, status.Errorf(codes.AlreadyExists, "volume %s already exists, but is incompatible with the request: %v", req.GetName(), err)
 	}
 
@@ -164,13 +182,6 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
 		return nil, status.Errorf(codes.Internal, "failed to grant access to volume %s: %v", share.Name, err)
 	}
 
-	var accessibleTopology []*csi.Topology
-	if cs.d.withTopology {
-		// All requisite/preferred topologies are considered valid. Nodes from those zones are required to be able to reach the storage.
-		// The operator is responsible for making sure that provided topology keys are valid and present on the nodes of the cluster.
-		accessibleTopology = req.GetAccessibilityRequirements().GetPreferred()
-	}
-
 	volCtx := filterParametersForVolumeContext(params, options.NodeVolumeContextFields())
 	volCtx["shareID"] = share.ID
 	volCtx["shareAccessID"] = accessRight.ID

pkg/csi/manila/options/shareoptions.go

@@ -24,6 +24,7 @@ type ControllerVolumeContext struct {
 	Protocol            string `name:"protocol" matches:"^(?i)CEPHFS|NFS$"`
 	Type                string `name:"type" value:"default:default"`
 	ShareNetworkID      string `name:"shareNetworkID" value:"optional"`
+	AutoTopology        string `name:"autoTopology" value:"default:false" matches:"(?i)^true|false$"`
 	AvailabilityZone    string `name:"availability" value:"optional"`
 	AppendShareMetadata string `name:"appendShareMetadata" value:"optional"`
 

pkg/util/util.go

@@ -6,12 +6,14 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/container-storage-interface/spec/lib/go/csi"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/klog/v2"
 )
 
 // MyDuration is the encoding.TextUnmarshaler interface for time.Duration
@@ -100,3 +102,28 @@ func PatchService(ctx context.Context, client clientset.Interface, cur, mod *v1.
 
 	return nil
 }
+
+func GetAZFromTopology(topologyKey string, requirement *csi.TopologyRequirement) string {
+	var zone string
+	var exists bool
+
+	defer func() { klog.V(1).Infof("detected AZ from the topology: %s", zone) }()
+	klog.V(4).Infof("preferred topology requirement: %+v", requirement.GetPreferred())
+	klog.V(4).Infof("requisite topology requirement: %+v", requirement.GetRequisite())
+
+	for _, topology := range requirement.GetPreferred() {
+		zone, exists = topology.GetSegments()[topologyKey]
+		if exists {
+			return zone
+		}
+	}
+
+	for _, topology := range requirement.GetRequisite() {
+		zone, exists = topology.GetSegments()[topologyKey]
+		if exists {
+			return zone
+		}
+	}
+
+	return zone
+}