Skip to content

Commit b0cb16c

Browse files
nkkashyapnkkashyap
authored
in IBM Cloud create SCs for IBM COS (#34)
* create SCs for IBM COS Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * remove sc suffix from SCs Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * fixed sc naming issue Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * fixed sc naming issue Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * create SCs for IBM COS Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * create SCs for IBM COS Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * create SCs for IBM COS Signed-off-by: Neeraj Kumar Kashyap <[email protected]> * create SCs for IBM COS Signed-off-by: Neeraj Kumar Kashyap <[email protected]> --------- Signed-off-by: Neeraj Kumar Kashyap <[email protected]>
1 parent ee26586 commit b0cb16c

File tree

7 files changed

+247
-28
lines changed

7 files changed

+247
-28
lines changed

‎controllers/constants/constants.go

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -71,17 +71,20 @@ const (
7171
CSINodeSCCClusterRole = "node-scc-clusterrole"
7272
CSINodeSCCClusterRoleBinding = "node-scc-clusterrolebinding"
7373

74+
75+
StorageClassNamePrefix = "ibm-object-storage"
76+
RetainPolicyTag = "retain"
77+
7478
ResourceReqLimitsConfigMap = "cos-csi-driver-configmap"
7579
ObjectCSIDriverOperatorDeployNS = "ibm-object-csi-operator"
7680
ObjectCSIDriver = "ibm-object-csi"
7781

78-
StorageClassPrefix = "ibm-object-storage-"
79-
StorageClassSuffix = "-sc"
82+
StorageClassPrefix = "ibm-object-storage"
8083

81-
RcloneRetainStorageClass = StorageClassPrefix + "rclone-retain" + StorageClassSuffix
82-
RcloneStorageClass = StorageClassPrefix + "rclone" + StorageClassSuffix
83-
S3fsRetainStorageClass = StorageClassPrefix + "s3fs-retain" + StorageClassSuffix
84-
S3fsStorageClass = StorageClassPrefix + "s3fs" + StorageClassSuffix
84+
RcloneRetainStorageClass = StorageClassPrefix + "-rclone-retain"
85+
RcloneStorageClass = StorageClassPrefix + "-rclone"
86+
S3fsRetainStorageClass = StorageClassPrefix + "-s3fs-retain"
87+
S3fsStorageClass = StorageClassPrefix + "-s3fs"
8588

8689
DefaultLogTailLines = 300
8790
DefaultNamespace = "default"

‎controllers/ibmobjectcsi_controller.go

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ func (r *IBMObjectCSIReconciler) Reconcile(ctx context.Context, req ctrl.Request
160160
r.reconcileServiceAccount,
161161
r.reconcileClusterRole,
162162
r.reconcileClusterRoleBinding,
163-
r.reconcileStorageClasses,
163+
// r.reconcileStorageClasses,
164164
} {
165165
if err = rec(instance); err != nil {
166166
return reconcile.Result{}, err
@@ -178,6 +178,11 @@ func (r *IBMObjectCSIReconciler) Reconcile(ctx context.Context, req ctrl.Request
178178
return reconcile.Result{}, err
179179
}
180180

181+
// TODO: Reconcile SC Check. First process RC, User may provide S3Provider in RC.
182+
if err = r.reconcileStorageClasses(instance); err != nil {
183+
return reconcile.Result{}, err
184+
}
185+
181186
if err := r.updateStatus(instance, originalStatus); err != nil {
182187
return reconcile.Result{}, err
183188
}
@@ -540,18 +545,44 @@ func (r *IBMObjectCSIReconciler) getClusterRoleBindings(instance *crutils.IBMObj
540545
}
541546

542547
func (r *IBMObjectCSIReconciler) getStorageClasses(instance *crutils.IBMObjectCSI) []*storagev1.StorageClass {
543-
rcloneRetainSC := instance.GenerateRcloneSC(constants.RcloneRetainStorageClass, corev1.PersistentVolumeReclaimRetain)
544-
rcloneSC := instance.GenerateRcloneSC(constants.RcloneStorageClass, corev1.PersistentVolumeReclaimDelete)
545548

546-
s3fsRetainSC := instance.GenerateS3fsSC(constants.S3fsRetainStorageClass, corev1.PersistentVolumeReclaimRetain)
547-
s3fsSC := instance.GenerateS3fsSC(constants.S3fsStorageClass, corev1.PersistentVolumeReclaimDelete)
549+
cosRegion := r.ControllerHelper.GetRegion()
550+
cosEP := r.ControllerHelper.GetCosEP()
551+
s3Provider := r.ControllerHelper.GetS3Provider()
552+
scNamePrefix := constants.StorageClassNamePrefix
553+
554+
k8sSCs := []*storagev1.StorageClass{}
555+
cosSCs := []string{}
556+
557+
reclaimPolicys := []corev1.PersistentVolumeReclaimPolicy{
558+
corev1.PersistentVolumeReclaimRetain,
559+
corev1.PersistentVolumeReclaimDelete}
560+
561+
isIBMCloud := r.ControllerHelper.IsIBMColud()
562+
if isIBMCloud && (len(s3Provider) == 0 || s3Provider == "ibm-cos") {
563+
cosEP = r.ControllerHelper.GetIBMCosEP()
564+
cosSCs = r.ControllerHelper.GetIBMCosSC()
565+
// for _, sc := range cosSC {
566+
// ibmCosSC := fmt.Sprintf("%s-%s", cosRegion, sc)
567+
// cosSCs = append(cosSCs, ibmCosSC)
568+
// }
548569

549-
return []*storagev1.StorageClass{
550-
rcloneRetainSC,
551-
rcloneSC,
552-
s3fsRetainSC,
553-
s3fsSC,
570+
} else {
571+
cosSCs = append(cosSCs, "standard")
572+
}
573+
574+
for _, sc := range cosSCs {
575+
for _, rp := range reclaimPolicys {
576+
k8sSc := instance.GenerateRcloneSC(scNamePrefix, rp,
577+
isIBMCloud, cosRegion, cosEP, sc)
578+
k8sSCs = append(k8sSCs, k8sSc)
579+
580+
k8sSc = instance.GenerateS3fsSC(scNamePrefix, rp,
581+
isIBMCloud, cosRegion, cosEP, sc)
582+
k8sSCs = append(k8sSCs, k8sSc)
583+
}
554584
}
585+
return k8sSCs
555586
}
556587

557588
func (r *IBMObjectCSIReconciler) deleteClusterRoles(instance *crutils.IBMObjectCSI) error {

‎controllers/ibmobjectcsi_controller_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package controllers
22

33
import (
44
"errors"
5+
"fmt"
56
"testing"
67

78
"github.com/IBM/ibm-object-csi-driver-operator/api/v1alpha1"
@@ -342,7 +343,7 @@ var (
342343

343344
rCloneSC = &storagev1.StorageClass{
344345
ObjectMeta: metav1.ObjectMeta{
345-
Name: constants.RcloneStorageClass,
346+
Name: fmt.Sprintf("%s-standard-rclone", constants.StorageClassNamePrefix),
346347
},
347348
Provisioner: constants.DriverName,
348349
ReclaimPolicy: &reclaimPolicyDelete,
@@ -369,7 +370,7 @@ var (
369370

370371
rCloneRetainSC = &storagev1.StorageClass{
371372
ObjectMeta: metav1.ObjectMeta{
372-
Name: constants.RcloneRetainStorageClass,
373+
Name: fmt.Sprintf("%s-standard-rclone-retain", constants.StorageClassNamePrefix),
373374
},
374375
Provisioner: constants.DriverName,
375376
ReclaimPolicy: &reclaimPolicyRetain,
@@ -396,7 +397,7 @@ var (
396397

397398
s3fsSC = &storagev1.StorageClass{
398399
ObjectMeta: metav1.ObjectMeta{
399-
Name: constants.S3fsStorageClass,
400+
Name: fmt.Sprintf("%s-standard-s3fs", constants.StorageClassNamePrefix),
400401
},
401402
Provisioner: constants.DriverName,
402403
ReclaimPolicy: &reclaimPolicyDelete,
@@ -420,7 +421,7 @@ var (
420421

421422
s3fsRetainSC = &storagev1.StorageClass{
422423
ObjectMeta: metav1.ObjectMeta{
423-
Name: constants.S3fsRetainStorageClass,
424+
Name: fmt.Sprintf("%s-standard-s3fs-retain", constants.StorageClassNamePrefix),
424425
},
425426
Provisioner: constants.DriverName,
426427
ReclaimPolicy: &reclaimPolicyRetain,

‎controllers/internal/crutils/static_resource_generator.go

Lines changed: 66 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
package crutils
33

44
import (
5+
"fmt"
6+
57
corev1 "k8s.io/api/core/v1"
68
rbacv1 "k8s.io/api/rbac/v1"
79
storagev1 "k8s.io/api/storage/v1"
@@ -215,7 +217,34 @@ func (c *IBMObjectCSI) GenerateSCCForNodeClusterRoleBinding() *rbacv1.ClusterRol
215217
}
216218

217219
// Generates3fsSC ...
218-
func (c *IBMObjectCSI) GenerateS3fsSC(storageClassName string, reclaimPolicy corev1.PersistentVolumeReclaimPolicy) *storagev1.StorageClass {
220+
func (c *IBMObjectCSI) GenerateS3fsSC(storageClassNamePrefix string,
221+
reclaimPolicy corev1.PersistentVolumeReclaimPolicy, isIBMColud bool,
222+
region string, cosEndpoint string, cosStorageClass string) *storagev1.StorageClass {
223+
// TODO: TIER Based SC
224+
var storageClassName string
225+
var cosEP string = ""
226+
var cosSC string = "standard"
227+
228+
if len(cosEndpoint) > 0 {
229+
cosEP = cosEndpoint
230+
}
231+
if len(cosStorageClass) > 0 {
232+
cosSC = cosStorageClass
233+
}
234+
235+
if reclaimPolicy == "Retain" {
236+
// "ibm-object-storage-standard-s3fs-retain"
237+
storageClassName = fmt.Sprintf("%s-%s-s3fs-%s", storageClassNamePrefix, cosSC, constants.RetainPolicyTag)
238+
} else {
239+
// "ibm-object-storage-standard-s3fs"
240+
storageClassName = fmt.Sprintf("%s-%s-s3fs", storageClassNamePrefix, cosSC)
241+
}
242+
243+
if isIBMColud == true {
244+
ibmCosSC := fmt.Sprintf("%s-%s", region, cosSC)
245+
cosSC = ibmCosSC
246+
}
247+
219248
return &storagev1.StorageClass{
220249
ObjectMeta: metav1.ObjectMeta{
221250
Name: storageClassName,
@@ -232,8 +261,10 @@ func (c *IBMObjectCSI) GenerateS3fsSC(storageClassName string, reclaimPolicy cor
232261
"kernel_cache",
233262
},
234263
Parameters: map[string]string{
235-
"mounter": "s3fs",
236-
"client": "awss3",
264+
"mounter": "s3fs",
265+
"client": "awss3",
266+
"cosEndpoint": cosEP,
267+
"locationConstraint": cosSC,
237268
"csi.storage.k8s.io/provisioner-secret-name": "${pvc.name}",
238269
"csi.storage.k8s.io/provisioner-secret-namespace": "${pvc.namespace}",
239270
"csi.storage.k8s.io/node-publish-secret-name": "${pvc.name}",
@@ -243,7 +274,34 @@ func (c *IBMObjectCSI) GenerateS3fsSC(storageClassName string, reclaimPolicy cor
243274
}
244275

245276
// GenerateRcloneSC ...
246-
func (c *IBMObjectCSI) GenerateRcloneSC(storageClassName string, reclaimPolicy corev1.PersistentVolumeReclaimPolicy) *storagev1.StorageClass {
277+
func (c *IBMObjectCSI) GenerateRcloneSC(storageClassNamePrefix string,
278+
reclaimPolicy corev1.PersistentVolumeReclaimPolicy, isIBMColud bool,
279+
region string, cosEndpoint string, cosStorageClass string) *storagev1.StorageClass {
280+
281+
var storageClassName string
282+
var cosEP string = ""
283+
var cosSC string = "standard"
284+
285+
if len(cosEndpoint) > 0 {
286+
cosEP = cosEndpoint
287+
}
288+
if len(cosStorageClass) > 0 {
289+
cosSC = cosStorageClass
290+
}
291+
292+
if reclaimPolicy == "Retain" {
293+
// "ibm-object-storage-standard-rclone-retain"
294+
storageClassName = fmt.Sprintf("%s-%s-rclone-%s", storageClassNamePrefix, cosSC, constants.RetainPolicyTag)
295+
} else {
296+
// "ibm-object-storage-standard-rclone"
297+
storageClassName = fmt.Sprintf("%s-%s-rclone", storageClassNamePrefix, cosSC)
298+
}
299+
300+
if isIBMColud == true {
301+
ibmCosSC := fmt.Sprintf("%s-%s", region, cosSC)
302+
cosSC = ibmCosSC
303+
}
304+
247305
return &storagev1.StorageClass{
248306
ObjectMeta: metav1.ObjectMeta{
249307
Name: storageClassName,
@@ -263,8 +321,10 @@ func (c *IBMObjectCSI) GenerateRcloneSC(storageClassName string, reclaimPolicy c
263321
"disable_checksum=true",
264322
},
265323
Parameters: map[string]string{
266-
"mounter": "rclone",
267-
"client": "awss3",
324+
"mounter": "rclone",
325+
"client": "awss3",
326+
"cosEndpoint": cosEP,
327+
"locationConstraint": cosSC,
268328
"csi.storage.k8s.io/provisioner-secret-name": "${pvc.name}",
269329
"csi.storage.k8s.io/provisioner-secret-namespace": "${pvc.namespace}",
270330
"csi.storage.k8s.io/node-publish-secret-name": "${pvc.name}",

‎controllers/recoverstalevolume_controller.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -301,7 +301,7 @@ func fetchCSIPVCAndPVNames(k8sOps *crutils.K8sResourceOps, log logr.Logger) (map
301301
continue
302302
}
303303

304-
if strings.HasPrefix(scName, constants.StorageClassPrefix) && strings.HasSuffix(scName, constants.StorageClassSuffix) {
304+
if strings.HasPrefix(scName, constants.StorageClassPrefix) {
305305
reqData[pvc.Name] = pvc.Spec.VolumeName
306306
}
307307
}

‎controllers/util/common/common.go

Lines changed: 104 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,20 +10,30 @@ import (
1010
"github.com/IBM/ibm-object-csi-driver-operator/controllers/internal/crutils"
1111
"github.com/IBM/ibm-object-csi-driver-operator/controllers/util"
1212
"github.com/go-logr/logr"
13+
corev1 "k8s.io/api/core/v1"
1314
rbacv1 "k8s.io/api/rbac/v1"
1415
storagev1 "k8s.io/api/storage/v1"
1516
"k8s.io/apimachinery/pkg/api/errors"
1617
"k8s.io/apimachinery/pkg/api/meta"
1718
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1819
"k8s.io/apimachinery/pkg/types"
20+
"k8s.io/client-go/kubernetes"
1921
"sigs.k8s.io/controller-runtime/pkg/client"
2022
"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
2123
)
2224

25+
var iaasIBMVPC = "ibm-vpc"
26+
var iaasIBMClassic = "ibm-classic"
27+
var cosIBMProvider = "ibm-cos"
28+
2329
// ControllerHelper ...
2430
type ControllerHelper struct {
2531
client.Client
26-
Log logr.Logger
32+
Log logr.Logger
33+
Region string
34+
CosEP string
35+
IaaSProvider string
36+
S3Provider string // IBM COS / AWS S3 / Wasabi
2737
}
2838

2939
// NewControllerHelper ...
@@ -243,3 +253,96 @@ func (ch *ControllerHelper) getAccessorAndFinalizerName(instance crutils.Instanc
243253
}
244254
return accessor, finalizerName, nil
245255
}
256+
257+
// Check the platform, if IBM Cloud then get Region and IaaS provider
258+
func (ch *ControllerHelper) GetIBMClusterInfo(clientset *kubernetes.Clientset) error {
259+
var listOptions = &client.ListOptions{}
260+
var err error
261+
nodes := corev1.NodeList{}
262+
263+
logger := ch.Log.WithName("getClusterInfo")
264+
265+
logger.Info("Checking cluster platform...")
266+
267+
if clientset != nil {
268+
list, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{})
269+
if err == nil {
270+
nodes = *list
271+
}
272+
} else {
273+
err = ch.List(context.TODO(), &nodes, listOptions)
274+
}
275+
if err != nil {
276+
logger.Error(err, "Get Cluster Info")
277+
return err
278+
}
279+
280+
logger.Info("Get cluster region...")
281+
if val, ok := nodes.Items[0].Labels["ibm-cloud.kubernetes.io/region"]; ok {
282+
ch.Region = val
283+
logger.Info("Detected IBM Cluster region: ", ch.Region)
284+
}
285+
286+
logger.Info("Get cluster IaaS Provider...")
287+
if val, ok := nodes.Items[0].Labels["ibm-cloud.kubernetes.io/iaas-provider"]; ok {
288+
logger.Info("Detected IBM IaaS provider: ", val)
289+
// ch.S3Provider = &cosIBMProvider Do not set Provider here user may specify S3 Provider in CR
290+
if val == "g2" {
291+
ch.IaaSProvider = iaasIBMVPC
292+
} else {
293+
ch.IaaSProvider = iaasIBMClassic
294+
}
295+
logger.Info("Detected endpoint type: ", ch.IaaSProvider)
296+
}
297+
return nil
298+
}
299+
300+
func (ch *ControllerHelper) GetS3Provider() string {
301+
return ch.S3Provider
302+
}
303+
304+
func (ch *ControllerHelper) GetIaaSProvider() string {
305+
return ch.IaaSProvider
306+
}
307+
308+
func (ch *ControllerHelper) GetRegion() string {
309+
return ch.Region
310+
}
311+
312+
func (ch *ControllerHelper) GetCosEP() string {
313+
return ch.CosEP
314+
}
315+
316+
func (ch *ControllerHelper) IsIBMColud() bool {
317+
retVal := false
318+
if len(ch.IaaSProvider) == 0 || len(ch.Region) == 0 {
319+
return retVal
320+
}
321+
if ch.IaaSProvider == iaasIBMVPC || ch.IaaSProvider == iaasIBMClassic {
322+
retVal = true
323+
}
324+
return retVal
325+
}
326+
327+
func (ch *ControllerHelper) GetIBMCosSC() []string {
328+
if len(ch.IaaSProvider) == 0 || len(ch.Region) == 0 {
329+
return make([]string, 0)
330+
}
331+
cosSC := []string{"standard", "smart"}
332+
return cosSC
333+
}
334+
335+
func (ch *ControllerHelper) GetIBMCosEP() string {
336+
var cosEP string
337+
if len(ch.IaaSProvider) == 0 || len(ch.Region) == 0 {
338+
return cosEP
339+
}
340+
if ch.IaaSProvider == iaasIBMVPC || ch.IaaSProvider == iaasIBMClassic {
341+
epType := "private"
342+
if ch.IaaSProvider == iaasIBMVPC {
343+
epType = "direct"
344+
}
345+
cosEP = fmt.Sprintf("https://s3.%s.%s.cloud-object-storage.appdomain.cloud", epType, ch.Region)
346+
}
347+
return cosEP
348+
}

0 commit comments

Comments
 (0)