feat(tenant): Support S3 Storage in Tenant

config/rbac/role.yaml

@@ -4,6 +4,13 @@ kind: ClusterRole
 metadata:
   name: manager-role
 rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
 - apiGroups:
   - databendlabs.io
   resources:

go.mod

@@ -3,6 +3,7 @@ module github.com/databendcloud/databend-operator
 go 1.22.0
 
 require (
+	github.com/aws/aws-sdk-go v1.55.5
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	k8s.io/api v0.31.0
@@ -47,6 +48,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
 	github.com/imdario/mergo v0.3.6 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

go.sum

@@ -2,6 +2,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
 github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
+github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
+github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -70,6 +72,10 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=

pkg/apis/databendlabs.io/v1alpha1/tenant_types.go

@@ -31,6 +31,35 @@ const (
 	NoPassword UserAuthType = "no_password"
 )
 
+const (
+	// TenantCreated means that the tenant creation has succeeded.
+	TenantCreated string = "Created"
+
+	// TenantError means that the tenant reaches an error state.
+	TenantError string = "Error"
+)
+
+const (
+	// TenantCreationSucceededReason is the "Created" condition reason.
+	// When the tenant creation succeeded, this is added.
+	TenantCreationSucceededReason = "TenantCreationSucceeded"
+
+	// TenantStorageErrorReason is the "Error" condition reason.
+	// When the tenant has an error storage configurations and reaches an error state,
+	// this is added.
+	TenantStorageErrorReason = "TenantStorageError"
+
+	// TenantMetaErrorReason is the "Error" condition reason.
+	// When the tenant has an error meta configurations and reaches an error state,
+	// this is added.
+	TenantMetaErrorReason = "TenantMetaError"
+
+	// TenantUserErrorReason is the "Error" condition reason.
+	// When the tenant has an error built-in user configurations and reaches an error state,
+	// this is added.
+	TenantUserErrorReason = "TenantUserError"
+)
+
 type Storage struct {
 	// Specification of S3 storage.
 	S3 *S3Storage `json:"s3,omitempty"`

pkg/common/const.go

@@ -0,0 +1,19 @@
+package common
+
+const (
+	// TenantCreationSucceededMessage is status condition message for the
+	// {"type": "Created", "status": "True", "reason": "TenantCreationSucceeded"} condition.
+	TenantCreationSucceededMessage = "Succeeded to create Tenant"
+
+	// TenantStorageErrorMessage is status condition message for the
+	// {"type": "Error", "status": "False", "reason": "TenantStorageError"} condition.
+	TenantStorageErrorMessage = "Invalid storage configuration"
+
+	// TenantStorageErrorMessage is status condition message for the
+	// {"type": "Error", "status": "False", "reason": "TenantMetaError"} condition.
+	TenantMetaErrorMessage = "Invalid meta configuration"
+
+	// TenantStorageErrorMessage is status condition message for the
+	// {"type": "Error", "status": "False", "reason": "TenantUserError"} condition.
+	TenantUserErrorMessage = "Invalid built-in user configurations"
+)

pkg/controller/tenant/tenant_controller.go

@@ -18,13 +18,34 @@ package tenant
 
 import (
 	"context"
+	"errors"
+	"fmt"
 
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog/v2"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/log"
 
 	databendv1alpha1 "github.com/databendcloud/databend-operator/pkg/apis/databendlabs.io/v1alpha1"
+	"github.com/databendcloud/databend-operator/pkg/common"
+)
+
+type opState int
+
+const (
+	creationSucceeded opState = iota
+	storageError      opState = iota
+	metaError         opState = iota
+	builtinUserError  opState = iota
 )
 
 // TenantReconciler reconciles a Tenant object
@@ -36,22 +57,138 @@ type TenantReconciler struct {
 // +kubebuilder:rbac:groups=databendlabs.io,resources=tenants,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=databendlabs.io,resources=tenants/status,verbs=get;update;patch
 // +kubebuilder:rbac:groups=databendlabs.io,resources=tenants/finalizers,verbs=update
+// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list
 
-// Reconcile is part of the main kubernetes reconciliation loop which aims to
-// move the current state of the cluster closer to the desired state.
-// TODO(user): Modify the Reconcile function to compare the state specified by
-// the Tenant object against the actual cluster state, and then
-// perform operations to make the cluster state reflect the state specified by
-// the user.
-//
-// For more details, check Reconcile and its Result here:
-// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
 func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
-	_ = log.FromContext(ctx)
+	var tenant databendv1alpha1.Tenant
+	if err := r.Get(ctx, req.NamespacedName, &tenant); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+	log := ctrl.LoggerFrom(ctx).WithValues("tenant", klog.KObj(&tenant))
+	ctx = ctrl.LoggerInto(ctx, log)
+	log.V(2).Info("Reconciling Tenant")
+
+	var err error
+	originStatus := tenant.Status.DeepCopy()
+
+	// Verify storage configuration
+	opState, storageErr := r.verifyStorage(ctx, &tenant)
+	if storageErr != nil {
+		err = errors.Join(err, storageErr)
+	}
+	log.V(5).Info("Succeeded to verify storage configurations")
+	setCondition(&tenant, opState)
+
+	// Verify meta configuration
+	opState, metaErr := r.verifyMeta(ctx, &tenant)
+	if metaErr != nil {
+		err = errors.Join(err, metaErr)
+	}
+	setCondition(&tenant, opState)
+
+	// Verify built-in users configuration
+	opState, userErr := r.verifyBuiltinUsers(ctx, &tenant)
+	if userErr != nil {
+		err = errors.Join(err, userErr)
+	}
+	setCondition(&tenant, opState)
+
+	if !equality.Semantic.DeepEqual(&tenant.Status, originStatus) {
+		return ctrl.Result{}, errors.Join(err, r.Status().Update(ctx, &tenant))
+	}
+	return ctrl.Result{}, err
+}
+
+func (r *TenantReconciler) verifyStorage(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
+	log := ctrl.LoggerFrom(ctx)
 
-	// TODO(user): your logic here
+	if tenant.Spec.Storage.S3 == nil {
+		return storageError, fmt.Errorf("missing S3 configurations")
+	}
+
+	// Get accessKey and secretKey
+	s3Config := tenant.Spec.Storage.S3
+	var accessKey, secretKey string
+	if s3Config.S3Auth.SecretRef != nil {
+		log.V(5).Info("Getting credentials from Secret")
+		var secret corev1.Secret
+		nn := types.NamespacedName{
+			Namespace: s3Config.S3Auth.SecretRef.Namespace,
+			Name:      s3Config.S3Auth.SecretRef.Name,
+		}
+		if err := r.Get(ctx, nn, &secret, &client.GetOptions{}); err != nil {
+			return storageError, fmt.Errorf("failed to get secret %v", nn)
+		}
+		accessKey, secretKey = string(secret.Data["accessKey"]), string(secret.Data["secretKey"])
+	} else {
+		accessKey, secretKey = s3Config.AccessKey, s3Config.SecretKey
+	}
+
+	// Test connection to S3
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(s3Config.Region),
+		Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
+		Endpoint:    aws.String(s3Config.Endpoint),
+	})
+	if err != nil {
+		return storageError, fmt.Errorf("failed to create session: %w", err)
+	}
+
+	// Check bucket
+	svc := s3.New(sess)
+	_, err = svc.GetBucketLocation(&s3.GetBucketLocationInput{
+		Bucket: aws.String(s3Config.BucketName),
+	})
+	if err != nil {
+		return storageError, fmt.Errorf("failed to connect to S3: %w", err)
+	}
+
+	return creationSucceeded, nil
+}
+
+func (r *TenantReconciler) verifyMeta(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
+	return creationSucceeded, nil
+}
+
+func (r *TenantReconciler) verifyBuiltinUsers(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
+	return creationSucceeded, nil
+}
 
-	return ctrl.Result{}, nil
+func setCondition(tenant *databendv1alpha1.Tenant, opState opState) {
+	var newCond metav1.Condition
+	switch opState {
+	case creationSucceeded:
+		newCond = metav1.Condition{
+			Type:    databendv1alpha1.TenantCreated,
+			Status:  metav1.ConditionTrue,
+			Message: common.TenantCreationSucceededMessage,
+			Reason:  databendv1alpha1.TenantCreationSucceededReason,
+		}
+	case storageError:
+		newCond = metav1.Condition{
+			Type:    databendv1alpha1.TenantError,
+			Status:  metav1.ConditionFalse,
+			Message: common.TenantStorageErrorMessage,
+			Reason:  databendv1alpha1.TenantStorageErrorReason,
+		}
+	case metaError:
+		newCond = metav1.Condition{
+			Type:    databendv1alpha1.TenantError,
+			Status:  metav1.ConditionFalse,
+			Message: common.TenantMetaErrorMessage,
+			Reason:  databendv1alpha1.TenantMetaErrorReason,
+		}
+	case builtinUserError:
+		newCond = metav1.Condition{
+			Type:    databendv1alpha1.TenantError,
+			Status:  metav1.ConditionFalse,
+			Message: common.TenantUserErrorMessage,
+			Reason:  databendv1alpha1.TenantUserErrorReason,
+		}
+	default:
+		return
+	}
+	meta.SetStatusCondition(&tenant.Status.Conditions, newCond)
 }
 
 // SetupWithManager sets up the controller with the Manager.