Skip to content

Commit 078a99f

Browse files
authored
Merge pull request #16 from Electronic-Waste/feat/tenant-controller-storage
feat(tenant): Support S3 Storage in Tenant
2 parents 753bb69 + 22cea0a commit 078a99f

File tree

6 files changed

+213
-13
lines changed

6 files changed

+213
-13
lines changed

config/rbac/role.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,13 @@ kind: ClusterRole
44
metadata:
55
name: manager-role
66
rules:
7+
- apiGroups:
8+
- ""
9+
resources:
10+
- secrets
11+
verbs:
12+
- get
13+
- list
714
- apiGroups:
815
- databendlabs.io
916
resources:

go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ module github.com/databendcloud/databend-operator
33
go 1.22.0
44

55
require (
6+
github.com/aws/aws-sdk-go v1.55.5
67
github.com/onsi/ginkgo/v2 v2.19.0
78
github.com/onsi/gomega v1.33.1
89
k8s.io/api v0.31.0
@@ -47,6 +48,7 @@ require (
4748
github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect
4849
github.com/imdario/mergo v0.3.6 // indirect
4950
github.com/inconshreveable/mousetrap v1.1.0 // indirect
51+
github.com/jmespath/go-jmespath v0.4.0 // indirect
5052
github.com/josharian/intern v1.0.0 // indirect
5153
github.com/json-iterator/go v1.1.12 // indirect
5254
github.com/mailru/easyjson v0.7.7 // indirect

go.sum

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8
22
github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g=
33
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA=
44
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
5+
github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU=
6+
github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
57
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
68
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
79
github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
@@ -70,6 +72,10 @@ github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
7072
github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
7173
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
7274
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
75+
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
76+
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
77+
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
78+
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
7379
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
7480
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
7581
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=

pkg/apis/databendlabs.io/v1alpha1/tenant_types.go

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,35 @@ const (
3131
NoPassword UserAuthType = "no_password"
3232
)
3333

34+
const (
35+
// TenantCreated means that the tenant creation has succeeded.
36+
TenantCreated string = "Created"
37+
38+
// TenantError means that the tenant reaches an error state.
39+
TenantError string = "Error"
40+
)
41+
42+
const (
43+
// TenantCreationSucceededReason is the "Created" condition reason.
44+
// When the tenant creation succeeded, this is added.
45+
TenantCreationSucceededReason = "TenantCreationSucceeded"
46+
47+
// TenantStorageErrorReason is the "Error" condition reason.
48+
// When the tenant has an error storage configurations and reaches an error state,
49+
// this is added.
50+
TenantStorageErrorReason = "TenantStorageError"
51+
52+
// TenantMetaErrorReason is the "Error" condition reason.
53+
// When the tenant has an error meta configurations and reaches an error state,
54+
// this is added.
55+
TenantMetaErrorReason = "TenantMetaError"
56+
57+
// TenantUserErrorReason is the "Error" condition reason.
58+
// When the tenant has an error built-in user configurations and reaches an error state,
59+
// this is added.
60+
TenantUserErrorReason = "TenantUserError"
61+
)
62+
3463
type Storage struct {
3564
// Specification of S3 storage.
3665
S3 *S3Storage `json:"s3,omitempty"`

pkg/common/const.go

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package common
2+
3+
const (
4+
// TenantCreationSucceededMessage is status condition message for the
5+
// {"type": "Created", "status": "True", "reason": "TenantCreationSucceeded"} condition.
6+
TenantCreationSucceededMessage = "Succeeded to create Tenant"
7+
8+
// TenantStorageErrorMessage is status condition message for the
9+
// {"type": "Error", "status": "False", "reason": "TenantStorageError"} condition.
10+
TenantStorageErrorMessage = "Invalid storage configuration"
11+
12+
// TenantStorageErrorMessage is status condition message for the
13+
// {"type": "Error", "status": "False", "reason": "TenantMetaError"} condition.
14+
TenantMetaErrorMessage = "Invalid meta configuration"
15+
16+
// TenantStorageErrorMessage is status condition message for the
17+
// {"type": "Error", "status": "False", "reason": "TenantUserError"} condition.
18+
TenantUserErrorMessage = "Invalid built-in user configurations"
19+
)

pkg/controller/tenant/tenant_controller.go

Lines changed: 150 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,34 @@ package tenant
1818

1919
import (
2020
"context"
21+
"errors"
22+
"fmt"
2123

24+
"github.com/aws/aws-sdk-go/aws"
25+
"github.com/aws/aws-sdk-go/aws/credentials"
26+
"github.com/aws/aws-sdk-go/aws/session"
27+
"github.com/aws/aws-sdk-go/service/s3"
28+
corev1 "k8s.io/api/core/v1"
29+
"k8s.io/apimachinery/pkg/api/equality"
30+
"k8s.io/apimachinery/pkg/api/meta"
31+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2232
"k8s.io/apimachinery/pkg/runtime"
33+
"k8s.io/apimachinery/pkg/types"
34+
"k8s.io/klog/v2"
2335
ctrl "sigs.k8s.io/controller-runtime"
2436
"sigs.k8s.io/controller-runtime/pkg/client"
25-
"sigs.k8s.io/controller-runtime/pkg/log"
2637

2738
databendv1alpha1 "github.com/databendcloud/databend-operator/pkg/apis/databendlabs.io/v1alpha1"
39+
"github.com/databendcloud/databend-operator/pkg/common"
40+
)
41+
42+
type opState int
43+
44+
const (
45+
creationSucceeded opState = iota
46+
storageError opState = iota
47+
metaError opState = iota
48+
builtinUserError opState = iota
2849
)
2950

3051
// TenantReconciler reconciles a Tenant object
@@ -36,22 +57,138 @@ type TenantReconciler struct {
3657
// +kubebuilder:rbac:groups=databendlabs.io,resources=tenants,verbs=get;list;watch;create;update;patch;delete
3758
// +kubebuilder:rbac:groups=databendlabs.io,resources=tenants/status,verbs=get;update;patch
3859
// +kubebuilder:rbac:groups=databendlabs.io,resources=tenants/finalizers,verbs=update
60+
// +kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list
3961

40-
// Reconcile is part of the main kubernetes reconciliation loop which aims to
41-
// move the current state of the cluster closer to the desired state.
42-
// TODO(user): Modify the Reconcile function to compare the state specified by
43-
// the Tenant object against the actual cluster state, and then
44-
// perform operations to make the cluster state reflect the state specified by
45-
// the user.
46-
//
47-
// For more details, check Reconcile and its Result here:
48-
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
4962
func (r *TenantReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
50-
_ = log.FromContext(ctx)
63+
var tenant databendv1alpha1.Tenant
64+
if err := r.Get(ctx, req.NamespacedName, &tenant); err != nil {
65+
return ctrl.Result{}, client.IgnoreNotFound(err)
66+
}
67+
log := ctrl.LoggerFrom(ctx).WithValues("tenant", klog.KObj(&tenant))
68+
ctx = ctrl.LoggerInto(ctx, log)
69+
log.V(2).Info("Reconciling Tenant")
70+
71+
var err error
72+
originStatus := tenant.Status.DeepCopy()
73+
74+
// Verify storage configuration
75+
opState, storageErr := r.verifyStorage(ctx, &tenant)
76+
if storageErr != nil {
77+
err = errors.Join(err, storageErr)
78+
}
79+
log.V(5).Info("Succeeded to verify storage configurations")
80+
setCondition(&tenant, opState)
81+
82+
// Verify meta configuration
83+
opState, metaErr := r.verifyMeta(ctx, &tenant)
84+
if metaErr != nil {
85+
err = errors.Join(err, metaErr)
86+
}
87+
setCondition(&tenant, opState)
88+
89+
// Verify built-in users configuration
90+
opState, userErr := r.verifyBuiltinUsers(ctx, &tenant)
91+
if userErr != nil {
92+
err = errors.Join(err, userErr)
93+
}
94+
setCondition(&tenant, opState)
95+
96+
if !equality.Semantic.DeepEqual(&tenant.Status, originStatus) {
97+
return ctrl.Result{}, errors.Join(err, r.Status().Update(ctx, &tenant))
98+
}
99+
return ctrl.Result{}, err
100+
}
101+
102+
func (r *TenantReconciler) verifyStorage(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
103+
log := ctrl.LoggerFrom(ctx)
51104

52-
// TODO(user): your logic here
105+
if tenant.Spec.Storage.S3 == nil {
106+
return storageError, fmt.Errorf("missing S3 configurations")
107+
}
108+
109+
// Get accessKey and secretKey
110+
s3Config := tenant.Spec.Storage.S3
111+
var accessKey, secretKey string
112+
if s3Config.S3Auth.SecretRef != nil {
113+
log.V(5).Info("Getting credentials from Secret")
114+
var secret corev1.Secret
115+
nn := types.NamespacedName{
116+
Namespace: s3Config.S3Auth.SecretRef.Namespace,
117+
Name: s3Config.S3Auth.SecretRef.Name,
118+
}
119+
if err := r.Get(ctx, nn, &secret, &client.GetOptions{}); err != nil {
120+
return storageError, fmt.Errorf("failed to get secret %v", nn)
121+
}
122+
accessKey, secretKey = string(secret.Data["accessKey"]), string(secret.Data["secretKey"])
123+
} else {
124+
accessKey, secretKey = s3Config.AccessKey, s3Config.SecretKey
125+
}
126+
127+
// Test connection to S3
128+
sess, err := session.NewSession(&aws.Config{
129+
Region: aws.String(s3Config.Region),
130+
Credentials: credentials.NewStaticCredentials(accessKey, secretKey, ""),
131+
Endpoint: aws.String(s3Config.Endpoint),
132+
})
133+
if err != nil {
134+
return storageError, fmt.Errorf("failed to create session: %w", err)
135+
}
136+
137+
// Check bucket
138+
svc := s3.New(sess)
139+
_, err = svc.GetBucketLocation(&s3.GetBucketLocationInput{
140+
Bucket: aws.String(s3Config.BucketName),
141+
})
142+
if err != nil {
143+
return storageError, fmt.Errorf("failed to connect to S3: %w", err)
144+
}
145+
146+
return creationSucceeded, nil
147+
}
148+
149+
func (r *TenantReconciler) verifyMeta(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
150+
return creationSucceeded, nil
151+
}
152+
153+
func (r *TenantReconciler) verifyBuiltinUsers(ctx context.Context, tenant *databendv1alpha1.Tenant) (opState, error) {
154+
return creationSucceeded, nil
155+
}
53156

54-
return ctrl.Result{}, nil
157+
func setCondition(tenant *databendv1alpha1.Tenant, opState opState) {
158+
var newCond metav1.Condition
159+
switch opState {
160+
case creationSucceeded:
161+
newCond = metav1.Condition{
162+
Type: databendv1alpha1.TenantCreated,
163+
Status: metav1.ConditionTrue,
164+
Message: common.TenantCreationSucceededMessage,
165+
Reason: databendv1alpha1.TenantCreationSucceededReason,
166+
}
167+
case storageError:
168+
newCond = metav1.Condition{
169+
Type: databendv1alpha1.TenantError,
170+
Status: metav1.ConditionFalse,
171+
Message: common.TenantStorageErrorMessage,
172+
Reason: databendv1alpha1.TenantStorageErrorReason,
173+
}
174+
case metaError:
175+
newCond = metav1.Condition{
176+
Type: databendv1alpha1.TenantError,
177+
Status: metav1.ConditionFalse,
178+
Message: common.TenantMetaErrorMessage,
179+
Reason: databendv1alpha1.TenantMetaErrorReason,
180+
}
181+
case builtinUserError:
182+
newCond = metav1.Condition{
183+
Type: databendv1alpha1.TenantError,
184+
Status: metav1.ConditionFalse,
185+
Message: common.TenantUserErrorMessage,
186+
Reason: databendv1alpha1.TenantUserErrorReason,
187+
}
188+
default:
189+
return
190+
}
191+
meta.SetStatusCondition(&tenant.Status.Conditions, newCond)
55192
}
56193

57194
// SetupWithManager sets up the controller with the Manager.

0 commit comments

Comments
 (0)