Skip to content

Commit 57b54c8

Browse files
pa250194squaremo
pa250194
authored and
squaremo
committed
Service Account Key Authentication to GCP Provider
Signed-off-by: pa250194 <[email protected]>
1 parent 2cc48fe commit 57b54c8

File tree

8 files changed

+511
-175
lines changed

8 files changed

+511
-175
lines changed

‎api/v1beta1/bucket_types.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ const (
3030
// BucketSpec defines the desired state of an S3 compatible bucket
3131
type BucketSpec struct {
3232
// The S3 compatible storage provider name, default ('generic').
33-
// +kubebuilder:validation:Enum=generic;aws
33+
// +kubebuilder:validation:Enum=generic;aws;gcp
3434
// +kubebuilder:default:=generic
3535
// +optional
3636
Provider string `json:"provider,omitempty"`

‎config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ spec:
6666
enum:
6767
- generic
6868
- aws
69+
- gcp
6970
type: string
7071
region:
7172
description: The bucket region.

‎controllers/bucket_controller.go‎

Lines changed: 55 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
177177
}
178178

179179
func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) {
180-
var tempDir string
181180
var err error
182181
var sourceBucket sourcev1.Bucket
182+
tempDir, err := os.MkdirTemp("", bucket.Name)
183+
if err != nil {
184+
err = fmt.Errorf("tmp dir error: %w", err)
185+
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
186+
}
187+
defer os.RemoveAll(tempDir)
183188
if bucket.Spec.Provider == sourcev1.GoogleBucketProvider {
184-
sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket)
189+
sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir)
185190
if err != nil {
186191
return sourceBucket, err
187192
}
188193
} else {
189-
sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket)
194+
sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir)
190195
if err != nil {
191196
return sourceBucket, err
192197
}
@@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1.
261266

262267
// reconcileWithGCP handles getting objects from a Google Cloud Platform bucket
263268
// using a gcp client
264-
func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
269+
func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
265270
gcpClient, err := r.authGCP(ctx, bucket)
266271
if err != nil {
267272
err = fmt.Errorf("auth error: %w", err)
268-
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
273+
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
269274
}
270275
defer gcpClient.Client.Close()
271-
// create tmp dir
272-
tempDir, err := os.MkdirTemp("", bucket.Name)
273-
if err != nil {
274-
err = fmt.Errorf("tmp dir error: %w", err)
275-
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
276-
}
277-
defer os.RemoveAll(tempDir)
278276

279277
ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
280278
defer cancel()
281279

282280
exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName)
283281
if err != nil {
284-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
282+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
285283
}
286284
if !exists {
287285
err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
288-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
286+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
289287
}
290288

291289
// Look for file with ignore rules first.
292290
path := filepath.Join(tempDir, sourceignore.IgnoreFile)
293291
if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil {
294-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
292+
if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" {
293+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
294+
}
295295
}
296296
ps, err := sourceignore.ReadIgnoreFile(path, nil)
297297
if err != nil {
298-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
298+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
299299
}
300300
// In-spec patterns take precedence
301301
if bucket.Spec.Ignore != nil {
@@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
311311
}
312312
if err != nil {
313313
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
314-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
314+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
315315
}
316316

317317
if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile {
@@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
323323
}
324324

325325
localPath := filepath.Join(tempDir, object.Name)
326-
// FGetObject - get and download bucket object
327326
if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil {
328327
err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
329-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
328+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
330329
}
331330
}
332-
return sourcev1.Bucket{}, tempDir, nil
331+
return sourcev1.Bucket{}, nil
333332
}
334333

335334
// reconcileWithMinio handles getting objects from an S3 compatible bucket
336335
// using a minio client
337-
func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
336+
func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
338337
s3Client, err := r.authMinio(ctx, bucket)
339338
if err != nil {
340339
err = fmt.Errorf("auth error: %w", err)
341-
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
340+
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
342341
}
343342

344-
// create tmp dir
345-
tempDir, err := os.MkdirTemp("", bucket.Name)
346-
if err != nil {
347-
err = fmt.Errorf("tmp dir error: %w", err)
348-
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
349-
}
350-
defer os.RemoveAll(tempDir)
351-
352343
ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
353344
defer cancel()
354345

355346
exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName)
356347
if err != nil {
357-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
348+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
358349
}
359350
if !exists {
360351
err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
361-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
352+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
362353
}
363354

364355
// Look for file with ignore rules first
@@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
367358
path := filepath.Join(tempDir, sourceignore.IgnoreFile)
368359
if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil {
369360
if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" {
370-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
361+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
371362
}
372363
}
373364
ps, err := sourceignore.ReadIgnoreFile(path, nil)
374365
if err != nil {
375-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
366+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
376367
}
377368
// In-spec patterns take precedence
378369
if bucket.Spec.Ignore != nil {
@@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
387378
}) {
388379
if object.Err != nil {
389380
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err)
390-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
381+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
391382
}
392383

393384
if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile {
@@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
402393
err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{})
403394
if err != nil {
404395
err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
405-
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
396+
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
406397
}
407398
}
408-
return sourcev1.Bucket{}, tempDir, nil
399+
return sourcev1.Bucket{}, nil
409400
}
410401

411402
// authGCP creates a new Google Cloud Platform storage client
412-
// to interact with the Storage service.
403+
// to interact with the storage service.
413404
func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) {
414-
client, err := gcp.NewClient(ctx)
415-
if err != nil {
416-
return nil, err
405+
var client *gcp.GCPClient
406+
var err error
407+
if bucket.Spec.SecretRef != nil {
408+
secretName := types.NamespacedName{
409+
Namespace: bucket.GetNamespace(),
410+
Name: bucket.Spec.SecretRef.Name,
411+
}
412+
413+
var secret corev1.Secret
414+
if err := r.Get(ctx, secretName, &secret); err != nil {
415+
return nil, fmt.Errorf("credentials secret error: %w", err)
416+
}
417+
if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil {
418+
return nil, err
419+
}
420+
serviceAccount := gcp.InitCredentialsWithSecret(secret.Data)
421+
client, err = gcp.NewClientWithSAKey(ctx, serviceAccount)
422+
if err != nil {
423+
return nil, err
424+
}
425+
} else {
426+
client, err = gcp.NewClient(ctx)
427+
if err != nil {
428+
return nil, err
429+
}
417430
}
418431
return client, nil
432+
419433
}
420434

421435
// authMinio creates a new Minio client to interact with S3

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ require (
2323
github.com/go-git/go-git/v5 v5.4.2
2424
github.com/go-logr/logr v0.4.0
2525
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
26+
github.com/golang/mock v1.6.0 // indirect
2627
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
2728
github.com/libgit2/git2go/v31 v31.6.1
2829
github.com/minio/minio-go/v7 v7.0.10

‎go.sum‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -419,6 +419,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
419419
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
420420
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
421421
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
422+
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
422423
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
423424
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
424425
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

0 commit comments

Comments
 (0)