Skip to content

Commit 070208f

Browse files
migmartrimigmartri
authored
feat(CAS): AWS S3 blob storage support (#390)
Signed-off-by: Miguel Martinez Trivino <[email protected]>
1 parent a344ad4 commit 070208f

File tree

22 files changed

+1067
-48
lines changed

22 files changed

+1067
-48
lines changed

‎app/artifact-cas/cmd/main.go‎

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,6 @@ import (
2525
"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf"
2626
"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server"
2727
backend "github.com/chainloop-dev/chainloop/internal/blobmanager"
28-
"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
29-
"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
3028
"github.com/chainloop-dev/chainloop/internal/credentials"
3129
"github.com/chainloop-dev/chainloop/internal/credentials/manager"
3230
"github.com/chainloop-dev/chainloop/internal/servicelogger"
@@ -64,16 +62,6 @@ type app struct {
6462
backend.Providers
6563
}
6664

67-
func loadCASBackendProviders(creader credentials.Reader) backend.Providers {
68-
// Initialize CAS backend providers
69-
ociProvider := oci.NewBackendProvider(creader)
70-
azureBlobProvider := azureblob.NewBackendProvider(creader)
71-
return backend.Providers{
72-
ociProvider.ID(): ociProvider,
73-
azureBlobProvider.ID(): azureBlobProvider,
74-
}
75-
}
76-
7765
func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, providers backend.Providers) *app {
7866
return &app{
7967
kratos.New(

‎app/artifact-cas/cmd/wire.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf"
2525
"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server"
2626
"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/service"
27+
"github.com/chainloop-dev/chainloop/internal/blobmanager/loader"
2728
"github.com/chainloop-dev/chainloop/internal/credentials"
2829
"github.com/go-kratos/kratos/v2/log"
2930
"github.com/google/wire"
@@ -35,7 +36,7 @@ func wireApp(*conf.Server, *conf.Auth, credentials.Reader, log.Logger) (*app, fu
3536
wire.Build(
3637
server.ProviderSet,
3738
service.ProviderSet,
38-
loadCASBackendProviders,
39+
loader.LoadProviders,
3940
newApp,
4041
serviceOpts,
4142
),

‎app/artifact-cas/cmd/wire_gen.go‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/cli/cmd/casbackend.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func newCASBackendAddCmd() *cobra.Command {
4141
cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization")
4242
cmd.PersistentFlags().String("description", "", "descriptive information for this registration")
4343

44-
cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd())
44+
cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd(), newCASBackendAddAWSS3Cmd())
4545
return cmd
4646
}
4747

@@ -54,7 +54,7 @@ func newCASBackendUpdateCmd() *cobra.Command {
5454
cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization")
5555
cmd.PersistentFlags().String("description", "", "descriptive information for this registration")
5656

57-
cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd())
57+
cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd(), newCASBackendUpdateAWSS3Cmd())
5858
return cmd
5959
}
6060

‎app/cli/cmd/casbackend_add_s3.go‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
//
2+
// Copyright 2023 The Chainloop Authors.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package cmd
17+
18+
import (
19+
"github.com/chainloop-dev/chainloop/app/cli/internal/action"
20+
"github.com/chainloop-dev/chainloop/internal/blobmanager/s3"
21+
"github.com/go-kratos/kratos/v2/log"
22+
"github.com/spf13/cobra"
23+
)
24+
25+
func newCASBackendAddAWSS3Cmd() *cobra.Command {
26+
var bucketName, accessKeyID, secretAccessKey, region string
27+
cmd := &cobra.Command{
28+
Use: "aws-s3",
29+
Short: "Register a AWS S3 storage bucket",
30+
RunE: func(cmd *cobra.Command, args []string) error {
31+
isDefault, err := cmd.Flags().GetBool("default")
32+
cobra.CheckErr(err)
33+
34+
description, err := cmd.Flags().GetString("description")
35+
cobra.CheckErr(err)
36+
37+
if isDefault {
38+
if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, ""); err != nil {
39+
return err
40+
} else if !confirmed {
41+
log.Info("Aborting...")
42+
return nil
43+
}
44+
}
45+
46+
opts := &action.NewCASBackendAddOpts{
47+
Location: bucketName,
48+
Provider: s3.ProviderID,
49+
Description: description,
50+
Credentials: map[string]any{
51+
"accessKeyID": accessKeyID,
52+
"secretAccessKey": secretAccessKey,
53+
"region": region,
54+
},
55+
Default: isDefault,
56+
}
57+
58+
res, err := action.NewCASBackendAdd(actionOpts).Run(opts)
59+
if err != nil {
60+
return err
61+
} else if res == nil {
62+
return nil
63+
}
64+
65+
return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput)
66+
},
67+
}
68+
69+
cmd.Flags().StringVar(&bucketName, "bucket", "", "S3 bucket name")
70+
err := cmd.MarkFlagRequired("bucket")
71+
cobra.CheckErr(err)
72+
73+
cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID")
74+
err = cmd.MarkFlagRequired("access-key-id")
75+
cobra.CheckErr(err)
76+
77+
cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key")
78+
err = cmd.MarkFlagRequired("secret-access-key")
79+
cobra.CheckErr(err)
80+
81+
cmd.Flags().StringVar(&region, "region", "", "AWS region for the bucket")
82+
err = cmd.MarkFlagRequired("region")
83+
cobra.CheckErr(err)
84+
85+
return cmd
86+
}

‎app/cli/cmd/casbackend_update_s3.go‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
//
2+
// Copyright 2023 The Chainloop Authors.
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
16+
package cmd
17+
18+
import (
19+
"github.com/chainloop-dev/chainloop/app/cli/internal/action"
20+
"github.com/go-kratos/kratos/v2/log"
21+
"github.com/spf13/cobra"
22+
)
23+
24+
func newCASBackendUpdateAWSS3Cmd() *cobra.Command {
25+
var backendID, accessKeyID, secretAccessKey, region string
26+
cmd := &cobra.Command{
27+
Use: "aws-s3",
28+
Short: "Update a AWS S3 CAS Backend description, credentials or default status",
29+
RunE: func(cmd *cobra.Command, args []string) error {
30+
// If we are setting the default, we list existing CAS backends
31+
// and ask the user to confirm the rewrite
32+
isDefault, err := cmd.Flags().GetBool("default")
33+
cobra.CheckErr(err)
34+
35+
description, err := cmd.Flags().GetString("description")
36+
cobra.CheckErr(err)
37+
38+
// If we are overriding the default we ask for confirmation
39+
if isDefault {
40+
if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, backendID); err != nil {
41+
return err
42+
} else if !confirmed {
43+
log.Info("Aborting...")
44+
return nil
45+
}
46+
} else {
47+
// If we are removing the default we ask for confirmation too
48+
if confirmed, err := confirmDefaultCASBackendUnset(backendID, "You are setting the default CAS backend to false", actionOpts); err != nil {
49+
return err
50+
} else if !confirmed {
51+
log.Info("Aborting...")
52+
return nil
53+
}
54+
}
55+
56+
opts := &action.NewCASBackendUpdateOpts{
57+
ID: backendID,
58+
Description: description,
59+
Credentials: map[string]any{
60+
"accessKeyID": accessKeyID,
61+
"secretAccessKey": secretAccessKey,
62+
"region": region,
63+
},
64+
Default: isDefault,
65+
}
66+
67+
// this means that we are not updating credentials
68+
if accessKeyID == "" && secretAccessKey == "" && region == "" {
69+
opts.Credentials = nil
70+
}
71+
72+
res, err := action.NewCASBackendUpdate(actionOpts).Run(opts)
73+
if err != nil {
74+
return err
75+
} else if res == nil {
76+
return nil
77+
}
78+
79+
return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput)
80+
},
81+
}
82+
83+
cmd.Flags().StringVar(&backendID, "id", "", "CAS Backend ID")
84+
err := cmd.MarkFlagRequired("id")
85+
cobra.CheckErr(err)
86+
87+
cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID")
88+
cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key")
89+
cmd.Flags().StringVar(&region, "region", "", "AWS region for the bucket")
90+
91+
return cmd
92+
}

‎app/controlplane/cmd/main.go‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@ import (
2828
"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
2929
"github.com/chainloop-dev/chainloop/app/controlplane/plugins"
3030
"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
31-
backends "github.com/chainloop-dev/chainloop/internal/blobmanager"
32-
"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
33-
"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
3431
"github.com/chainloop-dev/chainloop/internal/credentials"
3532
"github.com/chainloop-dev/chainloop/internal/credentials/manager"
3633
"github.com/chainloop-dev/chainloop/internal/servicelogger"
@@ -171,16 +168,6 @@ func maskArgs(keyvals []interface{}) {
171168
}
172169
}
173170

174-
func loadCASBackendProviders(creader credentials.Reader) backends.Providers {
175-
// Initialize CAS backend providers
176-
ociProvider := oci.NewBackendProvider(creader)
177-
azureBlobProvider := azureblob.NewBackendProvider(creader)
178-
return backends.Providers{
179-
ociProvider.ID(): ociProvider,
180-
azureBlobProvider.ID(): azureBlobProvider,
181-
}
182-
}
183-
184171
func initSentry(c *conf.Bootstrap, logger log.Logger) (cleanupFunc func(), err error) {
185172
cleanupFunc = func() {
186173
sentry.Flush(2 * time.Second)

‎app/controlplane/cmd/wire.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ import (
2828
"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
2929
"github.com/chainloop-dev/chainloop/app/controlplane/internal/service"
3030
"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
31+
"github.com/chainloop-dev/chainloop/internal/blobmanager/loader"
3132
"github.com/chainloop-dev/chainloop/internal/credentials"
3233
"github.com/go-kratos/kratos/v2/log"
3334
"github.com/google/wire"
@@ -40,7 +41,7 @@ func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger, sdk.Availabl
4041
server.ProviderSet,
4142
data.ProviderSet,
4243
biz.ProviderSet,
43-
loadCASBackendProviders,
44+
loader.LoadProviders,
4445
service.ProviderSet,
4546
wire.Bind(new(biz.CASClient), new(*biz.CASClientUseCase)),
4647
serviceOpts,

‎app/controlplane/cmd/wire_gen.go‎

Lines changed: 2 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎app/controlplane/internal/biz/casbackend.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
backend "github.com/chainloop-dev/chainloop/internal/blobmanager"
2727
"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
2828
"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
29+
"github.com/chainloop-dev/chainloop/internal/blobmanager/s3"
2930
"github.com/chainloop-dev/chainloop/internal/credentials"
3031
"github.com/chainloop-dev/chainloop/internal/servicelogger"
3132
"github.com/go-kratos/kratos/v2/log"
@@ -470,7 +471,7 @@ func (uc *CASBackendUseCase) PerformValidation(ctx context.Context, id string) (
470471

471472
// Implements https://pkg.go.dev/entgo.io/ent/schema/field#EnumValues
472473
func (CASBackendProvider) Values() (kinds []string) {
473-
for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline} {
474+
for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline, s3.ProviderID} {
474475
kinds = append(kinds, string(s))
475476
}
476477

0 commit comments

Comments
 (0)