Skip to content

Support optional/configurable IAMEndpoint for Minio Client (#32581) #32581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 11 commits into from
Nov 22, 2024
Merged

Support optional/configurable IAMEndpoint for Minio Client (#32581) #32581

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
1967ec2
Support optional/configurable IAMEndpoint for Minio Client instead of…
mowoc-ocp Nov 20, 2024
c904556
Merge branch 'main' into minioIamDiscoverySupport
mowoc-ocp Nov 20, 2024
c70c028
Merge branch 'main' into minioIamDiscoverySupport
mowoc-ocp Nov 20, 2024
c8f296a
Merge branch 'main' into minioIamDiscoverySupport
GiteaBot Nov 21, 2024
616481c
Add MINIO_IAM_ENDPOINT documentation to app.example.ini
mowoc-ocp Nov 21, 2024
83f4fe0
Merge branch 'minioIamDiscoverySupport' of github.com:mowoc-ocp/gitea…
mowoc-ocp Nov 21, 2024
bba9492
Merge branch 'main' into minioIamDiscoverySupport
lunny Nov 22, 2024
59f2e07
Merge branch 'main' into minioIamDiscoverySupport
GiteaBot Nov 22, 2024
5c24937
Merge branch 'main' into minioIamDiscoverySupport
GiteaBot Nov 22, 2024
7a5d787
Merge branch 'main' into minioIamDiscoverySupport
GiteaBot Nov 22, 2024
9904d72
Merge branch 'main' into minioIamDiscoverySupport
GiteaBot Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions custom/conf/app.example.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1944,6 +1944,13 @@ LEVEL = Info
;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
;MINIO_SECRET_ACCESS_KEY =
;;
;; Preferred IAM Endpoint to override Minio's default IAM Endpoint resolution only available when STORAGE_TYPE is `minio`.
;; If not provided and STORAGE_TYPE is `minio`, will search for and derive endpoint from known environment variables
;; (AWS_CONTAINER_AUTHORIZATION_TOKEN, AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE, AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,
;; AWS_CONTAINER_CREDENTIALS_FULL_URI, AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_REGION),
;; or the DefaultIAMRoleEndpoint if not provided otherwise.
;MINIO_IAM_ENDPOINT =
;;
;; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
;MINIO_BUCKET = gitea
;;
Expand Down Expand Up @@ -2688,6 +2695,13 @@ LEVEL = Info
;; Minio secretAccessKey to connect only available when STORAGE_TYPE is `minio`
;MINIO_SECRET_ACCESS_KEY =
;;
;; Preferred IAM Endpoint to override Minio's default IAM Endpoint resolution only available when STORAGE_TYPE is `minio`.
;; If not provided and STORAGE_TYPE is `minio`, will search for and derive endpoint from known environment variables
;; (AWS_CONTAINER_AUTHORIZATION_TOKEN, AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE, AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,
;; AWS_CONTAINER_CREDENTIALS_FULL_URI, AWS_WEB_IDENTITY_TOKEN_FILE, AWS_ROLE_ARN, AWS_ROLE_SESSION_NAME, AWS_REGION),
;; or the DefaultIAMRoleEndpoint if not provided otherwise.
;MINIO_IAM_ENDPOINT =
;;
;; Minio bucket to store the attachments only available when STORAGE_TYPE is `minio`
;MINIO_BUCKET = gitea
;;
Expand Down
1 change: 1 addition & 0 deletions modules/setting/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type MinioStorageConfig struct {
Endpoint string `ini:"MINIO_ENDPOINT" json:",omitempty"`
AccessKeyID string `ini:"MINIO_ACCESS_KEY_ID" json:",omitempty"`
SecretAccessKey string `ini:"MINIO_SECRET_ACCESS_KEY" json:",omitempty"`
IamEndpoint string `ini:"MINIO_IAM_ENDPOINT" json:",omitempty"`
Bucket string `ini:"MINIO_BUCKET" json:",omitempty"`
Location string `ini:"MINIO_LOCATION" json:",omitempty"`
BasePath string `ini:"MINIO_BASE_PATH" json:",omitempty"`
Expand Down
13 changes: 13 additions & 0 deletions modules/setting/storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,19 @@ MINIO_BASE_PATH = /prefix
cfg, err = NewConfigProviderFromData(`
[storage]
STORAGE_TYPE = minio
MINIO_IAM_ENDPOINT = 127.0.0.1
MINIO_USE_SSL = true
MINIO_BASE_PATH = /prefix
`)
assert.NoError(t, err)
assert.NoError(t, loadRepoArchiveFrom(cfg))
assert.EqualValues(t, "127.0.0.1", RepoArchive.Storage.MinioConfig.IamEndpoint)
assert.EqualValues(t, true, RepoArchive.Storage.MinioConfig.UseSSL)
assert.EqualValues(t, "/prefix/repo-archive/", RepoArchive.Storage.MinioConfig.BasePath)

cfg, err = NewConfigProviderFromData(`
[storage]
STORAGE_TYPE = minio
MINIO_ACCESS_KEY_ID = my_access_key
MINIO_SECRET_ACCESS_KEY = my_secret_key
MINIO_USE_SSL = true
Expand Down
8 changes: 5 additions & 3 deletions modules/storage/minio.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func NewMinioStorage(ctx context.Context, cfg *setting.Storage) (ObjectStorage,
}

minioClient, err := minio.New(config.Endpoint, &minio.Options{
Creds: buildMinioCredentials(config, credentials.DefaultIAMRoleEndpoint),
Creds: buildMinioCredentials(config),
Secure: config.UseSSL,
Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: config.InsecureSkipVerify}},
Region: config.Location,
Expand Down Expand Up @@ -164,7 +164,7 @@ func (m *MinioStorage) buildMinioDirPrefix(p string) string {
return p
}

func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string) *credentials.Credentials {
func buildMinioCredentials(config setting.MinioStorageConfig) *credentials.Credentials {
// If static credentials are provided, use those
if config.AccessKeyID != "" {
return credentials.NewStaticV4(config.AccessKeyID, config.SecretAccessKey, "")
Expand All @@ -184,7 +184,9 @@ func buildMinioCredentials(config setting.MinioStorageConfig, iamEndpoint string
&credentials.FileAWSCredentials{},
// read IAM role from EC2 metadata endpoint if available
&credentials.IAM{
Endpoint: iamEndpoint,
// passing in an empty Endpoint lets the IAM Provider
// decide which endpoint to resolve internally
Endpoint: config.IamEndpoint,
Client: &http.Client{
Transport: http.DefaultTransport,
},
Expand Down
21 changes: 13 additions & 8 deletions modules/storage/minio_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -107,8 +107,9 @@ func TestMinioCredentials(t *testing.T) {
cfg := setting.MinioStorageConfig{
AccessKeyID: ExpectedAccessKey,
SecretAccessKey: ExpectedSecretAccessKey,
IamEndpoint: FakeEndpoint,
}
creds := buildMinioCredentials(cfg, FakeEndpoint)
creds := buildMinioCredentials(cfg)
v, err := creds.Get()

assert.NoError(t, err)
Expand All @@ -117,13 +118,15 @@ func TestMinioCredentials(t *testing.T) {
})

t.Run("Chain", func(t *testing.T) {
cfg := setting.MinioStorageConfig{}
cfg := setting.MinioStorageConfig{
IamEndpoint: FakeEndpoint,
}

t.Run("EnvMinio", func(t *testing.T) {
t.Setenv("MINIO_ACCESS_KEY", ExpectedAccessKey+"Minio")
t.Setenv("MINIO_SECRET_KEY", ExpectedSecretAccessKey+"Minio")

creds := buildMinioCredentials(cfg, FakeEndpoint)
creds := buildMinioCredentials(cfg)
v, err := creds.Get()

assert.NoError(t, err)
Expand All @@ -135,7 +138,7 @@ func TestMinioCredentials(t *testing.T) {
t.Setenv("AWS_ACCESS_KEY", ExpectedAccessKey+"AWS")
t.Setenv("AWS_SECRET_KEY", ExpectedSecretAccessKey+"AWS")

creds := buildMinioCredentials(cfg, FakeEndpoint)
creds := buildMinioCredentials(cfg)
v, err := creds.Get()

assert.NoError(t, err)
Expand All @@ -144,11 +147,11 @@ func TestMinioCredentials(t *testing.T) {
})

t.Run("FileMinio", func(t *testing.T) {
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
// prevent loading any actual credentials files from the user
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/minio.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/fake")

creds := buildMinioCredentials(cfg, FakeEndpoint)
creds := buildMinioCredentials(cfg)
v, err := creds.Get()

assert.NoError(t, err)
Expand All @@ -161,7 +164,7 @@ func TestMinioCredentials(t *testing.T) {
t.Setenv("MINIO_SHARED_CREDENTIALS_FILE", "testdata/fake.json")
t.Setenv("AWS_SHARED_CREDENTIALS_FILE", "testdata/aws_credentials")

creds := buildMinioCredentials(cfg, FakeEndpoint)
creds := buildMinioCredentials(cfg)
v, err := creds.Get()

assert.NoError(t, err)
Expand All @@ -187,7 +190,9 @@ func TestMinioCredentials(t *testing.T) {
defer server.Close()

// Use the provided EC2 Instance Metadata server
creds := buildMinioCredentials(cfg, server.URL)
creds := buildMinioCredentials(setting.MinioStorageConfig{
IamEndpoint: server.URL,
})
v, err := creds.Get()

assert.NoError(t, err)
Expand Down
Loading