Skip to content

Commit 76576a8

Browse files
CRThazeCRThaze
authored
Merge pull request #12 from comet-ml/CRThaze/dev
Improve secret setting & fix GCS backend
2 parents 98e4116 + c7d6a60 commit 76576a8

File tree

7 files changed

+224
-199
lines changed

7 files changed

+224
-199
lines changed

‎charts/s3proxy/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ type: application
1515
# This is the chart version. This version number should be incremented each time you make changes
1616
# to the chart and its templates, including the app version.
1717
# Versions are expected to follow Semantic Versioning (https://semver.org/)
18-
version: 0.0.5
18+
version: 0.0.6
1919

2020
# This is the version number of the application being deployed. This version number should be
2121
# incremented each time you make changes to the application. Versions are not expected to

‎charts/s3proxy/override-values.example.yaml‎

Lines changed: 36 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -67,73 +67,76 @@ config:
6767
region: "us-west-2"
6868
# endpoint: "https://s3.amazonaws.com" # Optional custom endpoint (e.g., MinIO, Ceph)
6969
accessKeyID: "AKIAIOSFODNN7EXAMPLE"
70-
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
70+
secretAccessKey:
71+
value: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
72+
# existingSecret: "my-s3-secret"
73+
# secretKey: "secretAccessKey"
7174

7275
# Azure Blob Storage backend
7376
azureblob:
7477
enabled: false # Set to true to use Azure Blob backend
7578
provider: "azureblob" # Can be "azureblob" or "azureblob-sdk"
7679
account: "mystorageaccount"
77-
key: "base64encodedkey=="
80+
key:
81+
value: "base64encodedkey=="
82+
# existingSecret: "my-azure-secret"
83+
# secretKey: "accountKey"
7884
# endpoint: "https://mystorageaccount.blob.core.windows.net" # Optional
79-
# sasToken: "" # Optional SAS token
85+
sasToken:
86+
value: "" # Optional SAS token
87+
# existingSecret: "my-azure-sas-secret"
88+
# secretKey: "sasToken"
8089

8190
# Google Cloud Storage backend
8291
googleCloudStorage:
8392
enabled: false # Set to true to use GCS backend
8493
projectID: "my-project"
8594

86-
# Service account email or user email (required for both authentication methods)
95+
# Service account email or user email
8796
clientEmail: "[email protected]"
8897

89-
# Option 1: Using privateKey directly
90-
privateKey: "-----BEGIN RSA PRIVATE KEY-----\n..."
91-
92-
# Option 2: Using JSON credentials file (preferred for GCP)
93-
# This provides the credential (privateKey) via a mounted file
94-
jsonCredentials:
95-
enabled: false # Set to true to use JSON credentials
96-
97-
# Either provide the JSON content directly:
98-
# jsonContent: |
99-
# {
100-
# "type": "service_account",
101-
# "project_id": "my-project",
102-
# "private_key_id": "key-id",
103-
# "private_key": "-----BEGIN RSA PRIVATE KEY-----\n...",
104-
# "client_email": "[email protected]",
105-
# "client_id": "...",
106-
# "auth_uri": "https://accounts.google.com/o/oauth2/auth",
107-
# "token_uri": "https://oauth2.googleapis.com/token",
108-
# "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
109-
# "client_x509_cert_url": "..."
110-
# }
111-
112-
# Or reference an existing secret containing the JSON:
113-
# existingSecret: "my-gcp-credentials-secret"
114-
# secretKey: "credentials.json" # Key in the secret containing the JSON (default: credentials.json)
98+
# Private key configuration
99+
privateKey:
100+
# Option 1: Provide private key directly
101+
# value: |-
102+
# -----BEGIN RSA PRIVATE KEY-----
103+
# ...
104+
# -----END RSA PRIVATE KEY-----
105+
106+
# Option 2: Reference an existing secret containing the private key
107+
# existingSecret: "my-gcs-privatekey-secret"
108+
# secretKey: "privateKey"
115109

116110
# Backblaze B2 backend
117111
b2:
118112
enabled: false # Set to true to use B2 backend
119113
account: "account-id"
120-
applicationKey: "application-key"
114+
applicationKey:
115+
value: "application-key"
116+
# existingSecret: "my-b2-secret"
117+
# secretKey: "applicationKey"
121118

122119
# OpenStack Swift backend
123120
openstackSwift:
124121
enabled: false # Set to true to use Swift backend
125122
authURL: "https://auth.cloud.com/v2.0"
126123
tenantName: "my-tenant"
127124
userName: "my-user"
128-
password: "my-password"
125+
password:
126+
value: "my-password"
127+
# existingSecret: "my-swift-secret"
128+
# secretKey: "password"
129129
region: "RegionOne"
130130

131131
# Rackspace Cloud Files backend
132132
rackspaceCloudfiles:
133133
enabled: false # Set to true to use Rackspace Cloud Files backend
134134
region: "us" # Region: "us" or "uk"
135135
userName: "my-user"
136-
apiKey: "my-api-key"
136+
apiKey:
137+
value: "my-api-key"
138+
# existingSecret: "my-rackspace-secret"
139+
# secretKey: "apiKey"
137140

138141
# Persistence settings for filesystem backend
139142
persistence:
@@ -233,4 +236,3 @@ extraEnvVars:
233236
# values:
234237
# - s3proxy
235238
# topologyKey: kubernetes.io/hostname
236-

‎charts/s3proxy/templates/configmap.yaml‎

Lines changed: 105 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,67 +1,66 @@
11
{{- define "s3proxy.main.config" -}}
2-
# S3Proxy configuration
3-
s3proxy.endpoint=http://0.0.0.0:{{ .Values.service.targetPort }}
4-
s3proxy.authorization={{ .Values.config.auth.type }}
5-
{{- if .Values.config.virtualHost }}
6-
s3proxy.virtual-host={{ .Values.config.virtualHost }}
7-
{{- end }}
8-
9-
{{- if ne .Values.config.auth.type "none" }}
10-
# Authentication credentials will be merged from the secret properties file
11-
# s3proxy.identity and s3proxy.credential will be provided by the secret
12-
{{- end }}
13-
14-
{{- if .Values.config.cors.enabled }}
15-
# CORS configuration
16-
{{- if .Values.config.cors.allowOrigins }}
17-
s3proxy.cors-allow-origins={{ join " " .Values.config.cors.allowOrigins }}
2+
# S3Proxy configuration
3+
s3proxy.endpoint=http://0.0.0.0:{{ .Values.service.targetPort }}
4+
s3proxy.authorization={{ .Values.config.auth.type }}
5+
{{- if .Values.config.virtualHost }}
6+
s3proxy.virtual-host={{ .Values.config.virtualHost }}
187
{{- end }}
19-
{{- if .Values.config.cors.allowMethods }}
20-
s3proxy.cors-allow-methods={{ join " " .Values.config.cors.allowMethods }}
8+
9+
{{- if ne .Values.config.auth.type "none" }}
10+
# Authentication credentials will be merged from the secret properties file
11+
# s3proxy.identity and s3proxy.credential will be provided by the secret
2112
{{- end }}
22-
{{- if .Values.config.cors.allowHeaders }}
23-
s3proxy.cors-allow-headers={{ join " " .Values.config.cors.allowHeaders }}
13+
14+
{{- if .Values.config.cors.enabled }}
15+
# CORS configuration
16+
{{- if .Values.config.cors.allowOrigins }}
17+
s3proxy.cors-allow-origins={{ join " " .Values.config.cors.allowOrigins }}
18+
{{- end }}
19+
{{- if .Values.config.cors.allowMethods }}
20+
s3proxy.cors-allow-methods={{ join " " .Values.config.cors.allowMethods }}
21+
{{- end }}
22+
{{- if .Values.config.cors.allowHeaders }}
23+
s3proxy.cors-allow-headers={{ join " " .Values.config.cors.allowHeaders }}
24+
{{- end }}
25+
s3proxy.cors-allow-credential={{ .Values.config.cors.allowCredential }}
2426
{{- end }}
25-
s3proxy.cors-allow-credential={{ .Values.config.cors.allowCredential }}
26-
{{- end }}
2727

28-
{{- if .Values.config.buckets.alias }}
29-
# Bucket aliasing
30-
{{- range $key, $value := .Values.config.buckets.alias }}
31-
s3proxy.alias.{{ $key }}={{ $value }}
28+
{{- if .Values.config.buckets.alias }}
29+
# Bucket aliasing
30+
{{- range $key, $value := .Values.config.buckets.alias }}
31+
s3proxy.alias.{{ $key }}={{ $value }}
32+
{{- end }}
3233
{{- end }}
33-
{{- end }}
3434

35-
{{- if .Values.config.buckets.locator }}
36-
# Bucket locator
37-
{{- range $index, $bucket := .Values.config.buckets.locator }}
38-
s3proxy.bucket-locator.{{ add $index 1 }}={{ $bucket }}
35+
{{- if .Values.config.buckets.locator }}
36+
# Bucket locator
37+
{{- range $index, $bucket := .Values.config.buckets.locator }}
38+
s3proxy.bucket-locator.{{ add $index 1 }}={{ $bucket }}
39+
{{- end }}
3940
{{- end }}
40-
{{- end }}
4141

42-
{{- if .Values.config.middlewares.readOnly }}
43-
# Read-only middleware
44-
s3proxy.read-only-blobstore=true
45-
{{- end }}
42+
{{- if .Values.config.middlewares.readOnly }}
43+
# Read-only middleware
44+
s3proxy.read-only-blobstore=true
45+
{{- end }}
4646

47-
{{- if .Values.config.middlewares.eventualConsistency }}
48-
# Eventual consistency middleware
49-
s3proxy.eventual-consistency=true
50-
{{- end }}
47+
{{- if .Values.config.middlewares.eventualConsistency }}
48+
# Eventual consistency middleware
49+
s3proxy.eventual-consistency=true
50+
{{- end }}
5151

52-
{{- if .Values.config.middlewares.shardedBackend }}
53-
# Sharded backend middleware
54-
s3proxy.sharded-blobstore=true
55-
s3proxy.sharded-blobstore.prefix=2
56-
s3proxy.sharded-blobstore.levels=2
57-
{{- end }}
52+
{{- if .Values.config.middlewares.shardedBackend }}
53+
# Sharded backend middleware
54+
s3proxy.sharded-blobstore=true
55+
s3proxy.sharded-blobstore.prefix=2
56+
s3proxy.sharded-blobstore.levels=2
57+
{{- end }}
5858

59-
{{- if .Values.config.middlewares.largeObjectMocking }}
60-
# Large object mocking middleware
61-
s3proxy.large-object-mocking=true
62-
{{- end }}
59+
{{- if .Values.config.middlewares.largeObjectMocking }}
60+
# Large object mocking middleware
61+
s3proxy.large-object-mocking=true
62+
{{- end }}
6363
{{- end }}
64-
6564
apiVersion: v1
6665
kind: ConfigMap
6766
metadata:
@@ -70,8 +69,8 @@ metadata:
7069
{{- include "s3proxy.labels" . | nindent 4 }}
7170
data:
7271
{{- if .Values.config.backends.filesystem.enabled }}
73-
backend-filesystem.properties: |
74-
{{- include "s3proxy.main.config" . | nindent 4 }}
72+
backend-filesystem.properties: |-
73+
{{ include "s3proxy.main.config" . | nindent 4 }}
7574

7675
# Filesystem backend configuration
7776
{{- if .Values.config.backends.filesystem.nio2 }}
@@ -83,8 +82,8 @@ data:
8382
{{- end }}
8483

8584
{{- if .Values.config.backends.transient.enabled }}
86-
backend-transient.properties: |
87-
{{- include "s3proxy.main.config" . | nindent 4 }}
85+
backend-transient.properties: |-
86+
{{ include "s3proxy.main.config" . | nindent 4 }}
8887

8988
# Transient backend configuration
9089
{{- if .Values.config.backends.transient.nio2 }}
@@ -95,8 +94,8 @@ data:
9594
{{- end }}
9695

9796
{{- if .Values.config.backends.s3.enabled }}
98-
backend-s3.properties: |
99-
{{- include "s3proxy.main.config" . | nindent 4 }}
97+
backend-s3.properties: |-
98+
{{ include "s3proxy.main.config" . | nindent 4 }}
10099

101100
# S3 backend configuration
102101
{{- if .Values.config.backends.s3.aws }}
@@ -110,49 +109,69 @@ data:
110109
{{- if .Values.config.backends.s3.endpoint }}
111110
jclouds.endpoint={{ .Values.config.backends.s3.endpoint }}
112111
{{- end }}
113-
# Credentials will be merged from the secret properties file
114-
# jclouds.identity and jclouds.credential will be provided by the secret
112+
{{- if .Values.config.backends.s3.accessKeyID }}
113+
jclouds.identity={{ .Values.config.backends.s3.accessKeyID }}
114+
{{- end }}
115+
{{- if .Values.config.backends.s3.secretAccessKey.value }}
116+
# Credential will be merged from the secret properties file
117+
{{- end }}
115118
{{- end }}
116119

117120
{{- if .Values.config.backends.azureblob.enabled }}
118-
backend-azureblob.properties: |
119-
{{- include "s3proxy.main.config" . | nindent 4 }}
121+
backend-azureblob.properties: |-
122+
{{ include "s3proxy.main.config" . | nindent 4 }}
120123

121124
# Azure Blob backend configuration
122125
jclouds.provider={{ .Values.config.backends.azureblob.provider }}
123126
{{- if .Values.config.backends.azureblob.endpoint }}
124127
jclouds.azureblob.endpoint={{ .Values.config.backends.azureblob.endpoint }}
125128
{{- end }}
129+
{{- if .Values.config.backends.azureblob.account }}
130+
jclouds.identity={{ .Values.config.backends.azureblob.account }}
131+
{{- end }}
132+
{{- if or .Values.config.backends.azureblob.key.value .Values.config.backends.azureblob.sasToken.value }}
126133
# Credentials will be merged from the secret properties file
127-
# jclouds.identity, jclouds.credential, and jclouds.azureblob.sas will be provided by the secret
134+
{{- end }}
128135
{{- end }}
129136

130137
{{- if .Values.config.backends.googleCloudStorage.enabled }}
131-
backend-google-cloud-storage.properties: |
132-
{{- include "s3proxy.main.config" . | nindent 4 }}
138+
backend-google-cloud-storage.properties: |-
139+
{{ include "s3proxy.main.config" . | nindent 4 }}
133140

134141
# Google Cloud Storage backend configuration
135142
jclouds.provider=google-cloud-storage
136143
{{- if .Values.config.backends.googleCloudStorage.projectID }}
137144
jclouds.project-id={{ .Values.config.backends.googleCloudStorage.projectID }}
138145
{{- end }}
139-
# Credentials will be merged from the secret properties file
140-
# jclouds.identity and jclouds.credential will be provided by the secret
146+
{{- if .Values.config.backends.googleCloudStorage.clientEmail }}
147+
jclouds.identity={{ .Values.config.backends.googleCloudStorage.clientEmail }}
148+
{{- end }}
149+
{{- if .Values.config.backends.googleCloudStorage.privateKey.existingSecret }}
150+
# Using privateKey from mounted secret file
151+
jclouds.credential=/credentials/gcs-private.key
152+
{{- else if .Values.config.backends.googleCloudStorage.privateKey.value }}
153+
# Private key credential will be merged from the secret properties file
154+
# jclouds.credential will be provided by the secret
155+
{{- end }}
141156
{{- end }}
142157

143158
{{- if .Values.config.backends.b2.enabled }}
144-
backend-b2.properties: |
145-
{{- include "s3proxy.main.config" . | nindent 4 }}
159+
backend-b2.properties: |-
160+
{{ include "s3proxy.main.config" . | nindent 4 }}
146161

147162
# Backblaze B2 backend configuration
148163
jclouds.provider=b2
149-
# Credentials will be merged from the secret properties file
150-
# jclouds.identity and jclouds.credential will be provided by the secret
164+
{{- if .Values.config.backends.b2.account }}
165+
jclouds.identity={{ .Values.config.backends.b2.account }}
166+
{{- end }}
167+
{{- if .Values.config.backends.b2.applicationKey.value }}
168+
# Credential will be merged from the secret properties file
169+
{{- end }}
151170
{{- end }}
152171

153172
{{- if .Values.config.backends.openstackSwift.enabled }}
154-
backend-openstack-swift.properties: |
155-
{{- include "s3proxy.main.config" . | nindent 4 }}
173+
backend-openstack-swift.properties: |-
174+
{{ include "s3proxy.main.config" . | nindent 4 }}
156175

157176
# OpenStack Swift backend configuration
158177
jclouds.provider=openstack-swift
@@ -162,20 +181,28 @@ data:
162181
{{- if .Values.config.backends.openstackSwift.region }}
163182
jclouds.region={{ .Values.config.backends.openstackSwift.region }}
164183
{{- end }}
165-
# Credentials will be merged from the secret properties file
166-
# jclouds.identity and jclouds.credential will be provided by the secret
184+
{{- if and .Values.config.backends.openstackSwift.tenantName .Values.config.backends.openstackSwift.userName }}
185+
jclouds.identity={{ printf "%s:%s" .Values.config.backends.openstackSwift.tenantName .Values.config.backends.openstackSwift.userName }}
186+
{{- end }}
187+
{{- if .Values.config.backends.openstackSwift.password.value }}
188+
# Credential will be merged from the secret properties file
189+
{{- end }}
167190
{{- end }}
168191

169192
{{- if .Values.config.backends.rackspaceCloudfiles.enabled }}
170-
backend-rackspace-cloudfiles.properties: |
171-
{{- include "s3proxy.main.config" . | nindent 4 }}
193+
backend-rackspace-cloudfiles.properties: |-
194+
{{ include "s3proxy.main.config" . | nindent 4 }}
172195

173196
# Rackspace Cloud Files backend configuration
174197
{{- if eq .Values.config.backends.rackspaceCloudfiles.region "uk" }}
175198
jclouds.provider=rackspace-cloudfiles-uk
176199
{{- else }}
177200
jclouds.provider=rackspace-cloudfiles-us
178201
{{- end }}
179-
# Credentials will be merged from the secret properties file
180-
# jclouds.identity and jclouds.credential will be provided by the secret
202+
{{- if .Values.config.backends.rackspaceCloudfiles.userName }}
203+
jclouds.identity={{ .Values.config.backends.rackspaceCloudfiles.userName }}
204+
{{- end }}
205+
{{- if .Values.config.backends.rackspaceCloudfiles.apiKey.value }}
206+
# Credential will be merged from the secret properties file
207+
{{- end }}
181208
{{- end }}

0 commit comments

Comments
 (0)