Skip to content

Commit aab4476

Browse files
CRThazeCRThaze
committed
Fix config and support multiple backends
1 parent 6b3fc85 commit aab4476

File tree

8 files changed

+447
-158
lines changed

8 files changed

+447
-158
lines changed

‎charts/s3proxy/README.md.gotmpl‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -57,8 +57,10 @@ config:
5757
type: "aws-v4"
5858
identity: "myaccesskey"
5959
secret: "mysecretkey"
60-
backend:
61-
provider: "filesystem-nio2"
60+
backends:
61+
filesystem:
62+
enabled: true
63+
nio2: true
6264
filesystem:
6365
basedir: "/data/s3proxy"
6466
@@ -151,8 +153,10 @@ persistence:
151153
config:
152154
auth:
153155
type: "none"
154-
backend:
155-
provider: "transient-nio2" # In-memory storage
156+
backends:
157+
transient:
158+
enabled: true
159+
nio2: true # In-memory storage
156160
157161
persistence:
158162
enabled: false

‎charts/s3proxy/override-values.example.yaml‎

Lines changed: 73 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,23 @@
11
# Example values for s3proxy Helm chart
22
# This demonstrates a typical configuration using filesystem backend with authentication
33

4+
# Custom image configuration (optional)
5+
# image:
6+
# repository: andrewgaul/s3proxy
7+
# tag: "latest"
8+
# pullPolicy: Always
9+
10+
# Custom config merge container image configuration (optional)
11+
# configMergeImage:
12+
# repository: busybox
13+
# tag: "1.36"
14+
# pullPolicy: IfNotPresent
15+
416
# S3Proxy configuration
517
config:
18+
# Log level for S3Proxy (DEBUG, INFO, WARN, ERROR)
19+
logLevel: "INFO"
20+
621
auth:
722
# Authentication type for clients connecting to S3Proxy
823
# Options: none, aws-v2, aws-v4, aws-v2-or-v4
@@ -30,24 +45,67 @@ config:
3045
allowCredential: true
3146

3247
# Storage backend configuration
33-
backend:
34-
# Use filesystem backend for local storage
35-
provider: "filesystem-nio2"
48+
# Multiple backends can be enabled simultaneously
49+
# Properties files will be loaded in order: main properties first, then each backend's properties
50+
# Later properties can override earlier ones if there are conflicts
51+
backends:
52+
# Filesystem backend for local storage
3653
filesystem:
54+
enabled: true # Set to true to use filesystem backend
55+
nio2: true # Set to true for NIO2 implementation (filesystem-nio2), false for standard (filesystem)
3756
basedir: "/data/s3proxy"
3857

39-
# Example: AWS S3 backend (uncomment to use)
40-
# provider: "aws-s3"
41-
# awsS3:
42-
# region: "us-west-2"
43-
# accessKeyId: "AKIAIOSFODNN7EXAMPLE"
44-
# secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
45-
46-
# Example: Azure Blob backend (uncomment to use)
47-
# provider: "azureblob"
48-
# azureblob:
49-
# account: "mystorageaccount"
50-
# key: "base64encodedkey=="
58+
# Transient (in-memory) backend - useful for testing
59+
transient:
60+
enabled: false # Set to true to use transient backend
61+
nio2: true # Set to true for NIO2 implementation (transient-nio2), false for standard (transient)
62+
63+
# S3 backend (AWS S3 or S3-compatible storage)
64+
s3:
65+
enabled: false # Set to true to use S3 backend
66+
aws: true # Set to true for AWS S3 (aws-s3 provider), false for generic S3
67+
region: "us-west-2"
68+
# endpoint: "https://s3.amazonaws.com" # Optional custom endpoint (e.g., MinIO, Ceph)
69+
accessKeyId: "AKIAIOSFODNN7EXAMPLE"
70+
secretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
71+
72+
# Azure Blob Storage backend
73+
azureblob:
74+
enabled: false # Set to true to use Azure Blob backend
75+
provider: "azureblob" # Can be "azureblob" or "azureblob-sdk"
76+
account: "mystorageaccount"
77+
key: "base64encodedkey=="
78+
# endpoint: "https://mystorageaccount.blob.core.windows.net" # Optional
79+
# sasToken: "" # Optional SAS token
80+
81+
# Google Cloud Storage backend
82+
googleCloudStorage:
83+
enabled: false # Set to true to use GCS backend
84+
projectId: "my-project"
85+
privateKey: "-----BEGIN RSA PRIVATE KEY-----\n..."
86+
clientEmail: "service-account@my-project.iam.gserviceaccount.com"
87+
88+
# Backblaze B2 backend
89+
b2:
90+
enabled: false # Set to true to use B2 backend
91+
account: "account-id"
92+
applicationKey: "application-key"
93+
94+
# OpenStack Swift backend
95+
openstackSwift:
96+
enabled: false # Set to true to use Swift backend
97+
authUrl: "https://auth.cloud.com/v2.0"
98+
tenantName: "my-tenant"
99+
userName: "my-user"
100+
password: "my-password"
101+
region: "RegionOne"
102+
103+
# Rackspace Cloud Files backend
104+
rackspaceCloudfiles:
105+
enabled: false # Set to true to use Rackspace Cloud Files backend
106+
region: "us" # Region: "us" or "uk"
107+
userName: "my-user"
108+
apiKey: "my-api-key"
51109

52110
# Persistence settings for filesystem backend
53111
persistence:

‎charts/s3proxy/templates/NOTES.txt‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,40 @@
2222
{{- end }}
2323

2424
2. S3Proxy Configuration:
25-
- Backend Provider: {{ .Values.config.backend.provider }}
25+
- Backend Provider:
26+
{{- if .Values.config.backends.filesystem.enabled }}
27+
{{- if .Values.config.backends.filesystem.nio2 }}
28+
filesystem (filesystem-nio2)
29+
{{- else }}
30+
filesystem (filesystem)
31+
{{- end }}
32+
{{- else if .Values.config.backends.transient.enabled }}
33+
{{- if .Values.config.backends.transient.nio2 }}
34+
transient (transient-nio2)
35+
{{- else }}
36+
transient (transient)
37+
{{- end }}
38+
{{- else if .Values.config.backends.s3.enabled }}
39+
{{- if .Values.config.backends.s3.aws }}
40+
AWS S3 (aws-s3)
41+
{{- else }}
42+
Generic S3 (s3)
43+
{{- end }}
44+
{{- else if .Values.config.backends.azureblob.enabled }}
45+
Azure Blob ({{ .Values.config.backends.azureblob.provider }})
46+
{{- else if .Values.config.backends.googleCloudStorage.enabled }}
47+
Google Cloud Storage
48+
{{- else if .Values.config.backends.b2.enabled }}
49+
Backblaze B2
50+
{{- else if .Values.config.backends.openstackSwift.enabled }}
51+
OpenStack Swift
52+
{{- else if .Values.config.backends.rackspaceCloudfiles.enabled }}
53+
{{- if eq .Values.config.backends.rackspaceCloudfiles.region "uk" }}
54+
Rackspace Cloud Files UK
55+
{{- else }}
56+
Rackspace Cloud Files US
57+
{{- end }}
58+
{{- end }}
2659
- Authorization: {{ .Values.config.auth.type }}
2760
{{- if .Values.config.cors.enabled }}
2861
- CORS: Enabled
@@ -70,12 +103,22 @@
70103
kubectl logs -n {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "s3proxy.name" . }},app.kubernetes.io/instance={{ .Release.Name }}"
71104

72105
5. Important Notes:
73-
{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }}
74-
- Using filesystem backend at: {{ .Values.config.backend.filesystem.basedir }}
106+
{{- if .Values.config.backends.filesystem.enabled }}
107+
- Using filesystem backend at: {{ .Values.config.backends.filesystem.basedir }}
75108
{{- end }}
76109
{{- if not .Values.config.auth.identity }}
77110
- WARNING: S3Proxy identity not configured. Remember to set config.auth.identity and config.auth.secret for authentication.
78111
{{- end }}
79-
{{- if and (ne .Values.config.backend.provider "filesystem") (ne .Values.config.backend.provider "filesystem-nio2") (ne .Values.config.backend.provider "transient") (ne .Values.config.backend.provider "transient-nio2") }}
80-
- Make sure to configure backend credentials for {{ .Values.config.backend.provider }}
112+
{{- if .Values.config.backends.s3.enabled }}
113+
- Make sure to configure S3 backend credentials
114+
{{- else if .Values.config.backends.azureblob.enabled }}
115+
- Make sure to configure Azure Blob backend credentials
116+
{{- else if .Values.config.backends.googleCloudStorage.enabled }}
117+
- Make sure to configure Google Cloud Storage backend credentials
118+
{{- else if .Values.config.backends.b2.enabled }}
119+
- Make sure to configure Backblaze B2 backend credentials
120+
{{- else if .Values.config.backends.openstackSwift.enabled }}
121+
- Make sure to configure OpenStack Swift backend credentials
122+
{{- else if .Values.config.backends.rackspaceCloudfiles.enabled }}
123+
- Make sure to configure Rackspace Cloud Files backend credentials
81124
{{- end }}

‎charts/s3proxy/templates/configmap.yaml‎

Lines changed: 112 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,4 @@
1-
apiVersion: v1
2-
kind: ConfigMap
3-
metadata:
4-
name: {{ include "s3proxy.fullname" . }}
5-
labels:
6-
{{- include "s3proxy.labels" . | nindent 4 }}
7-
data:
8-
s3proxy.properties: |
1+
{{- define "s3proxy.main.config" -}}
92
# S3Proxy configuration
103
s3proxy.endpoint=http://0.0.0.0:{{ .Values.service.targetPort }}
114
s3proxy.authorization={{ .Values.config.auth.type }}
@@ -14,9 +7,8 @@ data:
147
{{- end }}
158

169
{{- if ne .Values.config.auth.type "none" }}
17-
# These will be overridden by environment variables from the secret
18-
s3proxy.identity=${S3PROXY_IDENTITY}
19-
s3proxy.credential=${S3PROXY_CREDENTIAL}
10+
# Authentication credentials will be merged from the secret properties file
11+
# s3proxy.identity and s3proxy.credential will be provided by the secret
2012
{{- end }}
2113

2214
{{- if .Values.config.cors.enabled }}
@@ -68,57 +60,122 @@ data:
6860
# Large object mocking middleware
6961
s3proxy.large-object-mocking=true
7062
{{- end }}
63+
{{- end }}
64+
65+
apiVersion: v1
66+
kind: ConfigMap
67+
metadata:
68+
name: {{ include "s3proxy.fullname" . }}
69+
labels:
70+
{{- include "s3proxy.labels" . | nindent 4 }}
71+
data:
72+
{{- if .Values.config.backends.filesystem.enabled }}
73+
backend-filesystem.properties: |
74+
{{- include "s3proxy.main.config" . | nindent 4 }}
75+
76+
# Filesystem backend configuration
77+
{{- if .Values.config.backends.filesystem.nio2 }}
78+
jclouds.provider=filesystem-nio2
79+
{{- else }}
80+
jclouds.provider=filesystem
81+
{{- end }}
82+
jclouds.filesystem.basedir={{ .Values.config.backends.filesystem.basedir }}
83+
{{- end }}
7184

72-
# JClouds backend configuration
73-
jclouds.provider={{ .Values.config.backend.provider }}
85+
{{- if .Values.config.backends.transient.enabled }}
86+
backend-transient.properties: |
87+
{{- include "s3proxy.main.config" . | nindent 4 }}
7488

75-
{{- if or (eq .Values.config.backend.provider "filesystem") (eq .Values.config.backend.provider "filesystem-nio2") }}
76-
# Filesystem backend
77-
jclouds.filesystem.basedir={{ .Values.config.backend.filesystem.basedir }}
78-
{{- else if or (eq .Values.config.backend.provider "aws-s3") (eq .Values.config.backend.provider "s3") }}
79-
# AWS S3 backend
80-
{{- if .Values.config.backend.awsS3.region }}
81-
jclouds.region={{ .Values.config.backend.awsS3.region }}
89+
# Transient backend configuration
90+
{{- if .Values.config.backends.transient.nio2 }}
91+
jclouds.provider=transient-nio2
92+
{{- else }}
93+
jclouds.provider=transient
8294
{{- end }}
83-
{{- if .Values.config.backend.awsS3.endpoint }}
84-
jclouds.endpoint={{ .Values.config.backend.awsS3.endpoint }}
95+
{{- end }}
96+
97+
{{- if .Values.config.backends.s3.enabled }}
98+
backend-s3.properties: |
99+
{{- include "s3proxy.main.config" . | nindent 4 }}
100+
101+
# S3 backend configuration
102+
{{- if .Values.config.backends.s3.aws }}
103+
jclouds.provider=aws-s3
104+
{{- else }}
105+
jclouds.provider=s3
85106
{{- end }}
86-
# Credentials will be set via environment variables
87-
jclouds.identity=${JCLOUDS_IDENTITY}
88-
jclouds.credential=${JCLOUDS_CREDENTIAL}
89-
{{- else if or (eq .Values.config.backend.provider "azureblob") (eq .Values.config.backend.provider "azureblob-sdk") }}
90-
# Azure Blob backend
91-
{{- if .Values.config.backend.azureblob.endpoint }}
92-
jclouds.azureblob.endpoint={{ .Values.config.backend.azureblob.endpoint }}
107+
{{- if .Values.config.backends.s3.region }}
108+
jclouds.region={{ .Values.config.backends.s3.region }}
93109
{{- end }}
94-
# Credentials will be set via environment variables
95-
jclouds.identity=${JCLOUDS_IDENTITY}
96-
jclouds.credential=${JCLOUDS_CREDENTIAL}
97-
{{- if .Values.config.backend.azureblob.sasToken }}
98-
jclouds.azureblob.sas=${JCLOUDS_AZURE_SAS}
110+
{{- if .Values.config.backends.s3.endpoint }}
111+
jclouds.endpoint={{ .Values.config.backends.s3.endpoint }}
99112
{{- end }}
100-
{{- else if eq .Values.config.backend.provider "google-cloud-storage" }}
101-
# Google Cloud Storage backend
102-
{{- if .Values.config.backend.googleCloudStorage.projectId }}
103-
jclouds.project-id={{ .Values.config.backend.googleCloudStorage.projectId }}
113+
# Credentials will be merged from the secret properties file
114+
# jclouds.identity and jclouds.credential will be provided by the secret
115+
{{- end }}
116+
117+
{{- if .Values.config.backends.azureblob.enabled }}
118+
backend-azureblob.properties: |
119+
{{- include "s3proxy.main.config" . | nindent 4 }}
120+
121+
# Azure Blob backend configuration
122+
jclouds.provider={{ .Values.config.backends.azureblob.provider }}
123+
{{- if .Values.config.backends.azureblob.endpoint }}
124+
jclouds.azureblob.endpoint={{ .Values.config.backends.azureblob.endpoint }}
104125
{{- end }}
105-
# Credentials will be set via environment variables
106-
jclouds.identity=${JCLOUDS_IDENTITY}
107-
jclouds.credential=${JCLOUDS_CREDENTIAL}
108-
{{- else if eq .Values.config.backend.provider "b2" }}
109-
# Backblaze B2 backend
110-
# Credentials will be set via environment variables
111-
jclouds.identity=${JCLOUDS_IDENTITY}
112-
jclouds.credential=${JCLOUDS_CREDENTIAL}
113-
{{- else if eq .Values.config.backend.provider "openstack-swift" }}
114-
# OpenStack Swift backend
115-
{{- if .Values.config.backend.swift.authUrl }}
116-
jclouds.keystone.auth-url={{ .Values.config.backend.swift.authUrl }}
126+
# Credentials will be merged from the secret properties file
127+
# jclouds.identity, jclouds.credential, and jclouds.azureblob.sas will be provided by the secret
128+
{{- end }}
129+
130+
{{- if .Values.config.backends.googleCloudStorage.enabled }}
131+
backend-google-cloud-storage.properties: |
132+
{{- include "s3proxy.main.config" . | nindent 4 }}
133+
134+
# Google Cloud Storage backend configuration
135+
jclouds.provider=google-cloud-storage
136+
{{- if .Values.config.backends.googleCloudStorage.projectId }}
137+
jclouds.project-id={{ .Values.config.backends.googleCloudStorage.projectId }}
117138
{{- end }}
118-
{{- if .Values.config.backend.swift.region }}
119-
jclouds.region={{ .Values.config.backend.swift.region }}
139+
# Credentials will be merged from the secret properties file
140+
# jclouds.identity and jclouds.credential will be provided by the secret
141+
{{- end }}
142+
143+
{{- if .Values.config.backends.b2.enabled }}
144+
backend-b2.properties: |
145+
{{- include "s3proxy.main.config" . | nindent 4 }}
146+
147+
# Backblaze B2 backend configuration
148+
jclouds.provider=b2
149+
# Credentials will be merged from the secret properties file
150+
# jclouds.identity and jclouds.credential will be provided by the secret
151+
{{- end }}
152+
153+
{{- if .Values.config.backends.openstackSwift.enabled }}
154+
backend-openstack-swift.properties: |
155+
{{- include "s3proxy.main.config" . | nindent 4 }}
156+
157+
# OpenStack Swift backend configuration
158+
jclouds.provider=openstack-swift
159+
{{- if .Values.config.backends.openstackSwift.authUrl }}
160+
jclouds.keystone.auth-url={{ .Values.config.backends.openstackSwift.authUrl }}
120161
{{- end }}
121-
# Credentials will be set via environment variables
122-
jclouds.identity=${JCLOUDS_IDENTITY}
123-
jclouds.credential=${JCLOUDS_CREDENTIAL}
162+
{{- if .Values.config.backends.openstackSwift.region }}
163+
jclouds.region={{ .Values.config.backends.openstackSwift.region }}
164+
{{- end }}
165+
# Credentials will be merged from the secret properties file
166+
# jclouds.identity and jclouds.credential will be provided by the secret
124167
{{- end }}
168+
169+
{{- if .Values.config.backends.rackspaceCloudfiles.enabled }}
170+
backend-rackspace-cloudfiles.properties: |
171+
{{- include "s3proxy.main.config" . | nindent 4 }}
172+
173+
# Rackspace Cloud Files backend configuration
174+
{{- if eq .Values.config.backends.rackspaceCloudfiles.region "uk" }}
175+
jclouds.provider=rackspace-cloudfiles-uk
176+
{{- else }}
177+
jclouds.provider=rackspace-cloudfiles-us
178+
{{- end }}
179+
# Credentials will be merged from the secret properties file
180+
# jclouds.identity and jclouds.credential will be provided by the secret
181+
{{- end }}

0 commit comments

Comments
 (0)