Skip to content

Commit 4086750

Browse files
RostislavDuginRostislavDugin
committed
FEATURE (s3): Add support of virtual-styled-domains and S3 prefix
1 parent 0bc9338 commit 4086750

18 files changed

+321
-101
lines changed

‎assets/logo.svg‎

Lines changed: 39 additions & 20 deletions
Loading

‎backend/internal/features/storages/models/s3/model.go‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ type S3Storage struct {
2222
S3AccessKey string `json:"s3AccessKey" gorm:"not null;type:text;column:s3_access_key"`
2323
S3SecretKey string `json:"s3SecretKey" gorm:"not null;type:text;column:s3_secret_key"`
2424
S3Endpoint string `json:"s3Endpoint" gorm:"type:text;column:s3_endpoint"`
25+
26+
S3Prefix string `json:"s3Prefix" gorm:"type:text;column:s3_prefix"`
27+
S3UseVirtualHostedStyle bool `json:"s3UseVirtualHostedStyle" gorm:"default:false;column:s3_use_virtual_hosted_style"`
2528
}
2629

2730
func (s *S3Storage) TableName() string {
@@ -34,11 +37,13 @@ func (s *S3Storage) SaveFile(logger *slog.Logger, fileID uuid.UUID, file io.Read
3437
return err
3538
}
3639

40+
objectKey := s.buildObjectKey(fileID.String())
41+
3742
// Upload the file using MinIO client with streaming (size = -1 for unknown size)
3843
_, err = client.PutObject(
3944
context.TODO(),
4045
s.S3Bucket,
41-
fileID.String(),
46+
objectKey,
4247
file,
4348
-1,
4449
minio.PutObjectOptions{},
@@ -56,10 +61,12 @@ func (s *S3Storage) GetFile(fileID uuid.UUID) (io.ReadCloser, error) {
5661
return nil, err
5762
}
5863

64+
objectKey := s.buildObjectKey(fileID.String())
65+
5966
object, err := client.GetObject(
6067
context.TODO(),
6168
s.S3Bucket,
62-
fileID.String(),
69+
objectKey,
6370
minio.GetObjectOptions{},
6471
)
6572
if err != nil {
@@ -90,11 +97,13 @@ func (s *S3Storage) DeleteFile(fileID uuid.UUID) error {
9097
return err
9198
}
9299

100+
objectKey := s.buildObjectKey(fileID.String())
101+
93102
// Delete the object using MinIO client
94103
err = client.RemoveObject(
95104
context.TODO(),
96105
s.S3Bucket,
97-
fileID.String(),
106+
objectKey,
98107
minio.RemoveObjectOptions{},
99108
)
100109
if err != nil {
@@ -150,14 +159,15 @@ func (s *S3Storage) TestConnection() error {
150159

151160
// Test write and delete permissions by uploading and removing a small test file
152161
testFileID := uuid.New().String() + "-test"
162+
testObjectKey := s.buildObjectKey(testFileID)
153163
testData := []byte("test connection")
154164
testReader := bytes.NewReader(testData)
155165

156166
// Upload test file
157167
_, err = client.PutObject(
158168
ctx,
159169
s.S3Bucket,
160-
testFileID,
170+
testObjectKey,
161171
testReader,
162172
int64(len(testData)),
163173
minio.PutObjectOptions{},
@@ -170,7 +180,7 @@ func (s *S3Storage) TestConnection() error {
170180
err = client.RemoveObject(
171181
ctx,
172182
s.S3Bucket,
173-
testFileID,
183+
testObjectKey,
174184
minio.RemoveObjectOptions{},
175185
)
176186
if err != nil {
@@ -189,6 +199,7 @@ func (s *S3Storage) Update(incoming *S3Storage) {
189199
s.S3Bucket = incoming.S3Bucket
190200
s.S3Region = incoming.S3Region
191201
s.S3Endpoint = incoming.S3Endpoint
202+
s.S3UseVirtualHostedStyle = incoming.S3UseVirtualHostedStyle
192203

193204
if incoming.S3AccessKey != "" {
194205
s.S3AccessKey = incoming.S3AccessKey
@@ -197,6 +208,24 @@ func (s *S3Storage) Update(incoming *S3Storage) {
197208
if incoming.S3SecretKey != "" {
198209
s.S3SecretKey = incoming.S3SecretKey
199210
}
211+
212+
// we do not allow to change the prefix after creation,
213+
// otherwise we will have to migrate all the data to the new prefix
214+
}
215+
216+
func (s *S3Storage) buildObjectKey(fileName string) string {
217+
if s.S3Prefix == "" {
218+
return fileName
219+
}
220+
221+
prefix := s.S3Prefix
222+
prefix = strings.TrimPrefix(prefix, "/")
223+
224+
if !strings.HasSuffix(prefix, "/") {
225+
prefix = prefix + "/"
226+
}
227+
228+
return prefix + fileName
200229
}
201230

202231
func (s *S3Storage) getClient() (*minio.Client, error) {
@@ -215,11 +244,18 @@ func (s *S3Storage) getClient() (*minio.Client, error) {
215244
endpoint = fmt.Sprintf("s3.%s.amazonaws.com", s.S3Region)
216245
}
217246

247+
// Configure bucket lookup strategy
248+
bucketLookup := minio.BucketLookupAuto
249+
if s.S3UseVirtualHostedStyle {
250+
bucketLookup = minio.BucketLookupDNS
251+
}
252+
218253
// Initialize the MinIO client
219254
minioClient, err := minio.New(endpoint, &minio.Options{
220-
Creds: credentials.NewStaticV4(s.S3AccessKey, s.S3SecretKey, ""),
221-
Secure: useSSL,
222-
Region: s.S3Region,
255+
Creds: credentials.NewStaticV4(s.S3AccessKey, s.S3SecretKey, ""),
256+
Secure: useSSL,
257+
Region: s.S3Region,
258+
BucketLookup: bucketLookup,
223259
})
224260
if err != nil {
225261
return nil, fmt.Errorf("failed to initialize MinIO client: %w", err)

‎backend/migrations/20251116075135_add_virtual_host_and_prefix_to_s3.sql‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
-- +goose Up
2+
-- +goose StatementBegin
3+
ALTER TABLE s3_storages
4+
ADD COLUMN s3_prefix TEXT;
5+
6+
ALTER TABLE s3_storages
7+
ADD COLUMN s3_use_virtual_hosted_style BOOLEAN NOT NULL DEFAULT FALSE;
8+
-- +goose StatementEnd
9+
10+
-- +goose Down
11+
-- +goose StatementBegin
12+
ALTER TABLE s3_storages
13+
DROP COLUMN s3_use_virtual_hosted_style;
14+
15+
ALTER TABLE s3_storages
16+
DROP COLUMN s3_prefix;
17+
-- +goose StatementEnd

‎frontend/src/entity/storages/models/S3Storage.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,6 @@ export interface S3Storage {
44
s3AccessKey: string;
55
s3SecretKey: string;
66
s3Endpoint?: string;
7+
s3Prefix?: string;
8+
s3UseVirtualHostedStyle?: boolean;
79
}

‎frontend/src/features/databases/ui/CreateDatabaseComponent.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import { EditDatabaseSpecificDataComponent } from './edit/EditDatabaseSpecificDa
1616
interface Props {
1717
workspaceId: string;
1818

19-
onCreated: () => void;
19+
onCreated: (databaseId: string) => void;
2020
onClose: () => void;
2121
}
2222

@@ -58,7 +58,7 @@ export const CreateDatabaseComponent = ({ workspaceId, onCreated, onClose }: Pro
5858
await backupsApi.makeBackup(createdDatabase.id);
5959
}
6060

61-
onCreated();
61+
onCreated(createdDatabase.id);
6262
onClose();
6363
} catch (error) {
6464
alert(error);

‎frontend/src/features/databases/ui/DatabasesComponent.tsx‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ interface Props {
1414
isCanManageDBs: boolean;
1515
}
1616

17+
const SELECTED_DATABASE_STORAGE_KEY = 'selectedDatabaseId';
18+
1719
export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }: Props) => {
1820
const [isLoading, setIsLoading] = useState(true);
1921
const [databases, setDatabases] = useState<Database[]>([]);
@@ -22,7 +24,16 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }:
2224
const [isShowAddDatabase, setIsShowAddDatabase] = useState(false);
2325
const [selectedDatabaseId, setSelectedDatabaseId] = useState<string | undefined>(undefined);
2426

25-
const loadDatabases = (isSilent = false) => {
27+
const updateSelectedDatabaseId = (databaseId: string | undefined) => {
28+
setSelectedDatabaseId(databaseId);
29+
if (databaseId) {
30+
localStorage.setItem(`${SELECTED_DATABASE_STORAGE_KEY}_${workspace.id}`, databaseId);
31+
} else {
32+
localStorage.removeItem(`${SELECTED_DATABASE_STORAGE_KEY}_${workspace.id}`);
33+
}
34+
};
35+
36+
const loadDatabases = (isSilent = false, selectDatabaseId?: string) => {
2637
if (!isSilent) {
2738
setIsLoading(true);
2839
}
@@ -31,8 +42,17 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }:
3142
.getDatabases(workspace.id)
3243
.then((databases) => {
3344
setDatabases(databases);
34-
if (!selectedDatabaseId && !isSilent) {
35-
setSelectedDatabaseId(databases[0]?.id);
45+
if (selectDatabaseId) {
46+
updateSelectedDatabaseId(selectDatabaseId);
47+
} else if (!selectedDatabaseId && !isSilent) {
48+
const savedDatabaseId = localStorage.getItem(
49+
`${SELECTED_DATABASE_STORAGE_KEY}_${workspace.id}`,
50+
);
51+
const databaseToSelect =
52+
savedDatabaseId && databases.some((db) => db.id === savedDatabaseId)
53+
? savedDatabaseId
54+
: databases[0]?.id;
55+
updateSelectedDatabaseId(databaseToSelect);
3656
}
3757
})
3858
.catch((e) => alert(e.message))
@@ -95,7 +115,7 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }:
95115
key={database.id}
96116
database={database}
97117
selectedDatabaseId={selectedDatabaseId}
98-
setSelectedDatabaseId={setSelectedDatabaseId}
118+
setSelectedDatabaseId={updateSelectedDatabaseId}
99119
/>
100120
))
101121
: searchQuery && (
@@ -119,10 +139,11 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }:
119139
loadDatabases();
120140
}}
121141
onDatabaseDeleted={() => {
122-
loadDatabases();
123-
setSelectedDatabaseId(
124-
databases.filter((database) => database.id !== selectedDatabaseId)[0]?.id,
142+
const remainingDatabases = databases.filter(
143+
(database) => database.id !== selectedDatabaseId,
125144
);
145+
updateSelectedDatabaseId(remainingDatabases[0]?.id);
146+
loadDatabases();
126147
}}
127148
isCanManageDBs={isCanManageDBs}
128149
/>
@@ -141,8 +162,8 @@ export const DatabasesComponent = ({ contentHeight, workspace, isCanManageDBs }:
141162

142163
<CreateDatabaseComponent
143164
workspaceId={workspace.id}
144-
onCreated={() => {
145-
loadDatabases();
165+
onCreated={(databaseId) => {
166+
loadDatabases(false, databaseId);
146167
setIsShowAddDatabase(false);
147168
}}
148169
onClose={() => setIsShowAddDatabase(false)}

‎frontend/src/features/notifiers/ui/edit/EditNotifierComponent.tsx‎

Lines changed: 24 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -255,46 +255,64 @@ export function EditNotifierComponent({
255255
<EditTelegramNotifierComponent
256256
notifier={notifier}
257257
setNotifier={setNotifier}
258-
setIsUnsaved={setIsUnsaved}
258+
setUnsaved={() => {
259+
setIsUnsaved(true);
260+
setIsTestNotificationSuccess(false);
261+
}}
259262
/>
260263
)}
261264

262265
{notifier?.notifierType === NotifierType.EMAIL && (
263266
<EditEmailNotifierComponent
264267
notifier={notifier}
265268
setNotifier={setNotifier}
266-
setIsUnsaved={setIsUnsaved}
269+
setUnsaved={() => {
270+
setIsUnsaved(true);
271+
setIsTestNotificationSuccess(false);
272+
}}
267273
/>
268274
)}
269275

270276
{notifier?.notifierType === NotifierType.WEBHOOK && (
271277
<EditWebhookNotifierComponent
272278
notifier={notifier}
273279
setNotifier={setNotifier}
274-
setIsUnsaved={setIsUnsaved}
280+
setUnsaved={() => {
281+
setIsUnsaved(true);
282+
setIsTestNotificationSuccess(false);
283+
}}
275284
/>
276285
)}
277286

278287
{notifier?.notifierType === NotifierType.SLACK && (
279288
<EditSlackNotifierComponent
280289
notifier={notifier}
281290
setNotifier={setNotifier}
282-
setIsUnsaved={setIsUnsaved}
291+
setUnsaved={() => {
292+
setIsUnsaved(true);
293+
setIsTestNotificationSuccess(false);
294+
}}
283295
/>
284296
)}
285297

286298
{notifier?.notifierType === NotifierType.DISCORD && (
287299
<EditDiscordNotifierComponent
288300
notifier={notifier}
289301
setNotifier={setNotifier}
290-
setIsUnsaved={setIsUnsaved}
302+
setUnsaved={() => {
303+
setIsUnsaved(true);
304+
setIsTestNotificationSuccess(false);
305+
}}
291306
/>
292307
)}
293308
{notifier?.notifierType === NotifierType.TEAMS && (
294309
<EditTeamsNotifierComponent
295310
notifier={notifier}
296311
setNotifier={setNotifier}
297-
setIsUnsaved={setIsUnsaved}
312+
setUnsaved={() => {
313+
setIsUnsaved(true);
314+
setIsTestNotificationSuccess(false);
315+
}}
298316
/>
299317
)}
300318
</div>

‎frontend/src/features/notifiers/ui/edit/notifiers/EditDiscordNotifierComponent.tsx‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ import type { Notifier } from '../../../../../entity/notifiers';
55
interface Props {
66
notifier: Notifier;
77
setNotifier: (notifier: Notifier) => void;
8-
setIsUnsaved: (isUnsaved: boolean) => void;
8+
setUnsaved: () => void;
99
}
1010

11-
export function EditDiscordNotifierComponent({ notifier, setNotifier, setIsUnsaved }: Props) {
11+
export function EditDiscordNotifierComponent({ notifier, setNotifier, setUnsaved }: Props) {
1212
return (
1313
<>
1414
<div className="flex">
@@ -26,7 +26,7 @@ export function EditDiscordNotifierComponent({ notifier, setNotifier, setIsUnsav
2626
channelWebhookUrl: e.target.value.trim(),
2727
},
2828
});
29-
setIsUnsaved(true);
29+
setUnsaved();
3030
}}
3131
size="small"
3232
className="w-full"

0 commit comments

Comments
 (0)