Skip to content

Commit 5bf4abd

Browse files
n1ru4lkamilkisiela
n1ru4l
and
kamilkisiela
authored
feat: write artifacts and persisted documents to s3 mirror (#5538)
Co-authored-by: Kamil Kisiela <[email protected]>
1 parent 785dc52 commit 5bf4abd

File tree

15 files changed

+300
-176
lines changed

15 files changed

+300
-176
lines changed

‎deployment/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import { deployPostgres } from './services/postgres';
1919
import { deployProxy } from './services/proxy';
2020
import { deployRateLimit } from './services/rate-limit';
2121
import { deployRedis } from './services/redis';
22-
import { deployS3 } from './services/s3';
22+
import { deployS3, deployS3Mirror } from './services/s3';
2323
import { deploySchema } from './services/schema';
2424
import { configureSentry } from './services/sentry';
2525
import { deploySentryEventsMonitor } from './services/sentry-events';
@@ -80,6 +80,7 @@ const postgres = deployPostgres();
8080
const redis = deployRedis({ environment });
8181
const kafka = deployKafka();
8282
const s3 = deployS3();
83+
const s3Mirror = deployS3Mirror();
8384

8485
const cdn = deployCFCDN({
8586
s3,
@@ -240,6 +241,7 @@ const graphql = deployGraphQL({
240241
emails,
241242
supertokens,
242243
s3,
244+
s3Mirror,
243245
zendesk,
244246
githubApp,
245247
sentry,

‎deployment/services/graphql.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ export function deployGraphQL({
5050
emails,
5151
supertokens,
5252
s3,
53+
s3Mirror,
5354
zendesk,
5455
docker,
5556
postgres,
@@ -70,6 +71,7 @@ export function deployGraphQL({
7071
redis: Redis;
7172
cdn: CDN;
7273
s3: S3;
74+
s3Mirror: S3;
7375
usage: Usage;
7476
usageEstimator: UsageEstimator;
7577
dbMigrations: DbMigrations;
@@ -151,6 +153,7 @@ export function deployGraphQL({
151153
observability.enabled && observability.tracingEndpoint
152154
? observability.tracingEndpoint
153155
: '',
156+
S3_MIRROR: '1',
154157
},
155158
exposesMetrics: true,
156159
port: 4000,
@@ -193,6 +196,11 @@ export function deployGraphQL({
193196
.withSecret('S3_SECRET_ACCESS_KEY', s3.secret, 'secretAccessKey')
194197
.withSecret('S3_BUCKET_NAME', s3.secret, 'bucket')
195198
.withSecret('S3_ENDPOINT', s3.secret, 'endpoint')
199+
// S3 Mirror
200+
.withSecret('S3_MIRROR_ACCESS_KEY_ID', s3Mirror.secret, 'accessKeyId')
201+
.withSecret('S3_MIRROR_SECRET_ACCESS_KEY', s3Mirror.secret, 'secretAccessKey')
202+
.withSecret('S3_MIRROR_BUCKET_NAME', s3Mirror.secret, 'bucket')
203+
.withSecret('S3_MIRROR_ENDPOINT', s3Mirror.secret, 'endpoint')
196204
// Auth
197205
.withSecret('SUPERTOKENS_API_KEY', supertokens.secret, 'apiKey')
198206
.withSecret('AUTH_GITHUB_CLIENT_ID', githubOAuthSecret, 'clientId')

‎deployment/services/s3.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,17 @@ export function deployS3() {
2121
return { secret };
2222
}
2323

24+
export function deployS3Mirror() {
25+
const s3Config = new pulumi.Config('s3');
26+
27+
const secret = new S3Secret('aws-s3', {
28+
endpoint: s3Config.require('endpoint'),
29+
bucket: s3Config.require('bucketName'),
30+
accessKeyId: s3Config.requireSecret('accessKeyId'),
31+
secretAccessKey: s3Config.requireSecret('secretAccessKey'),
32+
});
33+
34+
return { secret };
35+
}
36+
2437
export type S3 = ReturnType<typeof deployS3>;

‎packages/services/api/src/create.ts‎

Lines changed: 33 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ export function createRegistry({
107107
githubApp,
108108
cdn,
109109
s3,
110+
s3Mirror,
110111
encryptionSecret,
111112
feedback,
112113
billing,
@@ -135,6 +136,13 @@ export function createRegistry({
135136
secretAccessKeyId: string;
136137
sessionToken?: string;
137138
};
139+
s3Mirror: {
140+
bucketName: string;
141+
endpoint: string;
142+
accessKeyId: string;
143+
secretAccessKeyId: string;
144+
sessionToken?: string;
145+
} | null;
138146
encryptionSecret: string;
139147
feedback: {
140148
token: string;
@@ -150,16 +158,31 @@ export function createRegistry({
150158
organizationOIDC: boolean;
151159
pubSub: HivePubSub;
152160
}) {
153-
const s3Config: S3Config = {
154-
client: new AwsClient({
155-
accessKeyId: s3.accessKeyId,
156-
secretAccessKey: s3.secretAccessKeyId,
157-
sessionToken: s3.sessionToken,
158-
service: 's3',
159-
}),
160-
bucket: s3.bucketName,
161-
endpoint: s3.endpoint,
162-
};
161+
const s3Config: S3Config = [
162+
{
163+
client: new AwsClient({
164+
accessKeyId: s3.accessKeyId,
165+
secretAccessKey: s3.secretAccessKeyId,
166+
sessionToken: s3.sessionToken,
167+
service: 's3',
168+
}),
169+
bucket: s3.bucketName,
170+
endpoint: s3.endpoint,
171+
},
172+
];
173+
174+
if (s3Mirror) {
175+
s3Config.push({
176+
client: new AwsClient({
177+
accessKeyId: s3Mirror.accessKeyId,
178+
secretAccessKey: s3Mirror.secretAccessKeyId,
179+
sessionToken: s3Mirror.sessionToken,
180+
service: 's3',
181+
}),
182+
bucket: s3Mirror.bucketName,
183+
endpoint: s3Mirror.endpoint,
184+
});
185+
}
163186

164187
const artifactStorageWriter = new ArtifactStorageWriter(s3Config, logger);
165188

‎packages/services/api/src/modules/app-deployments/providers/app-deployments.ts‎

Lines changed: 54 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -388,30 +388,32 @@ export class AppDeployments {
388388
};
389389
}
390390

391-
const result = await this.s3.client.fetch(
392-
[
393-
this.s3.endpoint,
394-
this.s3.bucket,
395-
buildAppDeploymentIsEnabledKey(
396-
appDeployment.targetId,
397-
appDeployment.name,
398-
appDeployment.version,
399-
),
400-
].join('/'),
401-
{
402-
method: 'PUT',
403-
body: '1',
404-
headers: {
405-
'content-type': 'text/plain',
406-
},
407-
aws: {
408-
signQuery: true,
391+
for (const s3 of this.s3) {
392+
const result = await s3.client.fetch(
393+
[
394+
s3.endpoint,
395+
s3.bucket,
396+
buildAppDeploymentIsEnabledKey(
397+
appDeployment.targetId,
398+
appDeployment.name,
399+
appDeployment.version,
400+
),
401+
].join('/'),
402+
{
403+
method: 'PUT',
404+
body: '1',
405+
headers: {
406+
'content-type': 'text/plain',
407+
},
408+
aws: {
409+
signQuery: true,
410+
},
409411
},
410-
},
411-
);
412+
);
412413

413-
if (result.statusCode !== 200) {
414-
throw new Error(`Failed to enable app deployment: ${result.statusMessage}`);
414+
if (result.statusCode !== 200) {
415+
throw new Error(`Failed to enable app deployment: ${result.statusMessage}`);
416+
}
415417
}
416418

417419
const updatedAppDeployment = await this.pool
@@ -527,36 +529,38 @@ export class AppDeployments {
527529
};
528530
}
529531

530-
const result = await this.s3.client.fetch(
531-
[
532-
this.s3.endpoint,
533-
this.s3.bucket,
534-
buildAppDeploymentIsEnabledKey(
535-
appDeployment.targetId,
536-
appDeployment.name,
537-
appDeployment.version,
538-
),
539-
].join('/'),
540-
{
541-
method: 'DELETE',
542-
aws: {
543-
signQuery: true,
532+
for (const s3 of this.s3) {
533+
const result = await s3.client.fetch(
534+
[
535+
s3.endpoint,
536+
s3.bucket,
537+
buildAppDeploymentIsEnabledKey(
538+
appDeployment.targetId,
539+
appDeployment.name,
540+
appDeployment.version,
541+
),
542+
].join('/'),
543+
{
544+
method: 'DELETE',
545+
aws: {
546+
signQuery: true,
547+
},
544548
},
545-
},
546-
);
547-
548-
/** We receive a 204 status code if the DELETE operation was successful */
549-
if (result.statusCode !== 204) {
550-
this.logger.error(
551-
'Failed to disable app deployment (organizationId=%s, targetId=%s, appDeploymentId=%s, statusCode=%s)',
552-
args.organizationId,
553-
args.targetId,
554-
appDeployment.id,
555-
result.statusCode,
556-
);
557-
throw new Error(
558-
`Failed to disable app deployment. Request failed with status code "${result.statusMessage}".`,
559549
);
550+
551+
/** We receive a 204 status code if the DELETE operation was successful */
552+
if (result.statusCode !== 204) {
553+
this.logger.error(
554+
'Failed to disable app deployment (organizationId=%s, targetId=%s, appDeploymentId=%s, statusCode=%s)',
555+
args.organizationId,
556+
args.targetId,
557+
appDeployment.id,
558+
result.statusCode,
559+
);
560+
throw new Error(
561+
`Failed to disable app deployment. Request failed with status code "${result.statusMessage}".`,
562+
);
563+
}
560564
}
561565

562566
await this.clickhouse.query({

‎packages/services/api/src/modules/app-deployments/providers/persisted-document-ingester.ts‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -305,9 +305,8 @@ export class PersistedDocumentIngester {
305305

306306
tasks.push(
307307
this.promiseQueue.add(async () => {
308-
const response = await this.s3.client.fetch(
309-
[this.s3.endpoint, this.s3.bucket, s3Key].join('/'),
310-
{
308+
for (const s3 of this.s3) {
309+
const response = await s3.client.fetch([s3.endpoint, s3.bucket, s3Key].join('/'), {
311310
method: 'PUT',
312311
headers: {
313312
'content-type': 'text/plain',
@@ -317,13 +316,13 @@ export class PersistedDocumentIngester {
317316
// This boolean makes Google Cloud Storage & AWS happy.
318317
signQuery: true,
319318
},
320-
},
321-
);
319+
});
322320

323-
if (response.statusCode !== 200) {
324-
throw new Error(
325-
`Failed to upload operation to S3: [${response.statusCode}] ${response.statusMessage}`,
326-
);
321+
if (response.statusCode !== 200) {
322+
throw new Error(
323+
`Failed to upload operation to S3: [${response.statusCode}] ${response.statusMessage}`,
324+
);
325+
}
327326
}
328327
}),
329328
);

‎packages/services/api/src/modules/app-deployments/worker/persisted-documents-worker.ts‎

Lines changed: 34 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,15 @@ export function createWorker(
2727
readonly sessionToken: string | undefined;
2828
};
2929
};
30+
s3Mirror: {
31+
readonly bucketName: string;
32+
readonly endpoint: string;
33+
readonly credentials: {
34+
readonly accessKeyId: string;
35+
readonly secretAccessKey: string;
36+
readonly sessionToken: string | undefined;
37+
};
38+
} | null;
3039
clickhouse: {
3140
readonly host: string;
3241
readonly port: number;
@@ -36,16 +45,31 @@ export function createWorker(
3645
};
3746
},
3847
) {
39-
const s3Config: S3Config = {
40-
client: new AwsClient({
41-
accessKeyId: env.s3.credentials.accessKeyId,
42-
secretAccessKey: env.s3.credentials.secretAccessKey,
43-
sessionToken: env.s3.credentials.sessionToken,
44-
service: 's3',
45-
}),
46-
bucket: env.s3.bucketName,
47-
endpoint: env.s3.endpoint,
48-
};
48+
const s3Config: S3Config = [
49+
{
50+
client: new AwsClient({
51+
accessKeyId: env.s3.credentials.accessKeyId,
52+
secretAccessKey: env.s3.credentials.secretAccessKey,
53+
sessionToken: env.s3.credentials.sessionToken,
54+
service: 's3',
55+
}),
56+
bucket: env.s3.bucketName,
57+
endpoint: env.s3.endpoint,
58+
},
59+
];
60+
61+
if (env.s3Mirror) {
62+
s3Config.push({
63+
client: new AwsClient({
64+
accessKeyId: env.s3Mirror.credentials.accessKeyId,
65+
secretAccessKey: env.s3Mirror.credentials.secretAccessKey,
66+
sessionToken: env.s3Mirror.credentials.sessionToken,
67+
service: 's3',
68+
}),
69+
bucket: env.s3Mirror.bucketName,
70+
endpoint: env.s3Mirror.endpoint,
71+
});
72+
}
4973

5074
const logger = baseLogger.child({
5175
source: 'PersistedDocumentsWorker',

0 commit comments

Comments
 (0)