Skip to content

Commit fbb78cb

Browse files
parthlambdatestparthlambdatest
committed
initial commit for upload to s3 via lsrs
1 parent 3535c8a commit fbb78cb

File tree

6 files changed

+90
-13
lines changed

6 files changed

+90
-13
lines changed

‎src/lib/ctx.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ export default (options: Record<string, string>): Context => {
2424
let fetchResultsFileObj: string;
2525
let buildNameObj: string;
2626
let allowDuplicateSnapshotNames: boolean = false;
27+
let useLambdaInternal: boolean = false;
2728
try {
2829
if (options.config) {
2930
config = JSON.parse(fs.readFileSync(options.config, 'utf-8'));
@@ -96,6 +97,9 @@ export default (options: Record<string, string>): Context => {
9697
if (config.allowDuplicateSnapshotNames) {
9798
allowDuplicateSnapshotNames = true;
9899
}
100+
if (config.useLambdaInternal) {
101+
useLambdaInternal = true;
102+
}
99103

100104
return {
101105
env: env,
@@ -122,6 +126,7 @@ export default (options: Record<string, string>): Context => {
122126
userAgent: config.userAgent || '',
123127
requestHeaders: config.requestHeaders || {},
124128
allowDuplicateSnapshotNames: allowDuplicateSnapshotNames,
129+
useLambdaInternal: useLambdaInternal,
125130
},
126131
uploadFilePath: '',
127132
webStaticConfig: [],

‎src/lib/httpClient.ts‎

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,10 @@ export default class httpClient {
9292

9393
async request(config: AxiosRequestConfig, log: Logger): Promise<Record<string, any>> {
9494
log.debug(`http request: ${config.method} ${config.url}`);
95-
if (config && config.data && !config.data.name && !config.data.snapshot) {
95+
if (config && config.data && !config.data.skipLogging && !config.data.name && !config.data.snapshot) {
9696
log.debug(config.data);
9797
}
98-
if (config && config.data && config.data.snapshotUuid) {
98+
if (config && config.data && !config.data.skipLogging && config.data.snapshotUuid) {
9999
log.debug(config.data);
100100
}
101101
return this.axiosInstance.request(config)
@@ -495,6 +495,38 @@ export default class httpClient {
495495
}, ctx.log)
496496
}
497497

498+
sendDomToLSRS(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string) {
499+
return this.request({
500+
url: `/upload/dom`,
501+
method: 'POST',
502+
data: {
503+
buildId: ctx.build.id,
504+
snapshotName: snapshot.name,
505+
snapshotUuid: snapshotUuid,
506+
domContent: snapshot,
507+
skipLogging: true
508+
}
509+
}, ctx.log);
510+
}
511+
512+
sendDomToLSRSForCaps(ctx: Context, snapshot: ProcessedSnapshot, snapshotUuid: string, capsBuildId: string, capsProjectToken: string) {
513+
return this.request({
514+
url: `/upload/dom`,
515+
method: 'POST',
516+
headers: {
517+
'Content-Type': 'application/json',
518+
projectToken: capsProjectToken !== '' ? capsProjectToken : this.projectToken
519+
},
520+
data: {
521+
buildId: capsBuildId,
522+
snapshotName: snapshot.name,
523+
snapshotUuid: snapshotUuid,
524+
domContent: snapshot,
525+
skipLogging: true
526+
}
527+
}, ctx.log);
528+
}
529+
498530
uploadLogs(ctx: Context, uploadURL: string) {
499531
const fileStream = fs.createReadStream(constants.LOG_FILE_PATH);
500532
const { size } = fs.statSync(constants.LOG_FILE_PATH);
@@ -512,6 +544,20 @@ export default class httpClient {
512544
}, ctx.log)
513545
}
514546

547+
sendCliLogsToLSRS(ctx: Context) {
548+
const logContent = fs.readFileSync(constants.LOG_FILE_PATH, 'utf-8');
549+
550+
return this.request({
551+
url: `/upload/logs`,
552+
method: 'POST',
553+
data: {
554+
buildId: ctx.build.id,
555+
logContent: logContent,
556+
skipLogging: true
557+
}
558+
}, ctx.log);
559+
}
560+
515561
uploadSnapshotToS3(ctx: Context, uploadURL: string, snapshot: Snapshot) {
516562
return this.request({
517563
url: uploadURL,

‎src/lib/schemaValidation.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,10 @@ const ConfigSchema = {
262262
allowDuplicateSnapshotNames: {
263263
type: "boolean",
264264
errorMessage: "Invalid config; allowDuplicateSnapshotNames must be true/false"
265+
},
266+
useLambdaInternal: {
267+
type: "boolean",
268+
errorMessage: "Invalid config; useLambdaInternal must be true/false"
265269
}
266270
},
267271
anyOf: [

‎src/lib/snapshotQueue.ts‎

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import processSnapshot, {prepareSnapshot} from "./processSnapshot.js"
55
import { v4 as uuidv4 } from 'uuid';
66
import { startPolling, stopTunnelHelper } from "./utils.js";
77

8+
const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
89
export default class Queue {
910
private snapshots: Array<Snapshot> = [];
1011
private processedSnapshots: Array<Record<string, any>> = [];
@@ -333,10 +334,16 @@ export default class Queue {
333334
if (useCapsBuildId) {
334335
if (useKafkaFlowCaps) {
335336
const snapshotUuid = uuidv4();
336-
const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken);
337-
const uploadUrl = presignedResponse.data.url;
338-
339-
await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken)
337+
let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
338+
if (!uploadDomToS3) {
339+
this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL for CAPS`);
340+
const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUploadCaps(this.ctx, processedSnapshot.name, snapshotUuid, capsBuildId, capsProjectToken);
341+
const uploadUrl = presignedResponse.data.url;
342+
await this.ctx.client.uploadSnapshotToS3Caps(this.ctx, uploadUrl, processedSnapshot, capsProjectToken)
343+
} else {
344+
this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`);
345+
await this.ctx.client.sendDomToLSRSForCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken);
346+
}
340347
await this.ctx.client.processSnapshotCaps(this.ctx, processedSnapshot, snapshotUuid, capsBuildId, capsProjectToken, discoveryErrors);
341348
} else {
342349
await this.ctx.client.uploadSnapshotForCaps(this.ctx, processedSnapshot, capsBuildId, capsProjectToken, discoveryErrors);
@@ -372,10 +379,17 @@ export default class Queue {
372379
}
373380
if (this.ctx.build && this.ctx.build.useKafkaFlow) {
374381
const snapshotUuid = uuidv4();
375-
const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid);
376-
const uploadUrl = presignedResponse.data.url;
377-
378-
let snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot);
382+
let snapshotUploadResponse
383+
let uploadDomToS3 = this.ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
384+
if (!uploadDomToS3) {
385+
this.ctx.log.debug(`Uploading dom to S3 for snapshot using presigned URL`);
386+
const presignedResponse = await this.ctx.client.getS3PresignedURLForSnapshotUpload(this.ctx, processedSnapshot.name, snapshotUuid);
387+
const uploadUrl = presignedResponse.data.url;
388+
snapshotUploadResponse = await this.ctx.client.uploadSnapshotToS3(this.ctx, uploadUrl, processedSnapshot);
389+
} else {
390+
this.ctx.log.debug(`Uploading dom to S3 for snapshot using LSRS`);
391+
snapshotUploadResponse = await this.ctx.client.sendDomToLSRS(this.ctx, processedSnapshot, snapshotUuid);
392+
}
379393
if (!snapshotUploadResponse || Object.keys(snapshotUploadResponse).length === 0) {
380394
this.ctx.log.debug(`snapshot failed; Unable to upload dom to S3`);
381395
this.processedSnapshots.push({ name: snapshot?.name, error: `snapshot failed; Unable to upload dom to S3` });

‎src/tasks/finalizeBuild.ts‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { unlinkSync } from 'fs';
77
import constants from '../lib/constants.js';
88
import fs from 'fs';
99

10+
const uploadDomToS3ViaEnv = process.env.USE_LAMBDA_INTERNAL || false;
1011
export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRendererFactory> => {
1112
return {
1213
title: `Finalizing build`,
@@ -76,9 +77,15 @@ export default (ctx: Context): ListrTask<Context, ListrRendererFactory, ListrRen
7677
await ctx.server?.close();
7778
ctx.log.debug(`Closed server`);
7879
if (ctx.isSnapshotCaptured) {
79-
ctx.log.debug(`Log file to be uploaded`)
80-
let resp = await ctx.client.getS3PreSignedURL(ctx);
81-
await ctx.client.uploadLogs(ctx, resp.data.url);
80+
let uploadCLILogsToS3 = ctx.config.useLambdaInternal || uploadDomToS3ViaEnv;
81+
if (!uploadCLILogsToS3) {
82+
ctx.log.debug(`Log file to be uploaded`)
83+
let resp = await ctx.client.getS3PreSignedURL(ctx);
84+
await ctx.client.uploadLogs(ctx, resp.data.url);
85+
} else {
86+
ctx.log.debug(`Log file to be uploaded via LSRS`)
87+
let resp = ctx.client.sendCliLogsToLSRS(ctx);
88+
}
8289
}
8390
} catch (error: any) {
8491
ctx.log.debug(error);

‎src/types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ export interface Context {
3737
userAgent?: string;
3838
requestHeaders?: Array<Record<string, string>>;
3939
allowDuplicateSnapshotNames?: boolean;
40+
useLambdaInternal?: boolean;
4041
};
4142
uploadFilePath: string;
4243
webStaticConfig: WebStaticConfig;

0 commit comments

Comments
 (0)