Skip to content

Commit 85651c7

Browse files
drerniedrernie
authored
45 external file (#48)
* Create entry.json * feat: add step to extract external file IDs from entry data * add test events * feat: add task to fetch external file download URLs * fix: update external file response parsing in state machine * feat: add external file download and storage functionality * feat: add external files processing task to state machine * fix: test using mock * entry with external file * refactor: update Map construct to use non-deprecated APIs * only download first file * itemProcessor * fix ResultSelector * refactor: write external files directly to S3 in state machine * refactor: split URL parsing into separate state machine steps * fix: split path extraction into two steps to handle array indices properly * CallAwsService to WriteExternalFileToS3 * $$.Execution.Input.var.packageName * revert process-export * sansQuery * feat: add debug state to verify S3 key construction * remove late-stage vars * "$$.var.packageName * Execution.Input.message.resourceId * refactor fetchURL * remove explicit external files (all in zip) --------- Co-authored-by: Dr. Ernie Prabhakar <19791+drernie@users.noreply.github.com>
1 parent c382aaa commit 85651c7

File tree

4 files changed

+277
-59
lines changed

4 files changed

+277
-59
lines changed

‎lib/constants.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ export const MIME_TYPES = {
2020
GIF: "image/gif",
2121
TXT: "text/plain",
2222
PDF: "application/pdf",
23+
MD: "text/markdown",
24+
ZIP: "application/zip",
25+
YAML: "application/yaml",
26+
YML: "application/yaml",
2327
DEFAULT: "application/octet-stream",
2428
} as const;
2529

‎lib/state-machine.ts‎

Lines changed: 84 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,17 @@ import * as events from "aws-cdk-lib/aws-events";
77
import * as iam from "aws-cdk-lib/aws-iam";
88
import { Construct } from "constructs";
99

10-
import { StateMachineProps, ExportStatus } from "./types";
10+
import { ExportStatus, StateMachineProps } from "./types";
1111
import { EXPORT_STATUS, FILES } from "./constants";
1212
import { README_TEMPLATE } from "./templates/readme";
1313

1414
export class WebhookStateMachine extends Construct {
1515
public readonly stateMachine: stepfunctions.StateMachine;
16+
private readonly bucket: s3.IBucket;
1617

1718
constructor(scope: Construct, id: string, props: StateMachineProps) {
1819
super(scope, id);
20+
this.bucket = props.bucket;
1921
const definition = this.createDefinition(props);
2022

2123
const role = new iam.Role(scope, "StateMachineRole", {
@@ -92,17 +94,14 @@ export class WebhookStateMachine extends Construct {
9294
props.benchlingConnection,
9395
);
9496
const writeEntryToS3Task = this.createS3WriteTask(
95-
props.bucket,
9697
FILES.ENTRY_JSON,
9798
"$.entry.entryData",
9899
);
99100
const writeReadmeToS3Task = this.createS3WriteTask(
100-
props.bucket,
101101
FILES.README_MD,
102102
"$.var.readme",
103103
);
104104
const writeMetadataTask = this.createS3WriteTask(
105-
props.bucket,
106105
FILES.INPUT_JSON,
107106
"$.message",
108107
);
@@ -123,43 +122,71 @@ export class WebhookStateMachine extends Construct {
123122

124123
// Create export polling loop
125124
const exportTask = this.createExportTask(props.benchlingConnection);
126-
const pollExportTask = this.createPollExportTask(props.benchlingConnection);
125+
const pollExportTask = this.createPollExportTask(
126+
props.benchlingConnection,
127+
);
127128
const waitState = this.createWaitState();
128129

129130
exportTask.addCatch(errorHandler);
130131
pollExportTask.addCatch(errorHandler);
131132

132133
// Create export polling loop with proper state transitions
133-
const extractDownloadURL = new stepfunctions.Pass(this, "ExtractDownloadURL", {
134-
parameters: {
135-
"status.$": "$.exportStatus.status" as ExportStatus["status"],
136-
"downloadURL.$": "$.exportStatus.response.response.downloadURL",
137-
"packageName.$": "$.var.packageName",
138-
"registry.$": "$.var.registry",
134+
const extractDownloadURL = new stepfunctions.Pass(
135+
this,
136+
"ExtractDownloadURL",
137+
{
138+
parameters: {
139+
"status.$":
140+
"$.exportStatus.status" as ExportStatus["status"],
141+
"downloadURL.$":
142+
"$.exportStatus.response.response.downloadURL",
143+
"packageName.$": "$.var.packageName",
144+
"registry.$": "$.var.registry",
145+
},
146+
resultPath: "$.exportStatus",
139147
},
140-
resultPath: "$.exportStatus",
141-
});
148+
);
142149

143-
const processExportTask = new tasks.LambdaInvoke(this, "ProcessExport", {
144-
lambdaFunction: props.exportProcessor,
145-
payload: stepfunctions.TaskInput.fromObject({
146-
downloadURL: stepfunctions.JsonPath.stringAt("$.exportStatus.downloadURL"),
147-
packageName: stepfunctions.JsonPath.stringAt("$.exportStatus.packageName"),
148-
registry: stepfunctions.JsonPath.stringAt("$.exportStatus.registry"),
149-
}),
150-
resultPath: "$.processResult",
151-
});
150+
const processExportTask = new tasks.LambdaInvoke(
151+
this,
152+
"ProcessExport",
153+
{
154+
lambdaFunction: props.exportProcessor,
155+
payload: stepfunctions.TaskInput.fromObject({
156+
downloadURL: stepfunctions.JsonPath.stringAt(
157+
"$.exportStatus.downloadURL",
158+
),
159+
packageName: stepfunctions.JsonPath.stringAt(
160+
"$.exportStatus.packageName",
161+
),
162+
registry: stepfunctions.JsonPath.stringAt(
163+
"$.exportStatus.registry",
164+
),
165+
}),
166+
resultPath: "$.processResult",
167+
},
168+
);
152169

153170
const exportChoice = new stepfunctions.Choice(this, "CheckExportStatus")
154-
.when(stepfunctions.Condition.stringEquals("$.exportStatus.status", EXPORT_STATUS.RUNNING),
155-
waitState.next(pollExportTask))
156-
.when(stepfunctions.Condition.stringEquals("$.exportStatus.status", EXPORT_STATUS.SUCCEEDED),
171+
.when(
172+
stepfunctions.Condition.stringEquals(
173+
"$.exportStatus.status",
174+
EXPORT_STATUS.RUNNING,
175+
),
176+
waitState.next(pollExportTask),
177+
)
178+
.when(
179+
stepfunctions.Condition.stringEquals(
180+
"$.exportStatus.status",
181+
EXPORT_STATUS.SUCCEEDED,
182+
),
157183
extractDownloadURL
158184
.next(processExportTask)
159185
.next(writeEntryToS3Task)
160186
.next(writeReadmeToS3Task)
161187
.next(writeMetadataTask)
162-
.next(sendToSQSTask))
188+
.next(sendToSQSTask),
189+
)
163190
.otherwise(
164191
new stepfunctions.Fail(this, "ExportFailed", {
165192
cause: "Export task did not succeed",
@@ -168,7 +195,9 @@ export class WebhookStateMachine extends Construct {
168195
);
169196

170197
// Create channel choice state
171-
const createCanvasTask = this.createCanvasTask(props.benchlingConnection);
198+
const createCanvasTask = this.createCanvasTask(
199+
props.benchlingConnection,
200+
);
172201

173202
const channelChoice = new stepfunctions.Choice(this, "CheckChannel")
174203
.when(
@@ -177,22 +206,31 @@ export class WebhookStateMachine extends Construct {
177206
.next(fetchEntryTask)
178207
.next(exportTask)
179208
.next(pollExportTask)
180-
.next(exportChoice)
209+
.next(exportChoice),
181210
)
182211
.when(
183212
stepfunctions.Condition.or(
184-
stepfunctions.Condition.stringEquals("$.message.type", "v2.app.activateRequested"),
185-
stepfunctions.Condition.stringEquals("$.message.type", "v2-beta.canvas.created"),
186-
stepfunctions.Condition.stringEquals("$.message.type", "v2.canvas.initialized")
213+
stepfunctions.Condition.stringEquals(
214+
"$.message.type",
215+
"v2.app.activateRequested",
216+
),
217+
stepfunctions.Condition.stringEquals(
218+
"$.message.type",
219+
"v2-beta.canvas.created",
220+
),
221+
stepfunctions.Condition.stringEquals(
222+
"$.message.type",
223+
"v2.canvas.initialized",
224+
),
187225
),
188-
createCanvasTask
226+
createCanvasTask,
189227
)
190228
.otherwise(
191229
new stepfunctions.Pass(this, "EchoInput", {
192230
parameters: {
193231
"input.$": "$",
194232
},
195-
})
233+
}),
196234
);
197235

198236
// Main workflow
@@ -208,7 +246,8 @@ export class WebhookStateMachine extends Construct {
208246
Type: "Task",
209247
Resource: "arn:aws:states:::http:invoke",
210248
Parameters: {
211-
"ApiEndpoint.$": "States.Format('{}/api/v2/exports', $.var.baseURL)",
249+
"ApiEndpoint.$":
250+
"States.Format('{}/api/v2/exports', $.var.baseURL)",
212251
Method: "POST",
213252
Authentication: {
214253
ConnectionArn: benchlingConnection.attrArn,
@@ -233,7 +272,8 @@ export class WebhookStateMachine extends Construct {
233272
Type: "Task",
234273
Resource: "arn:aws:states:::http:invoke",
235274
Parameters: {
236-
"ApiEndpoint.$": "States.Format('{}/api/v2/tasks/{}', $.var.baseURL, $.exportTask.taskId)",
275+
"ApiEndpoint.$":
276+
"States.Format('{}/api/v2/tasks/{}', $.var.baseURL, $.exportTask.taskId)",
237277
Method: "GET",
238278
Authentication: {
239279
ConnectionArn: benchlingConnection.attrArn,
@@ -263,7 +303,7 @@ export class WebhookStateMachine extends Construct {
263303
Type: "Task",
264304
Resource: "arn:aws:states:::http:invoke",
265305
Parameters: {
266-
"ApiEndpoint.$":
306+
"ApiEndpoint.$":
267307
"States.Format('{}/api/v2/app-canvases/{}', $.var.baseURL, $.message.canvasId)",
268308
Method: "PATCH",
269309
Authentication: {
@@ -275,17 +315,17 @@ export class WebhookStateMachine extends Construct {
275315
"enabled": true,
276316
"id": "user_defined_id",
277317
"text": "Click me to submit",
278-
"type": "BUTTON"
279-
}
318+
"type": "BUTTON",
319+
},
280320
],
281321
"enabled": true,
282-
"featureId": "quilt_integration"
283-
}
322+
"featureId": "quilt_integration",
323+
},
284324
},
285325
ResultSelector: {
286-
"canvasId.$": "$.ResponseBody.id"
326+
"canvasId.$": "$.ResponseBody.id",
287327
},
288-
ResultPath: "$.canvas"
328+
ResultPath: "$.canvas",
289329
},
290330
});
291331
}
@@ -314,7 +354,6 @@ export class WebhookStateMachine extends Construct {
314354
}
315355

316356
private createS3WriteTask(
317-
bucket: s3.IBucket,
318357
filename: string,
319358
bodyPath: string,
320359
): tasks.CallAwsService {
@@ -329,12 +368,12 @@ export class WebhookStateMachine extends Construct {
329368
service: "s3",
330369
action: "putObject",
331370
parameters: {
332-
Bucket: bucket.bucketName,
371+
Bucket: this.bucket.bucketName,
333372
"Key.$":
334373
`States.Format('{}/{}', $.var.packageName, '${filename}')`,
335374
"Body.$": bodyPath,
336375
},
337-
iamResources: [bucket.arnForObjects("*")],
376+
iamResources: [this.bucket.arnForObjects("*")],
338377
resultPath: resultPath,
339378
});
340379
}

‎test/entry-updated.json‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
"createdAt": "2025-03-14T14:15:25.035271+00:00",
1313
"deprecated": false,
1414
"id": "evt_wXhBvbr8wqdx",
15-
"resourceId": "etr_rv7MuD30",
15+
"resourceId": "etr_0uKCQZ8f",
1616
"schema": {
1717
"id": "ts_sWLj9kNl"
1818
},
@@ -22,17 +22,5 @@
2222
]
2323
},
2424
"tenantId": "ten_dsrealahhb",
25-
"version": "0",
26-
"var": {
27-
"baseURL": "https://quilt-dtt.benchling.com",
28-
"registry": "quilt-bake",
29-
"typeFields": [
30-
"v2",
31-
"entry",
32-
"updated",
33-
"fields"
34-
],
35-
"packageName": "benchling/etr_rv7MuD30",
36-
"entity": "etr_rv7MuD30"
37-
}
25+
"version": "0"
3826
}

0 commit comments

Comments
 (0)