Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
abed40c
Create entry.json
drernie Mar 25, 2025
aa61d9c
feat: add step to extract external file IDs from entry data
drernie Mar 25, 2025
4629f51
add test events
drernie Mar 25, 2025
0555013
feat: add task to fetch external file download URLs
drernie Mar 25, 2025
530b75b
fix: update external file response parsing in state machine
drernie Mar 25, 2025
45998ea
feat: add external file download and storage functionality
drernie Mar 25, 2025
55c1d7c
feat: add external files processing task to state machine
drernie Mar 25, 2025
2ccb0f9
fix: test using mock
drernie Mar 25, 2025
e13b4c5
entry with external file
drernie Mar 25, 2025
fb25cc8
refactor: update Map construct to use non-deprecated APIs
drernie Mar 25, 2025
72f2264
only download first file
drernie Mar 25, 2025
afb9ea0
itemProcessor
drernie Mar 25, 2025
5c2d349
fix ResultSelector
drernie Mar 26, 2025
935cf6e
refactor: write external files directly to S3 in state machine
drernie Mar 26, 2025
2473147
refactor: split URL parsing into separate state machine steps
drernie Mar 26, 2025
2a91c31
fix: split path extraction into two steps to handle array indices pro…
drernie Mar 26, 2025
2ca388b
CallAwsService to WriteExternalFileToS3
drernie Mar 26, 2025
6dfd72b
$$.Execution.Input.var.packageName
drernie Mar 26, 2025
3d67e26
revert process-export
drernie Mar 26, 2025
cf84696
sansQuery
drernie Mar 26, 2025
63ec5fc
feat: add debug state to verify S3 key construction
drernie Mar 26, 2025
1ff26b1
remove late-stage vars
drernie Mar 27, 2025
a594304
"$$.var.packageName
drernie Mar 27, 2025
13784cd
Merge branch 'main' into 45-external_file
drernie Mar 30, 2025
60a8f2e
Execution.Input.message.resourceId
drernie Mar 30, 2025
ad23ad6
refactor fetchURL
drernie Mar 30, 2025
03ba4a6
remove explicit external files (all in zip)
drernie Mar 30, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions lib/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,10 @@ export const MIME_TYPES = {
GIF: "image/gif",
TXT: "text/plain",
PDF: "application/pdf",
MD: "text/markdown",
ZIP: "application/zip",
YAML: "application/yaml",
YML: "application/yaml",
DEFAULT: "application/octet-stream",
} as const;

129 changes: 84 additions & 45 deletions lib/state-machine.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@ import * as events from "aws-cdk-lib/aws-events";
import * as iam from "aws-cdk-lib/aws-iam";
import { Construct } from "constructs";

import { StateMachineProps, ExportStatus } from "./types";
import { ExportStatus, StateMachineProps } from "./types";
import { EXPORT_STATUS, FILES } from "./constants";
import { README_TEMPLATE } from "./templates/readme";

export class WebhookStateMachine extends Construct {
public readonly stateMachine: stepfunctions.StateMachine;
private readonly bucket: s3.IBucket;

constructor(scope: Construct, id: string, props: StateMachineProps) {
super(scope, id);
this.bucket = props.bucket;
const definition = this.createDefinition(props);

const role = new iam.Role(scope, "StateMachineRole", {
Expand Down Expand Up @@ -92,17 +94,14 @@ export class WebhookStateMachine extends Construct {
props.benchlingConnection,
);
const writeEntryToS3Task = this.createS3WriteTask(
props.bucket,
FILES.ENTRY_JSON,
"$.entry.entryData",
);
const writeReadmeToS3Task = this.createS3WriteTask(
props.bucket,
FILES.README_MD,
"$.var.readme",
);
const writeMetadataTask = this.createS3WriteTask(
props.bucket,
FILES.INPUT_JSON,
"$.message",
);
Expand All @@ -123,43 +122,71 @@ export class WebhookStateMachine extends Construct {

// Create export polling loop
const exportTask = this.createExportTask(props.benchlingConnection);
const pollExportTask = this.createPollExportTask(props.benchlingConnection);
const pollExportTask = this.createPollExportTask(
props.benchlingConnection,
);
const waitState = this.createWaitState();

exportTask.addCatch(errorHandler);
pollExportTask.addCatch(errorHandler);

// Create export polling loop with proper state transitions
const extractDownloadURL = new stepfunctions.Pass(this, "ExtractDownloadURL", {
parameters: {
"status.$": "$.exportStatus.status" as ExportStatus["status"],
"downloadURL.$": "$.exportStatus.response.response.downloadURL",
"packageName.$": "$.var.packageName",
"registry.$": "$.var.registry",
const extractDownloadURL = new stepfunctions.Pass(
this,
"ExtractDownloadURL",
{
parameters: {
"status.$":
"$.exportStatus.status" as ExportStatus["status"],
"downloadURL.$":
"$.exportStatus.response.response.downloadURL",
"packageName.$": "$.var.packageName",
"registry.$": "$.var.registry",
},
resultPath: "$.exportStatus",
},
resultPath: "$.exportStatus",
});
);

const processExportTask = new tasks.LambdaInvoke(this, "ProcessExport", {
lambdaFunction: props.exportProcessor,
payload: stepfunctions.TaskInput.fromObject({
downloadURL: stepfunctions.JsonPath.stringAt("$.exportStatus.downloadURL"),
packageName: stepfunctions.JsonPath.stringAt("$.exportStatus.packageName"),
registry: stepfunctions.JsonPath.stringAt("$.exportStatus.registry"),
}),
resultPath: "$.processResult",
});
const processExportTask = new tasks.LambdaInvoke(
this,
"ProcessExport",
{
lambdaFunction: props.exportProcessor,
payload: stepfunctions.TaskInput.fromObject({
downloadURL: stepfunctions.JsonPath.stringAt(
"$.exportStatus.downloadURL",
),
packageName: stepfunctions.JsonPath.stringAt(
"$.exportStatus.packageName",
),
registry: stepfunctions.JsonPath.stringAt(
"$.exportStatus.registry",
),
}),
resultPath: "$.processResult",
},
);

const exportChoice = new stepfunctions.Choice(this, "CheckExportStatus")
.when(stepfunctions.Condition.stringEquals("$.exportStatus.status", EXPORT_STATUS.RUNNING),
waitState.next(pollExportTask))
.when(stepfunctions.Condition.stringEquals("$.exportStatus.status", EXPORT_STATUS.SUCCEEDED),
.when(
stepfunctions.Condition.stringEquals(
"$.exportStatus.status",
EXPORT_STATUS.RUNNING,
),
waitState.next(pollExportTask),
)
.when(
stepfunctions.Condition.stringEquals(
"$.exportStatus.status",
EXPORT_STATUS.SUCCEEDED,
),
extractDownloadURL
.next(processExportTask)
.next(writeEntryToS3Task)
.next(writeReadmeToS3Task)
.next(writeMetadataTask)
.next(sendToSQSTask))
.next(sendToSQSTask),
)
.otherwise(
new stepfunctions.Fail(this, "ExportFailed", {
cause: "Export task did not succeed",
Expand All @@ -168,7 +195,9 @@ export class WebhookStateMachine extends Construct {
);

// Create channel choice state
const createCanvasTask = this.createCanvasTask(props.benchlingConnection);
const createCanvasTask = this.createCanvasTask(
props.benchlingConnection,
);

const channelChoice = new stepfunctions.Choice(this, "CheckChannel")
.when(
Expand All @@ -177,22 +206,31 @@ export class WebhookStateMachine extends Construct {
.next(fetchEntryTask)
.next(exportTask)
.next(pollExportTask)
.next(exportChoice)
.next(exportChoice),
)
.when(
stepfunctions.Condition.or(
stepfunctions.Condition.stringEquals("$.message.type", "v2.app.activateRequested"),
stepfunctions.Condition.stringEquals("$.message.type", "v2-beta.canvas.created"),
stepfunctions.Condition.stringEquals("$.message.type", "v2.canvas.initialized")
stepfunctions.Condition.stringEquals(
"$.message.type",
"v2.app.activateRequested",
),
stepfunctions.Condition.stringEquals(
"$.message.type",
"v2-beta.canvas.created",
),
stepfunctions.Condition.stringEquals(
"$.message.type",
"v2.canvas.initialized",
),
),
createCanvasTask
createCanvasTask,
)
.otherwise(
new stepfunctions.Pass(this, "EchoInput", {
parameters: {
"input.$": "$",
},
})
}),
);

// Main workflow
Expand All @@ -208,7 +246,8 @@ export class WebhookStateMachine extends Construct {
Type: "Task",
Resource: "arn:aws:states:::http:invoke",
Parameters: {
"ApiEndpoint.$": "States.Format('{}/api/v2/exports', $.var.baseURL)",
"ApiEndpoint.$":
"States.Format('{}/api/v2/exports', $.var.baseURL)",
Method: "POST",
Authentication: {
ConnectionArn: benchlingConnection.attrArn,
Expand All @@ -233,7 +272,8 @@ export class WebhookStateMachine extends Construct {
Type: "Task",
Resource: "arn:aws:states:::http:invoke",
Parameters: {
"ApiEndpoint.$": "States.Format('{}/api/v2/tasks/{}', $.var.baseURL, $.exportTask.taskId)",
"ApiEndpoint.$":
"States.Format('{}/api/v2/tasks/{}', $.var.baseURL, $.exportTask.taskId)",
Method: "GET",
Authentication: {
ConnectionArn: benchlingConnection.attrArn,
Expand Down Expand Up @@ -263,7 +303,7 @@ export class WebhookStateMachine extends Construct {
Type: "Task",
Resource: "arn:aws:states:::http:invoke",
Parameters: {
"ApiEndpoint.$":
"ApiEndpoint.$":
"States.Format('{}/api/v2/app-canvases/{}', $.var.baseURL, $.message.canvasId)",
Method: "PATCH",
Authentication: {
Expand All @@ -275,17 +315,17 @@ export class WebhookStateMachine extends Construct {
"enabled": true,
"id": "user_defined_id",
"text": "Click me to submit",
"type": "BUTTON"
}
"type": "BUTTON",
},
],
"enabled": true,
"featureId": "quilt_integration"
}
"featureId": "quilt_integration",
},
},
ResultSelector: {
"canvasId.$": "$.ResponseBody.id"
"canvasId.$": "$.ResponseBody.id",
},
ResultPath: "$.canvas"
ResultPath: "$.canvas",
},
});
}
Expand Down Expand Up @@ -314,7 +354,6 @@ export class WebhookStateMachine extends Construct {
}

private createS3WriteTask(
bucket: s3.IBucket,
filename: string,
bodyPath: string,
): tasks.CallAwsService {
Expand All @@ -329,12 +368,12 @@ export class WebhookStateMachine extends Construct {
service: "s3",
action: "putObject",
parameters: {
Bucket: bucket.bucketName,
Bucket: this.bucket.bucketName,
"Key.$":
`States.Format('{}/{}', $.var.packageName, '${filename}')`,
"Body.$": bodyPath,
},
iamResources: [bucket.arnForObjects("*")],
iamResources: [this.bucket.arnForObjects("*")],
resultPath: resultPath,
});
}
Expand Down
16 changes: 2 additions & 14 deletions test/entry-updated.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
"createdAt": "2025-03-14T14:15:25.035271+00:00",
"deprecated": false,
"id": "evt_wXhBvbr8wqdx",
"resourceId": "etr_rv7MuD30",
"resourceId": "etr_0uKCQZ8f",
"schema": {
"id": "ts_sWLj9kNl"
},
Expand All @@ -22,17 +22,5 @@
]
},
"tenantId": "ten_dsrealahhb",
"version": "0",
"var": {
"baseURL": "https://quilt-dtt.benchling.com",
"registry": "quilt-bake",
"typeFields": [
"v2",
"entry",
"updated",
"fields"
],
"packageName": "benchling/etr_rv7MuD30",
"entity": "etr_rv7MuD30"
}
"version": "0"
}
Loading