Skip to content
This repository was archived by the owner on Aug 9, 2023. It is now read-only.

Commit 655f3b4

Browse files
itzhapazitzhapaz
committed
genomics pipeline using CDK
1 parent a614fed commit 655f3b4

34 files changed

+9409
-0
lines changed

‎src/aws-genomics-cdk/.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
*.js
2+
!jest.config.js
3+
*.d.ts
4+
node_modules
5+
6+
# CDK asset staging directory
7+
.cdk.staging
8+
cdk.out

‎src/aws-genomics-cdk/.npmignore‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
*.ts
2+
!*.d.ts
3+
4+
# CDK asset staging directory
5+
.cdk.staging
6+
cdk.out

‎src/aws-genomics-cdk/README.md‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
# Welcome to your CDK TypeScript project!
2+
3+
This is a blank project for TypeScript development with CDK.
4+
5+
The `cdk.json` file tells the CDK Toolkit how to execute your app.
6+
7+
## Useful commands
8+
9+
* `npm run build` compile typescript to js
10+
* `npm run watch` watch for changes and compile
11+
* `npm run test` perform the jest unit tests
12+
* `cdk deploy` deploy this stack to your default AWS account/region
13+
* `cdk diff` compare deployed stack with current state
14+
* `cdk synth` emits the synthesized CloudFormation template

‎src/aws-genomics-cdk/app.config.json‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
{
2+
"accountID": "111111111111",
3+
"region": "us-west-2",
4+
"S3": {
5+
"existingBucket": true,
6+
"bucketName": ""
7+
},
8+
"VPC": {
9+
"createVPC": true,
10+
"existingVPCName": "",
11+
"maxAZs": 2,
12+
"cidr": "10.0.0.0/16",
13+
"cidrMask": 24
14+
},
15+
"batch": {
16+
"defaultVolumeSize": 100,
17+
"spotMaxVCPUs": 128,
18+
"onDemendMaxVCPUs": 128,
19+
"instanceTypes": [
20+
"c4.large",
21+
"c4.xlarge",
22+
"c4.2xlarge",
23+
"c4.4xlarge",
24+
"c4.8xlarge",
25+
"c5.large",
26+
"c5.xlarge",
27+
"c5.2xlarge",
28+
"c5.4xlarge",
29+
"c5.9xlarge",
30+
"c5.12xlarge",
31+
"c5.18xlarge",
32+
"c5.24xlarge"
33+
]
34+
},
35+
"stepFunctions": {
36+
"launchDemoPipeline": true,
37+
"jobDefinitions": {
38+
"fastqc": {
39+
"repository": "genomics/fastqc",
40+
"memoryLimit": 8000,
41+
"vcpus": 4,
42+
"spot": true,
43+
"retryAttempts":1,
44+
"timeout": 600
45+
},
46+
"minimap2": {
47+
"repository": "genomics/minimap2",
48+
"memoryLimit": 16000,
49+
"vcpus": 8,
50+
"spot": true,
51+
"retryAttempts":1,
52+
"timeout": 3600
53+
}
54+
}
55+
}
56+
}

‎src/aws-genomics-cdk/assets/genomics-policy-s3.json‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
{
2+
"Version": "2012-10-17",
3+
"Statement": [
4+
{
5+
"Effect": "Deny",
6+
"Action": [
7+
"s3:Delete*",
8+
"s3:PutBucket*"
9+
],
10+
"Resource": [
11+
"arn:aws:s3:::BUCKET_NAME"
12+
]
13+
},
14+
{
15+
"Effect": "Allow",
16+
"Action": [
17+
"s3:ListBucket*"
18+
],
19+
"Resource": [
20+
"arn:aws:s3:::BUCKET_NAME"
21+
]
22+
},
23+
{
24+
"Effect": "Allow",
25+
"Action": [
26+
"s3:*"
27+
],
28+
"Resource": [
29+
"arn:aws:s3:::BUCKET_NAME/*"
30+
]
31+
}
32+
]
33+
}

‎src/aws-genomics-cdk/assets/launch_template_user_data.txt‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
MIME-Version: 1.0
2+
Content-Type: multipart/mixed; boundary="==BOUNDARY=="
3+
4+
--==BOUNDARY==
5+
Content-Type: text/cloud-config; charset="us-ascii"
6+
7+
packages:
8+
- jq
9+
- btrfs-progs
10+
- sed
11+
- git
12+
- amazon-ssm-agent
13+
- unzip
14+
15+
runcmd:
16+
# install aws-cli v2 and copy the static binary in an easy to find location for bind-mounts into containers
17+
- curl -s "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "/tmp/awscliv2.zip"
18+
- unzip -q /tmp/awscliv2.zip -d /tmp
19+
- /tmp/aws/install -b /usr/bin
20+
21+
# check that the aws-cli was actually installed. if not shutdown (terminate) the instance
22+
- command -v aws || shutdown -P now
23+
24+
- mkdir -p /opt/aws-cli/bin
25+
- cp -a $(dirname $(find /usr/local/aws-cli -name 'aws' -type f))/. /opt/aws-cli/bin/
26+
27+
28+
# enable ecs spot instance draining
29+
- echo ECS_ENABLE_SPOT_INSTANCE_DRAINING=true >> /etc/ecs/ecs.config
30+
31+
- systemctl enable amazon-ssm-agent
32+
- systemctl start amazon-ssm-agent
33+
34+
--==BOUNDARY==--

‎src/aws-genomics-cdk/bin/aws-genomics-cdk.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env node
2+
import 'source-map-support/register';
3+
import * as cdk from '@aws-cdk/core';
4+
import { AwsGenomicsCdkStack } from '../lib/aws-genomics-cdk-stack';
5+
import * as config from "../app.config.json";
6+
7+
const env = {
8+
account: process.env.CDK_DEFAULT_ACCOUNT ?? config.accountID,
9+
region: process.env.CDK_DEFAULT_REGION ?? config.region
10+
}
11+
12+
const app = new cdk.App();
13+
new AwsGenomicsCdkStack(app, 'AwsGenomicsCdkStack', {env: env});

‎src/aws-genomics-cdk/cdk.context.json‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
{
2+
"availability-zones:account=549173009724:region=us-west-2": [
3+
"us-west-2a",
4+
"us-west-2b",
5+
"us-west-2c",
6+
"us-west-2d"
7+
],
8+
"vpc-provider:account=549173009724:filter.tag:Name=GenomicsPipelineStack/genomics-vpc-stack/genomics-vpc:region=us-west-2:returnAsymmetricSubnets=true": {
9+
"vpcId": "vpc-0e12b34b410ce432d",
10+
"vpcCidrBlock": "10.0.0.0/16",
11+
"availabilityZones": [],
12+
"subnetGroups": [
13+
{
14+
"name": "public",
15+
"type": "Public",
16+
"subnets": [
17+
{
18+
"subnetId": "subnet-01f8be5070f8f0c41",
19+
"cidr": "10.0.3.0/24",
20+
"availabilityZone": "us-west-2a",
21+
"routeTableId": "rtb-0db455d5a3da84065"
22+
},
23+
{
24+
"subnetId": "subnet-0c0e2287b8f96dff9",
25+
"cidr": "10.0.4.0/24",
26+
"availabilityZone": "us-west-2b",
27+
"routeTableId": "rtb-0c5cd45dbc8e33ef7"
28+
},
29+
{
30+
"subnetId": "subnet-0c2c15f728465a4c2",
31+
"cidr": "10.0.5.0/24",
32+
"availabilityZone": "us-west-2c",
33+
"routeTableId": "rtb-0c2e59776f577a3b7"
34+
}
35+
]
36+
},
37+
{
38+
"name": "private",
39+
"type": "Private",
40+
"subnets": [
41+
{
42+
"subnetId": "subnet-0d82fc3969299134f",
43+
"cidr": "10.0.0.0/24",
44+
"availabilityZone": "us-west-2a",
45+
"routeTableId": "rtb-0d6ddf764ea60615b"
46+
},
47+
{
48+
"subnetId": "subnet-095bc96659c1f7af4",
49+
"cidr": "10.0.1.0/24",
50+
"availabilityZone": "us-west-2b",
51+
"routeTableId": "rtb-0de1df34a81474cf1"
52+
},
53+
{
54+
"subnetId": "subnet-013b7a044af9c6542",
55+
"cidr": "10.0.2.0/24",
56+
"availabilityZone": "us-west-2c",
57+
"routeTableId": "rtb-0541671d21b141dc3"
58+
}
59+
]
60+
}
61+
]
62+
}
63+
}

‎src/aws-genomics-cdk/cdk.json‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
{
2+
"app": "npx ts-node --prefer-ts-exts bin/aws-genomics-cdk.ts",
3+
"context": {
4+
"@aws-cdk/core:enableStackNameDuplicates": "true",
5+
"aws-cdk:enableDiffNoFail": "true",
6+
"@aws-cdk/core:stackRelativeExports": "true",
7+
"@aws-cdk/aws-ecr-assets:dockerIgnoreSupport": true,
8+
"@aws-cdk/aws-secretsmanager:parseOwnedSecretName": true,
9+
"@aws-cdk/aws-kms:defaultKeyPolicies": true,
10+
"@aws-cdk/aws-s3:grantWriteWithoutAcl": true,
11+
"@aws-cdk/aws-ecs-patterns:removeDefaultDesiredCount": true
12+
}
13+
}

0 commit comments

Comments
 (0)