examples cleanup

itzhapaz · itzhapaz · commit 182e361988b4 · 2021-04-08T17:30:16.000Z
diff --git a/src/aws-genomics-cdk/app.config.json b/src/aws-genomics-cdk/app.config.json
@@ -1,5 +1,5 @@
 {
-    "accountID": "840249831562",
+    "accountID": "111111111111",
     "region": "us-west-2",
     "projectName": "genomics",
     "tags": [{
@@ -13,7 +13,7 @@
     ],
     "S3": {
         "existingBucket": true,
-        "bucketName": "genomics-pipelines-840249831562"
+        "bucketName": "YOUR-BUCKET-NAME"
     },
     "VPC": {
         "createVPC": true,
diff --git a/src/aws-genomics-cdk/examples/batch-bwa-job.json b/src/aws-genomics-cdk/examples/batch-bwa-job.json
@@ -1,12 +1,12 @@
 {
     "jobName": "bwa",
     "jobQueue": "genomics-default-queue",
-    "jobDefinition": "bwa:3",
+    "jobDefinition": "bwa:1",
     "containerOverrides": {
         "command": ["bwa mem -t 8 -p -o ${SAMPLE_ID}.sam ${REFERENCE_NAME}.fasta ${SAMPLE_ID}_*1*.fastq.gz"],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/sample/ s3://genomics-pipelines-840249831562/hg38/v0/"
+                "value": "s3://aws-batch-genomics-shared/secondary-analysis/example-files/fastq/NIST7035* s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta*"
             },
             {
                 "name": "SAMPLE_ID",
@@ -22,11 +22,7 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
-            },
-            {
-                "name": "JOB_INPUT_S3_COPY_METHOD",
-                "value": "s3sync"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             }
         ]
     }
diff --git a/src/aws-genomics-cdk/examples/batch-fastqc-job.json b/src/aws-genomics-cdk/examples/batch-fastqc-job.json
@@ -1,7 +1,7 @@
 {
     "jobName": "fastqc",
     "jobQueue": "genomics-default-queue",
-    "jobDefinition": "fastqc:3",
+    "jobDefinition": "fastqc:1",
     "containerOverrides": {
         "command": ["fastqc *.gz"],
         "environment": [{
@@ -14,7 +14,7 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             }
         ]
     }
diff --git a/src/aws-genomics-cdk/examples/batch-gatk-dictionary.json b/src/aws-genomics-cdk/examples/batch-gatk-dictionary.json
@@ -6,15 +6,15 @@
         "command": ["java -jar /usr/app/picard.jar CreateSequenceDictionary R=Homo_sapiens_assembly38.fasta O=Homo_sapiens_assembly38.dict"],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/hg38/v0/Homo_sapiens_assembly38.fasta"
+                "value": "s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta"
             },
             {
                 "name": "JOB_OUTPUTS",
                 "value": "*.dict"
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/hg38/v0"
+                "value": "s3://YOUR-BUCKET-NAME/ref"
             }
         ]
     }
diff --git a/src/aws-genomics-cdk/examples/batch-gatk-htc.json b/src/aws-genomics-cdk/examples/batch-gatk-htc.json
@@ -6,7 +6,7 @@
         "command": ["gatk --java-options \"-Xmx4g\" HaplotypeCaller -R ${REFERENCE_NAME}.fasta -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.vcf.gz -bamout ${SAMPLE_ID}.out.bam"],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/output/ s3://genomics-pipelines-840249831562/hg38/v0/"
+                "value": "s3://YOUR-BUCKET-NAME/ref s3://YOUR-BUCKET-NAME/samples"
             },
             {
                 "name": "SAMPLE_ID",
@@ -22,7 +22,7 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             },
             {
                 "name": "JOB_INPUT_S3_COPY_METHOD",
diff --git a/src/aws-genomics-cdk/examples/batch-minimap2-job.json b/src/aws-genomics-cdk/examples/batch-minimap2-job.json
@@ -7,21 +7,17 @@
         "memory": 16000,
         "command": ["minimap2 -ax map-pb Homo_sapiens_assembly38.fasta NIST7035_R1_trim_samp-0p1.fastq.gz > NIST7035.sam"],
         "environment": [{
-            "name": "JOB_INPUTS",
-            "value": "s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta s3://aws-batch-genomics-shared/secondary-analysis/example-files/fastq/NIST7035_R1_trim_samp-0p1.fastq.gz"
-        },
-        {
-            "name": "JOB_OUTPUTS",
-            "value": "*.sam"
-        },
-        {
-            "name": "JOB_OUTPUT_PREFIX",
-            "value": "s3://[YOUR BUCKET NAME]/[SOME PREFIX]"
-        },
-        {
-            "name": "JOB_AWS_CLI_PATH",
-            "value": "/opt/aws-cli/bin"
-        }
+                "name": "JOB_INPUTS",
+                "value": "s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta s3://aws-batch-genomics-shared/secondary-analysis/example-files/fastq/NIST7035_R1_trim_samp-0p1.fastq.gz"
+            },
+            {
+                "name": "JOB_OUTPUTS",
+                "value": "*.sam"
+            },
+            {
+                "name": "JOB_OUTPUT_PREFIX",
+                "value": "s3://YOUR-BUCKET-NAME/output"
+            }
         ]
     }
-}
+}
diff --git a/src/aws-genomics-cdk/examples/batch-picard-add-missing-groups.json b/src/aws-genomics-cdk/examples/batch-picard-add-missing-groups.json
@@ -4,13 +4,13 @@
     "jobDefinition": "picardAddMissingGroups:1",
     "containerOverrides": {
         "command": ["java -jar /usr/picard/picard.jar AddOrReplaceReadGroups",
-        " -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.rg.bam -RGID 4 --RGLB lib1 ",
-        " --RGPL ILLUMINA --RGPU unit1 --RGSM 20;",
-        " cp ${SAMPLE_ID}.rg.bam ${SAMPLE_ID}.bam;",
-        " rm ${SAMPLE_ID}.rg.bam"],
+            " -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.rg.bam -RGID 4 --RGLB lib1 ",
+            " --RGPL ILLUMINA --RGPU unit1 --RGSM 20;",
+            " mv ${SAMPLE_ID}.rg.bam ${SAMPLE_ID}.bam;"
+        ],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/output/NIST7035.bam"
+                "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.bam"
             },
             {
                 "name": "SAMPLE_ID",
@@ -22,8 +22,8 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             }
         ]
     }
-}
+}
diff --git a/src/aws-genomics-cdk/examples/batch-samtools-index.json b/src/aws-genomics-cdk/examples/batch-samtools-index.json
@@ -1,12 +1,12 @@
 {
     "jobName": "samtoolsIndex",
     "jobQueue": "genomics-default-queue",
-    "jobDefinition": "samtoolsIndex:2",
+    "jobDefinition": "samtoolsIndex:1",
     "containerOverrides": {
         "command": ["samtools index ${SAMPLE_ID}.bam"],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/output/NIST7035.bam"
+                "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.bam"
             },
             {
                 "name": "SAMPLE_ID",
@@ -18,7 +18,7 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             }
         ]
     }
diff --git a/src/aws-genomics-cdk/examples/batch-samtools-sort.json b/src/aws-genomics-cdk/examples/batch-samtools-sort.json
@@ -6,7 +6,7 @@
         "command": ["samtools sort -@ 4 -o ${SAMPLE_ID}.bam ${SAMPLE_ID}.sam"],
         "environment": [{
                 "name": "JOB_INPUTS",
-                "value": "s3://genomics-pipelines-840249831562/output/NIST7035.sam"
+                "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.sam"
             },
             {
                 "name": "SAMPLE_ID",
@@ -18,7 +18,7 @@
             },
             {
                 "name": "JOB_OUTPUT_PREFIX",
-                "value": "s3://genomics-pipelines-840249831562/output"
+                "value": "s3://YOUR-BUCKET-NAME/output"
             }
         ]
     }

Original file line number	Diff line number	Diff line change
`@@ -1,12 +1,12 @@`
`1`	`1`	`{`
`2`	`2`	`"jobName": "bwa",`
`3`	`3`	`"jobQueue": "genomics-default-queue",`
`4`		`- "jobDefinition": "bwa:3",`
	`4`	`+ "jobDefinition": "bwa:1",`
`5`	`5`	`"containerOverrides": {`
`6`	`6`	`"command": ["bwa mem -t 8 -p -o ${SAMPLE_ID}.sam ${REFERENCE_NAME}.fasta ${SAMPLE_ID}_1.fastq.gz"],`
`7`	`7`	`"environment": [{`
`8`	`8`	`"name": "JOB_INPUTS",`
`9`		`- "value": "s3://genomics-pipelines-840249831562/sample/ s3://genomics-pipelines-840249831562/hg38/v0/"`
	`9`	`+ "value": "s3://aws-batch-genomics-shared/secondary-analysis/example-files/fastq/NIST7035* s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta*"`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`"name": "SAMPLE_ID",`
`@@ -22,11 +22,7 @@`
`22`	`22`	`},`
`23`	`23`	`{`
`24`	`24`	`"name": "JOB_OUTPUT_PREFIX",`
`25`		`- "value": "s3://genomics-pipelines-840249831562/output"`
`26`		`- },`
`27`		`- {`
`28`		`- "name": "JOB_INPUT_S3_COPY_METHOD",`
`29`		`- "value": "s3sync"`
	`25`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`30`	`26`	`}`
`31`	`27`	`]`
`32`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`{`
`2`	`2`	`"jobName": "fastqc",`
`3`	`3`	`"jobQueue": "genomics-default-queue",`
`4`		`- "jobDefinition": "fastqc:3",`
	`4`	`+ "jobDefinition": "fastqc:1",`
`5`	`5`	`"containerOverrides": {`
`6`	`6`	`"command": ["fastqc *.gz"],`
`7`	`7`	`"environment": [{`
`@@ -14,7 +14,7 @@`
`14`	`14`	`},`
`15`	`15`	`{`
`16`	`16`	`"name": "JOB_OUTPUT_PREFIX",`
`17`		`- "value": "s3://genomics-pipelines-840249831562/output"`
	`17`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`18`	`18`	`}`
`19`	`19`	`]`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,15 +6,15 @@`
`6`	`6`	`"command": ["java -jar /usr/app/picard.jar CreateSequenceDictionary R=Homo_sapiens_assembly38.fasta O=Homo_sapiens_assembly38.dict"],`
`7`	`7`	`"environment": [{`
`8`	`8`	`"name": "JOB_INPUTS",`
`9`		`- "value": "s3://genomics-pipelines-840249831562/hg38/v0/Homo_sapiens_assembly38.fasta"`
	`9`	`+ "value": "s3://broad-references/hg38/v0/Homo_sapiens_assembly38.fasta"`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`"name": "JOB_OUTPUTS",`
`13`	`13`	`"value": "*.dict"`
`14`	`14`	`},`
`15`	`15`	`{`
`16`	`16`	`"name": "JOB_OUTPUT_PREFIX",`
`17`		`- "value": "s3://genomics-pipelines-840249831562/hg38/v0"`
	`17`	`+ "value": "s3://YOUR-BUCKET-NAME/ref"`
`18`	`18`	`}`
`19`	`19`	`]`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`"command": ["gatk --java-options \"-Xmx4g\" HaplotypeCaller -R ${REFERENCE_NAME}.fasta -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.vcf.gz -bamout ${SAMPLE_ID}.out.bam"],`
`7`	`7`	`"environment": [{`
`8`	`8`	`"name": "JOB_INPUTS",`
`9`		`- "value": "s3://genomics-pipelines-840249831562/output/ s3://genomics-pipelines-840249831562/hg38/v0/"`
	`9`	`+ "value": "s3://YOUR-BUCKET-NAME/ref s3://YOUR-BUCKET-NAME/samples"`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`"name": "SAMPLE_ID",`
`@@ -22,7 +22,7 @@`
`22`	`22`	`},`
`23`	`23`	`{`
`24`	`24`	`"name": "JOB_OUTPUT_PREFIX",`
`25`		`- "value": "s3://genomics-pipelines-840249831562/output"`
	`25`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`26`	`26`	`},`
`27`	`27`	`{`
`28`	`28`	`"name": "JOB_INPUT_S3_COPY_METHOD",`
Original file line number	Diff line number	Diff line change
`@@ -4,13 +4,13 @@`
`4`	`4`	`"jobDefinition": "picardAddMissingGroups:1",`
`5`	`5`	`"containerOverrides": {`
`6`	`6`	`"command": ["java -jar /usr/picard/picard.jar AddOrReplaceReadGroups",`
`7`		`- " -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.rg.bam -RGID 4 --RGLB lib1 ",`
`8`		`- " --RGPL ILLUMINA --RGPU unit1 --RGSM 20;",`
`9`		`- " cp ${SAMPLE_ID}.rg.bam ${SAMPLE_ID}.bam;",`
`10`		`- " rm ${SAMPLE_ID}.rg.bam"],`
	`7`	`+ " -I ${SAMPLE_ID}.bam -O ${SAMPLE_ID}.rg.bam -RGID 4 --RGLB lib1 ",`
	`8`	`+ " --RGPL ILLUMINA --RGPU unit1 --RGSM 20;",`
	`9`	`+ " mv ${SAMPLE_ID}.rg.bam ${SAMPLE_ID}.bam;"`
	`10`	`+ ],`
`11`	`11`	`"environment": [{`
`12`	`12`	`"name": "JOB_INPUTS",`
`13`		`- "value": "s3://genomics-pipelines-840249831562/output/NIST7035.bam"`
	`13`	`+ "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.bam"`
`14`	`14`	`},`
`15`	`15`	`{`
`16`	`16`	`"name": "SAMPLE_ID",`
`@@ -22,8 +22,8 @@`
`22`	`22`	`},`
`23`	`23`	`{`
`24`	`24`	`"name": "JOB_OUTPUT_PREFIX",`
`25`		`- "value": "s3://genomics-pipelines-840249831562/output"`
	`25`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`26`	`26`	`}`
`27`	`27`	`]`
`28`	`28`	`}`
`29`		`-}`
	`29`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -1,12 +1,12 @@`
`1`	`1`	`{`
`2`	`2`	`"jobName": "samtoolsIndex",`
`3`	`3`	`"jobQueue": "genomics-default-queue",`
`4`		`- "jobDefinition": "samtoolsIndex:2",`
	`4`	`+ "jobDefinition": "samtoolsIndex:1",`
`5`	`5`	`"containerOverrides": {`
`6`	`6`	`"command": ["samtools index ${SAMPLE_ID}.bam"],`
`7`	`7`	`"environment": [{`
`8`	`8`	`"name": "JOB_INPUTS",`
`9`		`- "value": "s3://genomics-pipelines-840249831562/output/NIST7035.bam"`
	`9`	`+ "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.bam"`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`"name": "SAMPLE_ID",`
`@@ -18,7 +18,7 @@`
`18`	`18`	`},`
`19`	`19`	`{`
`20`	`20`	`"name": "JOB_OUTPUT_PREFIX",`
`21`		`- "value": "s3://genomics-pipelines-840249831562/output"`
	`21`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`22`	`22`	`}`
`23`	`23`	`]`
`24`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -6,7 +6,7 @@`
`6`	`6`	`"command": ["samtools sort -@ 4 -o ${SAMPLE_ID}.bam ${SAMPLE_ID}.sam"],`
`7`	`7`	`"environment": [{`
`8`	`8`	`"name": "JOB_INPUTS",`
`9`		`- "value": "s3://genomics-pipelines-840249831562/output/NIST7035.sam"`
	`9`	`+ "value": "s3://YOUR-BUCKET-NAME/samples/NIST7035.sam"`
`10`	`10`	`},`
`11`	`11`	`{`
`12`	`12`	`"name": "SAMPLE_ID",`
`@@ -18,7 +18,7 @@`
`18`	`18`	`},`
`19`	`19`	`{`
`20`	`20`	`"name": "JOB_OUTPUT_PREFIX",`
`21`		`- "value": "s3://genomics-pipelines-840249831562/output"`
	`21`	`+ "value": "s3://YOUR-BUCKET-NAME/output"`
`22`	`22`	`}`
`23`	`23`	`]`
`24`	`24`	`}`