Skip to content

Commit 1f7b334

Browse files
nikellepetrillonikellepetrillo
authored
Merge pull request #1796 from broadinstitute/staging
Staging to master
2 parents 195fa9f + f538068 commit 1f7b334

File tree

59 files changed

+1066
-176
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+1066
-176
lines changed

‎.dockstore.yml‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,11 @@ workflows:
44
- name: aggregate_rsem_results
55
subclass: WDL
66
primaryDescriptorPath: /all_of_us/rna_seq/aggregate_rsem_results.wdl
7-
7+
88
- name: AggregateSusieWorkflow
99
subclass: WDL
1010
primaryDescriptorPath: /all_of_us/rna_seq/AggregateSusieWorkflow.wdl
11-
11+
1212
- name: ArrayImputationQuotaConsumed
1313
subclass: WDL
1414
primaryDescriptorPath: /pipelines/wdl/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl
@@ -25,19 +25,19 @@ workflows:
2525
- name: Beagle5Phasing
2626
subclass: WDL
2727
primaryDescriptorPath: /all_of_us/phasing/Beagle5Phasing.wdl
28-
28+
2929
- name: BuildIndices
3030
subclass: WDL
3131
primaryDescriptorPath: /pipelines/wdl/build_indices/BuildIndices.wdl
3232

3333
- name: CalculatePhenotypeGroups
3434
subclass: WDL
3535
primaryDescriptorPath: /all_of_us/rna_seq/CalculatePhenotypeGroups.wdl
36-
36+
3737
- name: convert_vcf_to_plink_bed
3838
subclass: WDL
3939
primaryDescriptorPath: /all_of_us/admixture/convert_vcf_to_plink_bed.wdl
40-
40+
4141
- name: CramToUnmappedBams
4242
subclass: WDL
4343
primaryDescriptorPath: /pipelines/wdl/reprocessing/cram_to_unmapped_bams/CramToUnmappedBams.wdl
@@ -66,6 +66,10 @@ workflows:
6666
subclass: WDL
6767
primaryDescriptorPath: /all_of_us/cmrg/FixItFelixAndVariantCall.wdl
6868

69+
- name: Glimpse2LowPassImputation
70+
subclass: WDL
71+
primaryDescriptorPath: /pipelines/wdl/glimpse/low_pass_imputation/Glimpse2LowPassImputation.wdl
72+
6973
- name: get_wgs_median_coverage
7074
subclass: WDL
7175
primaryDescriptorPath: /all_of_us/mitochondria/get_wgs_median_coverage.wdl
@@ -216,7 +220,7 @@ workflows:
216220
- name: SusieR_Workflow
217221
subclass: WDL
218222
primaryDescriptorPath: /all_of_us/rna_seq/susieR_workflow.wdl
219-
223+
220224
- name: TestArrayImputationQC
221225
subclass: WDL
222226
primaryDescriptorPath: /verification/test-wdls/TestArrayImputationQC.wdl

‎all_of_us/mitochondria/mt_coverage_merge.wdl‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -566,9 +566,11 @@ task merge_mt_shards {
566566
567567
# Copy tarball to stable GCS location for call caching
568568
command -v gcloud
569-
DEST_ROOT="~{output_bucket}"
570-
DEST_ROOT="${DEST_ROOT%/}"
571-
DEST_PATH="${DEST_ROOT}/~{out_mt_name}.tar.gz"
569+
# Disable parallel composite uploads so md5Hash is available.
570+
gcloud config set storage/parallel_composite_upload_enabled False
571+
DEST_ROOT="~{output_bucket}"
572+
DEST_ROOT="${DEST_ROOT%/}"
573+
DEST_PATH="${DEST_ROOT}/~{out_mt_name}.tar.gz"
572574
gcloud storage cp "~{out_mt_name}.tar.gz" "${DEST_PATH}"
573575
574576
LOCAL_MD5_B64=$(python3 - <<'PY'
@@ -614,15 +616,16 @@ task finalize_mt_with_covdb {
614616
String artifact_prone_sites_reference = "default"
615617
String file_name
616618
Int minimum_homref_coverage = 100
617-
Int homref_position_block_size = 2048
619+
Int homref_position_block_size = 1024
618620
Int n_final_partitions = 1000
619621
Boolean overwrite = false
620622

621623
# Runtime parameters
622-
Int memory_gb = 256
623-
Int cpu = 40
624+
Int memory_gb = 768
625+
Int cpu = 96
624626
Int disk_gb = 4000
625627
String disk_type = "SSD"
628+
String machine_type = "n2d-highmem-96"
626629
}
627630

628631
command <<<
@@ -703,6 +706,7 @@ task finalize_mt_with_covdb {
703706
memory: memory_gb + " GB"
704707
cpu: cpu
705708
disks: "local-disk " + disk_gb + " " + disk_type
709+
predefinedMachineType: machine_type
706710
}
707711
}
708712

‎pipeline_versions.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ BuildIndices 5.1.0 2026-02-13
44
CramToUnmappedBams 1.1.3 2024-08-02
55
ExomeGermlineSingleSample 3.2.7 2026-01-21
66
ExomeReprocessing 3.3.7 2026-01-21
7+
Glimpse2LowPassImputation 0.0.1 2026-02-26
78
IlluminaGenotypingArray 1.12.27 2026-01-21
89
Imputation 1.1.23 2025-10-03
910
ImputationBeagle 3.0.1 2026-02-23

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HDGP1000G_500_chr20_qc_fails_no_valid_contigs.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_only_chr20.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_only_chr20.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["this_contig_is_not_in_inputs_1", "this_contig_is_not_in_inputs_2", "this_contig_is_not_in_inputs_3"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_no_valid_contigs"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HDGP1000G_500_qc_fails_not_bgzipped.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_not_bgzipped.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_not_bgzipped.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_not_bgzipped"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HDGP1000G_500_qc_fails_v3.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_vcf_version_3.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_vcf_version_3.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_vcf_version_3"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HGDP1000G_500_chr20_qc_succeeds.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_only_chr20.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_only_chr20.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_subset_succeeds"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HGDP1000G_500_qc_fails_bad_contig_header_length.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_bad_contig_header_length.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_bad_contig_header_length.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_bad_contig_header_length"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HGDP1000G_500_qc_fails_multiple_errors.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_v3_no_data_not_bgzipped.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_v3_no_data_not_bgzipped.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_multiple_errors"
88
}

‎pipelines/wdl/arrays/imputation_beagle/input_qc/test_inputs/Plumbing/HGDP1000G_500_qc_fails_wrong_contig_name_header.json‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
{
2-
"ArrayImputationQC.multi_sample_vcf": "gs://broad-gotc-test-storage/imputation_beagle/input_qc/scientific/vcfs/test_qc_wrong_contig_name_header.vcf.gz",
2+
"ArrayImputationQC.multi_sample_vcf": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/vcfs/test_qc_wrong_contig_name_header.vcf.gz",
33
"ArrayImputationQC.ref_dict": "gs://gcp-public-data--broad-references/hg38/v0/Homo_sapiens_assembly38.dict",
4-
"ArrayImputationQC.reference_panel_path_prefix": "gs://broad-gotc-test-storage/imputation_beagle/scientific/1000G_HGDP_no_singletons_reference_panel/hgdp.tgp.gwaspy.AN_added.bcf.ac2",
4+
"ArrayImputationQC.reference_panel_path_prefix": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/1000G_HGDP_no_singletons_reference_panelhgdp.tgp.gwaspy.AN_added.bcf.ac2",
55
"ArrayImputationQC.contigs": ["chr1","chr2","chr3","chr4","chr5","chr6","chr7","chr8","chr9","chr10","chr11","chr12","chr13","chr14","chr15","chr16","chr17","chr18","chr19","chr20","chr21","chr22"],
6-
"ArrayImputationQC.genetic_maps_path": "gs://broad-gotc-test-storage/imputation_beagle/scientific/plink-genetic-maps/",
6+
"ArrayImputationQC.genetic_maps_path": "gs://pd-test-storage-public/ArrayImputationQC/input/plumbing/plink-genetic-maps/",
77
"ArrayImputationQC.output_basename": "plumbing_test_fail_wrong_contig_name_header"
88
}

0 commit comments

Comments
 (0)