Np add whitelist metadata to multiome (#1781)

nikellepetrillo · actions-user · web-flow · commit 04365043268b · 2026-02-26T10:28:06.000-05:00
* dummy commit

* add whitelist metadata to h5ad and as txt files

* python indentation

* need strings not files

* need strings not files

* add whitelist file to optimus h5ad metadata

* syntax

* syntax

* syntax

* syntax

* need quotes

* add to h5adwithexons task too

* changelogs

* Updated pipeline_versions.txt with all pipeline version information

* changelogs

* Updated pipeline_versions.txt with all pipeline version information

* changelogs

* changelogs

* doc changes

---------

Co-authored-by: GitHub Action &lt;action@github.com&gt;
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -8,16 +8,16 @@ IlluminaGenotypingArray	 1.12.27	2026-01-21
 Imputation	 1.1.23	2025-10-03 
 ImputationBeagle	 3.0.1	2026-02-23 
 JointGenotyping	 1.7.3	2025-08-11 
-MultiSampleSmartSeq2SingleNucleus	 2.2.4	2026-01-21 
-Multiome	 6.1.4	2026-01-22 
-Optimus	 8.0.5	2026-01-22 
-PairedTag	 2.1.10	2026-01-22 
+MultiSampleSmartSeq2SingleNucleus	 2.2.5	2026-02-24 
+Multiome	 6.1.5	2026-02-24 
+Optimus	 8.0.6	2026-02-24 
+PairedTag	 2.1.11	2026-02-24 
 PeakCalling	 1.0.1	2025-08-11 
 Pipeline Name	Version	Date of Last Commit
 RNAWithUMIsPipeline	 1.0.20	2026-01-21 
 ReblockGVCF	 2.4.4	2026-01-29 
-SlideSeq	 3.6.4	2026-01-22 
-SlideTags	 1.0.7	2026-01-26 
+SlideSeq	 3.6.5	2026-02-24 
+SlideTags	 1.0.8	2026-02-24 
 UltimaGenomicsJointGenotyping	 1.2.3	2025-08-11 
 UltimaGenomicsWholeGenomeCramOnly	 1.1.3	2026-01-21 
 UltimaGenomicsWholeGenomeGermline	 1.2.2	2026-01-29 
diff --git a/pipelines/wdl/multiome/Multiome.changelog.md b/pipelines/wdl/multiome/Multiome.changelog.md
@@ -1,3 +1,9 @@
+# 6.1.5
+2026-02-24 (Date of Last Commit)
+
+* Added 2 new outputs to Multiome.wdl: gex_whitelist_used and atac_whitelist_used; these outputs indicate the whitelist used for the gene expression and ATAC pipelines, respectively. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to JoinMultiomeBarcodes by storing the GEX and ATAC whitelist paths in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 6.1.4
 2026-01-22 (Date of Last Commit)
 
diff --git a/pipelines/wdl/multiome/Multiome.wdl b/pipelines/wdl/multiome/Multiome.wdl
@@ -10,7 +10,7 @@ import "../../../tasks/wdl/Utilities.wdl" as utils
 workflow Multiome {
 
 
-    String pipeline_version = "6.1.4"
+    String pipeline_version = "6.1.5"
 
 
     input {
@@ -76,6 +76,8 @@ workflow Multiome {
     # Determine which whitelist files to use based on cloud provider
     File gex_whitelist = if cloud_provider == "gcp" then gcp_gex_whitelist else azure_gex_whitelist
     File atac_whitelist = if cloud_provider == "gcp" then gcp_atac_whitelist else azure_atac_whitelist
+    String gex_whitelist_gs_path = gex_whitelist
+    String atac_whitelist_gs_path = atac_whitelist
 
     # Make sure either 'gcp' or 'azure' is supplied as cloud_provider input. If not, raise an error
     if ((cloud_provider != "gcp") && (cloud_provider != "azure")) {
@@ -143,7 +145,9 @@ workflow Multiome {
             atac_whitelist = atac_whitelist,
             atac_fragment = Atac.fragment_file,
             input_gtf = annotations_gtf,
-            input_bwa_reference = tar_bwa_reference
+            input_bwa_reference = tar_bwa_reference,
+            gex_whitelist_gs_path = gex_whitelist,
+            atac_whitelist_gs_path = atac_whitelist
     }
 
     if (run_peak_calling) {
@@ -165,6 +169,9 @@ workflow Multiome {
     output {
         
         String multiome_pipeline_version_out = pipeline_version
+        File gex_whitelist_used = JoinBarcodes.gex_whitelist_name_file
+        File atac_whitelist_used = JoinBarcodes.atac_whitelist_name_file
+
 
         # atac outputs
         File bam_aligned_output_atac = Atac.bam_aligned_output
diff --git a/pipelines/wdl/optimus/Optimus.changelog.md b/pipelines/wdl/optimus/Optimus.changelog.md
@@ -1,3 +1,9 @@
+# 8.0.6
+2026-02-24 (Date of Last Commit)
+
+* Added 1 new output to Optimus.wdl: whitelist_input_used; this output indicates the whitelist used. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to OptimusH5adGeneration and SingleNucleusOptimusH5adOutput by storing the whitelist path in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 8.0.5
 2026-01-22 (Date of Last Commit)
 
diff --git a/pipelines/wdl/optimus/Optimus.wdl b/pipelines/wdl/optimus/Optimus.wdl
@@ -78,7 +78,7 @@ workflow Optimus {
   }
 
   # Version of this pipeline
-  String pipeline_version = "8.0.5"
+  String pipeline_version = "8.0.6"
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
   Array[Int] indices = range(length(r1_fastq))
@@ -230,7 +230,8 @@ workflow Optimus {
         empty_drops_result = RunEmptyDrops.empty_drops_result,
         counting_mode = counting_mode,
         pipeline_version = "Optimus_v~{pipeline_version}",
-        warp_tools_docker_path = docker_prefix + warp_tools_docker
+        warp_tools_docker_path = docker_prefix + warp_tools_docker,
+        gex_whitelist_gs_path = whitelist
     }
   }
 
@@ -256,7 +257,8 @@ workflow Optimus {
         cell_id_exon = STARsoloFastq.row_index,
         gene_id_exon = STARsoloFastq.col_index,
         pipeline_version = "Optimus_v~{pipeline_version}",
-        warp_tools_docker_path = docker_prefix + warp_tools_docker
+        warp_tools_docker_path = docker_prefix + warp_tools_docker,
+        gex_whitelist_gs_path = whitelist
     }
   }
 
@@ -295,12 +297,14 @@ workflow Optimus {
   }
 
   File final_h5ad_output = select_first([OptimusH5adGenerationWithExons.h5ad_output, OptimusH5adGeneration.h5ad_output])
+  File final_whitelist_input = select_first([OptimusH5adGenerationWithExons.whitelist_name_file, OptimusH5adGeneration.whitelist_name_file])
   File final_library_metrics = select_first([OptimusH5adGenerationWithExons.library_metrics, OptimusH5adGeneration.library_metrics])
 
   output {
     # version of this pipeline
     String pipeline_version_out = pipeline_version
     File genomic_reference_version = ReferenceCheck.genomic_ref_version
+    File whitelist_input_used = final_whitelist_input
    
     # Metrics outputs
     File cell_metrics = CellMetrics.cell_metrics
diff --git a/pipelines/wdl/paired_tag/PairedTag.changelog.md b/pipelines/wdl/paired_tag/PairedTag.changelog.md
@@ -1,3 +1,9 @@
+# 2.1.11
+2026-02-24 (Date of Last Commit)
+
+* Added 2 new outputs to Multiome.wdl: gex_whitelist_used and atac_whitelist_used; these outputs indicate the whitelist used for the gene expression and ATAC pipelines, respectively. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to JoinMultiomeBarcodes by storing the GEX and ATAC whitelist paths in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 2.1.10
 2026-01-22 (Date of Last Commit)
 
diff --git a/pipelines/wdl/paired_tag/PairedTag.wdl b/pipelines/wdl/paired_tag/PairedTag.wdl
@@ -9,7 +9,7 @@ import "../../../tasks/wdl/Utilities.wdl" as utils
 
 workflow PairedTag {
 
-    String pipeline_version = "2.1.10"
+    String pipeline_version = "2.1.11"
 
     input {
         String input_id
diff --git a/pipelines/wdl/slideseq/SlideSeq.changelog.md b/pipelines/wdl/slideseq/SlideSeq.changelog.md
@@ -1,3 +1,9 @@
+# 3.6.5
+2026-02-24 (Date of Last Commit)
+
+* Added 2 new outputs to Multiome.wdl: gex_whitelist_used and atac_whitelist_used; these outputs indicate the whitelist used for the gene expression and ATAC pipelines, respectively. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to JoinMultiomeBarcodes by storing the GEX and ATAC whitelist paths in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 3.6.4
 2026-01-22 (Date of Last Commit)
 
diff --git a/pipelines/wdl/slideseq/SlideSeq.wdl b/pipelines/wdl/slideseq/SlideSeq.wdl
@@ -25,7 +25,7 @@ import "../../../tasks/wdl/Utilities.wdl" as utils
 
 workflow SlideSeq {
 
-    String pipeline_version = "3.6.4"
+    String pipeline_version = "3.6.5"
 
     input {
         Array[File] r1_fastq
diff --git a/pipelines/wdl/slidetags/SlideTags.changelog.md b/pipelines/wdl/slidetags/SlideTags.changelog.md
@@ -1,3 +1,9 @@
+# 1.0.8
+2026-02-24 (Date of Last Commit)
+
+* Added 2 new outputs to Multiome.wdl: gex_whitelist_used and atac_whitelist_used; these outputs indicate the whitelist used for the gene expression and ATAC pipelines, respectively. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to JoinMultiomeBarcodes by storing the GEX and ATAC whitelist paths in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 1.0.7
 2026-01-26 (Date of Last Commit)
 
diff --git a/pipelines/wdl/slidetags/SlideTags.wdl b/pipelines/wdl/slidetags/SlideTags.wdl
@@ -6,7 +6,7 @@ import "../optimus/Optimus.wdl" as optimus
 
 workflow SlideTags {
 
-    String pipeline_version = "1.0.7"
+    String pipeline_version = "1.0.8"
 
     input {
 
diff --git a/pipelines/wdl/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md b/pipelines/wdl/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md
@@ -1,3 +1,9 @@
+# 2.2.5
+2026-02-24 (Date of Last Commit)
+
+* Added 2 new outputs to Multiome.wdl: gex_whitelist_used and atac_whitelist_used; these outputs indicate the whitelist used for the gene expression and ATAC pipelines, respectively. This change is provenance-only and introduces no functional changes to pipeline outputs
+* Added whitelist provenance tracking to JoinMultiomeBarcodes by storing the GEX and ATAC whitelist paths in the h5ad unstructured metadata (.uns). This change is provenance-only and introduces no functional changes to pipeline outputs
+
 # 2.2.4
 2026-01-21 (Date of Last Commit)
 
diff --git a/pipelines/wdl/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl b/pipelines/wdl/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.wdl
@@ -59,7 +59,7 @@ workflow MultiSampleSmartSeq2SingleNucleus {
   }
 
   # Version of this pipeline
-  String pipeline_version = "2.2.4"
+  String pipeline_version = "2.2.5"
 
   if (false) {
      String? none = "None"
diff --git a/tasks/wdl/H5adUtils.wdl b/tasks/wdl/H5adUtils.wdl
@@ -1,9 +1,8 @@
 version 1.0
 
-
-
 task OptimusH5adGeneration {
 
+
   input {
     #runtime values
     String warp_tools_docker_path
@@ -35,7 +34,7 @@ task OptimusH5adGeneration {
     #String counting_mode = "sc_rna"
     String add_emptydrops_data = "yes"
     String gtf_path = annotation_file
-
+    String gex_whitelist_gs_path
 
     String pipeline_version
 
@@ -58,6 +57,9 @@ task OptimusH5adGeneration {
 
     touch empty_drops_result.csvs
 
+    whitelist_name=$(basename ~{gex_whitelist_gs_path})
+    echo "$whitelist_name" > whitelist_input.txt
+
     if [ "~{counting_mode}" == "sc_rna" ]; then
         python3 /warptools/scripts/create_h5ad_optimus.py \
           ~{if defined(empty_drops_result) then "--empty_drops_file  " + empty_drops_result  else "--empty_drops_file empty_drops_result.csv "  } \
@@ -104,6 +106,15 @@ task OptimusH5adGeneration {
      --counting_mode ~{counting_mode} \
      --expected_cells ~{expected_cells}
 
+
+    python3 <<CODE
+    import anndata as ad
+    adata = ad.read_h5ad("~{input_id}.h5ad")
+    adata.uns["whitelist"] = {"gex_whitelist_gs_path": "~{gex_whitelist_gs_path}"}
+    adata.write("~{input_id}.h5ad")
+    CODE
+
+
     mv library_metrics.csv ~{input_id}_~{gex_nhash_id}_library_metrics.csv
 
   >>>
@@ -120,6 +131,7 @@ task OptimusH5adGeneration {
   output {
     File h5ad_output = "~{input_id}.h5ad"
     File library_metrics = "~{input_id}_~{gex_nhash_id}_library_metrics.csv"
+    File whitelist_name_file = "whitelist_input.txt"
   }
 }
 
@@ -162,6 +174,7 @@ task SingleNucleusOptimusH5adOutput {
         # Cell calls from starsolo in TSV format
         File? cellbarcodes
         String gtf_path = annotation_file
+        String gex_whitelist_gs_path
 
         String pipeline_version
 
@@ -182,6 +195,9 @@ task SingleNucleusOptimusH5adOutput {
     command <<<
         set -euo pipefail
 
+        whitelist_name=$(basename ~{gex_whitelist_gs_path})
+        echo "$whitelist_name" > whitelist_input.txt
+
         python3 /warptools/scripts/create_snrna_optimus_exons_h5ad.py \
         --annotation_file ~{annotation_file} \
         --cell_metrics ~{cell_metrics} \
@@ -211,6 +227,13 @@ task SingleNucleusOptimusH5adOutput {
         --counting_mode ~{counting_mode} \
         --expected_cells ~{expected_cells}
 
+        python3 <<CODE
+        import anndata as ad
+        adata = ad.read_h5ad("~{input_id}.h5ad")
+        adata.uns["whitelist"] = {"gex_whitelist_gs_path": "~{gex_whitelist_gs_path}"}
+        adata.write("~{input_id}.h5ad")
+        CODE
+
 
         mv library_metrics.csv ~{input_id}_~{gex_nhash_id}_library_metrics.csv
 
@@ -227,6 +250,7 @@ task SingleNucleusOptimusH5adOutput {
     output {
         File h5ad_output = "~{input_id}.h5ad"
         File library_metrics = "~{input_id}_~{gex_nhash_id}_library_metrics.csv"
+        File whitelist_name_file = "whitelist_input.txt"
     }
 }
 
@@ -237,6 +261,8 @@ task JoinMultiomeBarcodes {
     File gex_h5ad
     File gex_whitelist
     File atac_whitelist
+    String gex_whitelist_gs_path
+    String atac_whitelist_gs_path
     String input_gtf
     String input_bwa_reference
 
@@ -279,6 +305,8 @@ task JoinMultiomeBarcodes {
     gex_h5ad = "~{gex_h5ad}"
     gex_whitelist = "~{gex_whitelist}"
     atac_whitelist = "~{atac_whitelist}"
+    gex_whitelist_gs = "~{gex_whitelist_gs_path}"
+    atac_whitelist_gs = "~{atac_whitelist_gs_path}"
     input_gtf = "~{input_gtf}"
     input_bwa_reference = "~{input_bwa_reference}"
 
@@ -327,6 +355,29 @@ task JoinMultiomeBarcodes {
     print("Setting Optimus obs to new dataframe")
     gex_data.obs = df_gex
 
+    import os
+
+    # Add whitelist provenance metadata
+    gex_data.uns["whitelists"] = {
+    "gex_whitelist_gs_path": gex_whitelist_gs,
+    "atac_whitelist_gs_path": atac_whitelist_gs
+    }
+
+    atac_data.uns["whitelists"] = {
+    "gex_whitelist_gs_path": gex_whitelist_gs,
+    "atac_whitelist_gs_path": atac_whitelist_gs
+    }
+
+    # write out the names of the whitelists in separate text files for provenance tracking
+    gex_whitelist_name = os.path.basename(gex_whitelist)
+    atac_whitelist_name = os.path.basename(atac_whitelist)
+
+    with open("gex_whitelist_input.txt", "w") as f:
+        f.write(gex_whitelist_name)
+
+    with open("atac_whitelist_input.txt", "w") as f:
+        f.write(atac_whitelist_name)
+
     # write out the files
     gex_data.write("~{gex_base_name}.h5ad")
     atac_data.write_h5ad("~{atac_base_name}.h5ad")
@@ -361,6 +412,8 @@ task JoinMultiomeBarcodes {
     File atac_h5ad_file = "~{atac_base_name}.h5ad"
     File atac_fragment_tsv = "~{atac_fragment_base}.sorted.tsv.gz"
     File atac_fragment_tsv_index = "~{atac_fragment_base}.sorted.tsv.gz.csi"
+    File gex_whitelist_name_file = "gex_whitelist_input.txt"
+    File atac_whitelist_name_file = "atac_whitelist_input.txt"
   }
 }
 
diff --git a/website/docs/Pipelines/Multiome_Pipeline/README.md b/website/docs/Pipelines/Multiome_Pipeline/README.md
@@ -107,6 +107,8 @@ The Multiome workflow calls two WARP subworkflows, one external subworkflow (opt
 | Output variable name | Filename, if applicable | Output format and description |
 |--- | --- | --- | 
 | multiome_pipeline_version_out | N.A. | String describing the version of the Multiome pipeline used. |
+| gex_whitelist_used | `gex_whitelist_input.txt` | File describing the whitelist used for the Optimus pipeline |
+| atac_whitelist_used | `atac_whitelist_input.txt` | File describing the whitelist used for the ATAC pipeline |
 | bam_aligned_output_atac | `<input_id>_atac.bam` | BAM file containing aligned reads from ATAC workflow. |
 | fragment_file_atac | `<input_id>_atac.fragments.sorted.tsv.gz` | Sorted and bgzipped TSV file containing fragment start and stop coordinates per barcode. The columns are "Chromosome", "Start", "Stop", "ATAC Barcode", "Number of reads", and "GEX Barcode". | 
 | fragment_file_index |  `<input_id>_atac.fragments.sorted.tsv.gz.csi` | Tabix CSI index file for the fragment file. |
@@ -139,6 +141,16 @@ The Multiome workflow calls two WARP subworkflows, one external subworkflow (opt
 | cellbybin_h5ad_file | h5ad | Cell by bin matrix produced by SnapATAC2 peak calling. This matrix contains (unmerged) peaks in the MACS3 unstructured metadata (adata.uns['MACS3']). The matrix consists of insertion counts per 500 bp genomic bin and cell barcode. |
 | cellbypeak_h5ad_file | h5ad | Cell by peak matrix produced by SnapATAC2 peak calling. This matrix contains insertion counts per (merged) peak coordinates and per cell barcode. | 
 
+**Provenance metadata (whitelist):** The v6.1.5 Multiome pipeline now records the barcode whitelist used during processing in the .uns metadata of both GEX and ATAC h5ad outputs.. This is a provenance-only update and does not alter any counts, metrics, or downstream results.
+
+You can inspect the whitelist file path in Python as follows:
+
+```python
+import anndata
+
+adata = anndata.read_h5ad("<file_name>.h5ad")
+print(adata.uns["whitelists"])
+```
 
 
 ## Versioning and testing
diff --git a/website/docs/Pipelines/Optimus_Pipeline/README.md b/website/docs/Pipelines/Optimus_Pipeline/README.md

Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ workflow Optimus {`
`78`	`78`	`}`
`79`	`79`
`80`	`80`	`# Version of this pipeline`
`81`		`- String pipeline_version = "8.0.5"`
	`81`	`+ String pipeline_version = "8.0.6"`
`82`	`82`
`83`	`83`	`# this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays`
`84`	`84`	`Array[Int] indices = range(length(r1_fastq))`
`@@ -230,7 +230,8 @@ workflow Optimus {`
`230`	`230`	`empty_drops_result = RunEmptyDrops.empty_drops_result,`
`231`	`231`	`counting_mode = counting_mode,`
`232`	`232`	`pipeline_version = "Optimus_v~{pipeline_version}",`
`233`		`- warp_tools_docker_path = docker_prefix + warp_tools_docker`
	`233`	`+ warp_tools_docker_path = docker_prefix + warp_tools_docker,`
	`234`	`+ gex_whitelist_gs_path = whitelist`
`234`	`235`	`}`
`235`	`236`	`}`
`236`	`237`
`@@ -256,7 +257,8 @@ workflow Optimus {`
`256`	`257`	`cell_id_exon = STARsoloFastq.row_index,`
`257`	`258`	`gene_id_exon = STARsoloFastq.col_index,`
`258`	`259`	`pipeline_version = "Optimus_v~{pipeline_version}",`
`259`		`- warp_tools_docker_path = docker_prefix + warp_tools_docker`
	`260`	`+ warp_tools_docker_path = docker_prefix + warp_tools_docker,`
	`261`	`+ gex_whitelist_gs_path = whitelist`
`260`	`262`	`}`
`261`	`263`	`}`
`262`	`264`
`@@ -295,12 +297,14 @@ workflow Optimus {`
`295`	`297`	`}`
`296`	`298`
`297`	`299`	`File final_h5ad_output = select_first([OptimusH5adGenerationWithExons.h5ad_output, OptimusH5adGeneration.h5ad_output])`
	`300`	`+ File final_whitelist_input = select_first([OptimusH5adGenerationWithExons.whitelist_name_file, OptimusH5adGeneration.whitelist_name_file])`
`298`	`301`	`File final_library_metrics = select_first([OptimusH5adGenerationWithExons.library_metrics, OptimusH5adGeneration.library_metrics])`
`299`	`302`
`300`	`303`	`output {`
`301`	`304`	`# version of this pipeline`
`302`	`305`	`String pipeline_version_out = pipeline_version`
`303`	`306`	`File genomic_reference_version = ReferenceCheck.genomic_ref_version`
	`307`	`+ File whitelist_input_used = final_whitelist_input`
`304`	`308`
`305`	`309`	`# Metrics outputs`
`306`	`310`	`File cell_metrics = CellMetrics.cell_metrics`