Merge pull request #365 from PNNL-CompBio/bladder_pdo

BladderPDO and Schema Fixes

Author: sgosline

build/bladderpdo/00_createBladderPDOSampleFile.py

@@ -1,12 +1,49 @@
+#!/usr/bin/env python3
 import synapseclient
 import pandas as pd
 import numpy as np
 import argparse
 import os
+import re
+import subprocess
 
+# Helper functions
+def _clean_geo_id(s):
+    """
+    Normalise GEO sample IDs so they match Synapse naming.
+      • 11.2  → 11_2
+      • **_Tumor → *_Parental
+      • *_orgP2 → *_Organoid_P2
+      • *_xenoorgP4 → *_XenoOrganoid_P4
+    """
+    s = s.strip()
+    s = re.sub(r"(?<=\d)\.(?=\d)", "_", s)          # dots between digits
+    s = s.replace("_tumor", "_Parental")            # tumour alias
+    # lower-case 'orgP' / 'xenoorgP' fix
+    s = re.sub(r"_(org)P(\d+)",      r"_Organoid_P\2",      s, flags=re.IGNORECASE)
+    s = re.sub(r"_(xenoorg)P(\d+)",  r"_XenoOrganoid_P\2",  s, flags=re.IGNORECASE)
+    return s
 
+
+def _parse_model_type(sample_id):
+    """Derive model_type from Sample ID."""
+    low = sample_id.lower()
+    if "_xenoorganoid" in low:
+        return "xenograft derived organoid"
+    if "_organoid" in low:
+        return "organoid"
+    if "_xenograft" in low:
+        return "patient derived xenograft"
+    if "_parental" in low:
+        return "tumor"
+    return "unknown"
+
+#Generate Samples Data
 def get_bladder_pdo_samples(synLoginObject, maxval):
 
+    
+    #Part 1: Get Data from Synapse
+    
     # download from Synapse..
     samples_syn = synLoginObject.get('syn64765486')
     # and read the file
@@ -19,7 +56,43 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
     samples.loc[:,['other_id_source']] = 'Synapse'
     samples.loc[:,['other_names'] ]= ''
     samples.loc[:,['cancer_type']]=samples['cancer_type'].str.lower()
-    samples.loc[:, ['model_type']] = samples['model_type'].str.lower()
+    samples["model_type"] = samples["other_id"].apply(_parse_model_type)
+
+    #Part 2: Get Data from Geo
+    subprocess.call (["Rscript", "--vanilla", "obtainGSMidLink.R"])
+    GEO_ids_link = "./gsmlinkDf.csv"
+
+    geo_map  = pd.read_csv(GEO_ids_link)
+    geo_ids  = geo_map["sampleid"].dropna().map(_clean_geo_id).unique()
+    missing  = sorted(set(geo_ids) - set(samples["other_id"]))
+
+    if missing:                    
+        print(f"Adding {len(missing)} GEO samples not in Synapse sheet")
+
+    rows = []                      
+    for oid in missing:            
+        common = oid.split("_")[0]
+        ctype  = (
+            samples.loc[samples["common_name"] == common, "cancer_type"]
+            .iloc[0]
+            if (samples["common_name"] == common).any()
+            else "bladder urothelial carcinoma"
+        )
+        rows.append(
+            {
+                "other_id":        oid,
+                "common_name":     common,
+                "cancer_type":     ctype,
+                "model_type":      _parse_model_type(oid),
+                "species":         "Homo sapiens(Human)",
+                "other_id_source": "GEO",
+                "other_names":     "",
+            }
+        )
+    if rows:                       
+        samples = pd.concat([samples, pd.DataFrame(rows)], ignore_index=True)
+
+    samples = samples.sort_values("other_id").reset_index(drop=True)
 
     samples['improve_sample_id'] = range(maxval+1, maxval+1+samples.shape[0])
 
@@ -29,11 +102,8 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description="This script handles downloading, processing and formatting of sample files for the Sarcoma PDO project into a single samplesheet")
-    
     parser.add_argument('-t', '--token', type=str, help='Synapse Token')
-
     parser.add_argument("-p", '--prevSamples', nargs="?", type=str, default ="", const  = "", help = "Use this to provide previous sample file, will run sample file generation")
-
     args = parser.parse_args()
 
     print("Logging into Synapse")
@@ -46,5 +116,4 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
         prev_max_improve_id = 0
 
     bladder_pdo_samples = get_bladder_pdo_samples(synObject, prev_max_improve_id)
-
     bladder_pdo_samples.to_csv("/tmp/bladderpdo_samples.csv", index=False)

build/bladderpdo/01_createBladderPDOOmicsFiles.py

@@ -7,6 +7,7 @@
 import gzip
 import subprocess
 import math
+import re
 
 def get_copy_call(a):
     """
@@ -31,7 +32,21 @@ def get_copy_call(a):
     else:
         return 'amp'
 
-    return pd.Series([get_copy_call(a) for a in arr])
+
+def normalise_id(s):
+    """
+    Make GEO sample IDs line up with 'other_id' in bladderpdo_samples.csv.
+    """
+    if pd.isna(s):
+        return s
+    s = s.strip()
+    s = re.sub(r"(?<=\d)\.(?=\d)", "_", s)                 # dots → underscore
+    s = s.replace("_tumor", "_Parental")                   # tumour alias
+    s = re.sub(r"_(org)P(\d+)",      r"_Organoid_P\2",     s, flags=re.IGNORECASE)
+    s = re.sub(r"_(xenoorg)P(\d+)", r"_XenoOrganoid_P\2",  s, flags=re.IGNORECASE)
+    return s
+
+
 
 def get_bladder_pdo_transcriptomics(GEO_id_link_table, samples, genes):
 
@@ -40,30 +55,42 @@ def get_bladder_pdo_transcriptomics(GEO_id_link_table, samples, genes):
     transcriptomics = pd.read_csv(transcriptomic_txt, compression='gzip', sep="\t")
     subprocess.call (["/usr/bin/Rscript", "--vanilla", "obtainGSMidLink.R"])
 
-    GEO_ids_link = pd.read_csv("./gsmlinkDf.csv")
+    GEO_ids = pd.read_csv(GEO_id_link_table)
+    print(GEO_ids)
     fpkm_totals = transcriptomics.iloc[:, 1:43].sum()
     transcriptomics.iloc[:, 1:43] = transcriptomics.iloc[:, 1:43].div(fpkm_totals).mul(1e6)
     transcriptomics['ensembl'] = transcriptomics['Unnamed: 0'].str.split("_", expand=True)[0]
     mapped_df = transcriptomics.merge(genes[['entrez_id', 'other_id']].drop_duplicates(), left_on='ensembl', right_on='other_id', how='left')
     # transform data to long format
+    print(mapped_df)
 
-    mapped_df.drop('other_id', axis=1)
+    mapped_df = mapped_df.drop('other_id', axis=1)
     value_variables = transcriptomics.columns[transcriptomics.columns.str.contains("M")]
     melted_txomics = mapped_df.melt(id_vars = "entrez_id", value_vars = value_variables, var_name='sample_name')
     # use info from GEO to get Sample IDS
-    txomics_with_GEOid = melted_txomics.merge(GEO_ids_link, how = 'left', left_on = "sample_name", right_on='RNAid')
+    m1 = melted_txomics.merge(GEO_ids, how="left", left_on="sample_name", right_on="RNAid")
+    m1["sampleid"] = m1["sampleid"].apply(normalise_id)
+    print(m1)
+    print(m1.sampleid.unique())
     # use samplesheet to link sample_ids to improve ids
-    txomics_with_GEOid['sampleid'] = txomics_with_GEOid['sampleid'].str.replace("org", "Organoid_")
-    txomics_with_GEOid['sampleid'] = txomics_with_GEOid['sampleid'].str.replace("tumor", "Tumor")
-    txomics_with_improveid = txomics_with_GEOid.merge(samples, left_on="sampleid", right_on="other_id", how="left")
-    final_transcriptomics = txomics_with_improveid[['entrez_id', 'value', 'improve_sample_id']]
-    final_transcriptomics['source'] = "Gene Expression Omnibus"
-    final_transcriptomics['study'] = "Lee etal 2018 Bladder PDOs"
-    final_transcriptomics.rename({'value' : 'transcriptomics' })
-    # remove duplicates
-    toreturn = final_transcriptomics.drop_duplicates()
-
-    return toreturn
+    tx_with_ids = m1.merge(
+        samples, left_on="sampleid", right_on="other_id", how="left"
+    )
+    print(tx_with_ids)
+
+    final_tx = (
+        tx_with_ids[["entrez_id", "value", "improve_sample_id"]]
+        .drop_duplicates()
+        .assign(source="Gene Expression Omnibus",
+                study="Lee et al. 2018 Bladder PDOs")
+    )
+    final_tx.rename(columns= {"value":"transcriptomics"},inplace=True)
+    final_tx = final_tx.drop_duplicates()
+    final_tx = final_tx.dropna(subset=["entrez_id"])
+    final_tx["improve_sample_id"] = final_tx["improve_sample_id"].astype(int)
+    final_tx["entrez_id"]         = final_tx["entrez_id"].astype(int)
+
+    return final_tx
 
 def get_bladder_pdo_mutations(synObject, samples, genes):
     print(samples.head)
@@ -74,10 +101,11 @@ def get_bladder_pdo_mutations(synObject, samples, genes):
     selectioncols_mutations = mutations_df[['Entrez_Gene_Id',"Variant_Classification", "Tumor_Sample_Barcode", "mutation"]]
     merged_mutations = selectioncols_mutations.merge(samples, left_on="Tumor_Sample_Barcode", right_on="other_id", how="left")
     merged_mutations_renamed = merged_mutations.rename({"Entrez_Gene_Id" : 'entrez_id', 'Variant_Classification' : "variant_classification"}, axis=1)
-    print(merged_mutations_renamed.head)
     final_mutations = merged_mutations_renamed[['entrez_id', "mutation", "variant_classification", "improve_sample_id"]]
     final_mutations['study'] = "Lee etal 2018 Bladder PDOs"
-    print(final_mutations.head)
+    final_mutations = final_mutations.dropna(subset=["entrez_id"])
+    final_mutations["improve_sample_id"] = final_mutations["improve_sample_id"].astype(int)
+    final_mutations["entrez_id"]         = final_mutations["entrez_id"].astype(int)
     return final_mutations
 
 def get_bladder_pdo_copynumber(synObject, samples, genes):
@@ -94,7 +122,9 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
     final_copynumber = copynumber_with_correct_colnames[['entrez_id', 'improve_sample_id', 'copy_number', 'copy_call']]
     final_copynumber['source'] = "Synapse"
     final_copynumber['study'] = "Lee etal 2018 Bladder PDOs"
-
+    final_copynumber = final_copynumber.dropna(subset=["entrez_id"])
+    final_copynumber["improve_sample_id"] = final_copynumber["improve_sample_id"].astype(int)
+    final_copynumber["entrez_id"]         = final_copynumber["entrez_id"].astype(int)
     return final_copynumber
 
 
@@ -108,7 +138,7 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
     parser.add_argument('-c', '--copy', help='Flag to capture copy number data', action='store_true', default=False)
     parser.add_argument('-m', '--mutation', help='Flag to capture mutation data', action='store_true', default=False)
     parser.add_argument('-e', '--expression', help='Flag to capture transcriptomic data', action='store_true', default=False)
-    parser.add_argument('-i', '--geolink', help=".csv file that is the output of 'CNV-segfile-anotation.R")
+    parser.add_argument('-i', '--geolink', default = "./gsmlinkDf.csv", help=".csv file that is the output of 'CNV-segfile-anotation.R")
     parser.add_argument('-t', '--token', help='Synapse token')
 
     args = parser.parse_args()
@@ -129,4 +159,4 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
         get_bladder_pdo_mutations(synObject, samples, genes).to_csv('/tmp/bladderpdo_mutations.csv', index=False)
 
     if args.copy:
-        get_bladder_pdo_copynumber(synObject, samples, genes).to_csv("/tmp/bladderpdo_copynumber.csv", index=False)
+        get_bladder_pdo_copynumber(synObject, samples, genes).to_csv("/tmp/bladderpdo_copy_number.csv", index=False)

build/bladderpdo/build_exp.sh

@@ -6,4 +6,6 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit
 echo "Running 04-drug_dosage_and_curves.py with drugfile $2 and curSampleFile $1"
 python3 03_createBladderPDOExperimentFile.py --token $SYNAPSE_AUTH_TOKEN --drugfile $2 --curSampleFile $1 --output /tmp/bladderpdo_doserep.tsv
 
-python3 fit_curve.py --input /tmp/bladderpdo_doserep.tsv --output /tmp/bladderpdo_experiments.tsv
+python3 fit_curve.py --input /tmp/bladderpdo_doserep.tsv --output /tmp/bladderpdo_experiments.tsv
+rm /tmp/bladderpdo_doserep.tsv
+mv /tmp/bladderpdo_experiments.tsv.0 /tmp/bladderpdo_experiments.tsv

build/improve_drug_mapping.json

@@ -2,7 +2,7 @@
   "metadata": {
     "builds": [
       {
-        "build_date": "01_24_25",
+        "build_date": "2025-01-24",
         "version": "2.0.0"
       },
       {