final touches to prep for PR

Removed samples corresponding to organoids in experiments file, fixed checking of prior drug file using code from pancPDO directory, removed some print lines and comments, general tidying. Added 'sarcpdo' to build_datasets.py and tested. Also added a few lines to the docker-compose so that build_datasets.py would run.

Author: RubyFore

build/build_dataset.py

@@ -43,6 +43,7 @@ def process_docker(dataset,validate):
         'mpnst': ['mpnst'],
         'mpnstpdx': ['mpnstpdx'],
         'cptac': ['cptac'],
+        'sarcpdo': ['sarcpdo'],
         'genes': ['genes'],
         'upload': ['upload']
     }
@@ -123,7 +124,8 @@ def process_omics(executor, dataset, should_continue):
         'broad_sanger': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'cptac': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'hcmi': ['mutations', 'transcriptomics'],
-        'mpnstpdx':['copy_number', 'mutations', 'proteomics', 'transcriptomics']
+        'mpnstpdx':['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
+        'sarcpdo': ['mutations', 'transcriptomics']
     }
 
     expected_omics = dataset_omics_files.get(dataset, [])

build/docker/docker-compose.yml

@@ -53,6 +53,7 @@ services:
         HTTPS_PROXY: ${HTTPS_PROXY}
     platform: linux/amd64
     image: mpnstpdx:latest
+
   cptac:
     build:
       context: ../../
@@ -62,6 +63,15 @@ services:
     platform: linux/amd64
     image: cptac:latest
 
+  sarcpdo:
+    build:
+      context: ../../
+      dockerfile: build/docker/Dockerfile.sarcpdo
+      args:
+        HTTPS_PROXY: ${HTTPS_PROXY}
+    platform: linux/amd64
+    image: sarcpdo:latest
+
   genes:
     build:
       context: ../../

build/sarcpdo/00_createSarcPDOSampleFile.py

@@ -92,42 +92,34 @@ def download_and_format_rna_samples(synLoginObject):
 
     return rna_samples
 
-    #def generate_samples_file(prev_samples_path):
-
-    # if prev_samples_path == "":
-        #maxval = 0
-   # else:
-    #    maxval = max(pd.read_csv(prev_samples_path).improve_sample_id)
+    
 
 if __name__ == "__main__":
-    print('in main')
+    
     parser = argparse.ArgumentParser(description="This script handles downloading, processing and formatting of sample files for the Sarcoma PDO project into a single samplesheet")
-    print('in line 97')
+    
     parser.add_argument('-t', '--token', type=str, help='Synapse Token')
 
     parser.add_argument("-p", '--prevSamples', nargs="?", type=str, default ="", const  = "", help = "Use this to provide previous sample file, will run sample file generation")
 
     args = parser.parse_args()
-    print(args)
+   
     print("Logging into Synapse")
     PAT = args.token
     synObject = synapseclient.login(authToken=PAT)
 
     rnaTable = download_and_format_rna_samples(synObject)
-    print(rnaTable.shape)
     geneticTable = download_and_format_genetic_samples(synObject)
-    print(geneticTable.shape)
     merged = rnaTable.merge(geneticTable, how='outer')
-    print(merged.shape)
-    # change dash to underscore to align with omics data
-    #merged['other_id'] = merged['other_id'].str.replace("-2", "_2")
+    
+
+    if (args.prevSamples):
+        prev_max_improve_id = max(pd.read_csv(args.prevSamples).improve_sample_id)
+    else: 
+        prev_max_improve_id = 0
 
-    prev_max_improve_id = max(pd.read_csv(args.prevSamples).improve_sample_id)
     merged['improve_sample_id'] = range(prev_max_improve_id+1, prev_max_improve_id+merged.shape[0]+1) 
 
     merged.to_csv('/tmp/sarcpdo_samples.csv', index=False)
 
-        # validate with: linkml validate -s coderdata/schema/coderdata.yaml ~/Downloads/sarcpdo_samples.csv
-
-    # test script : python3 00_createSarcPDOSampleFile.py -t $SYNAPSE_AUTH_TOKEN -p '~/Downloads/mpnstpdx_samples.csv'
-
+        

build/sarcpdo/01_createSarcPDOOmicsFiles.py

@@ -55,10 +55,9 @@ def download_and_format_genomic_mutation(synLoginObject, genesTable, samplesTabl
     # reformat variant classification column to be accepted by linkML and correct
     mutation_merged["variant_classification"] =mutation_merged['Canonical_Variant_Classification']
 
-    #mutation_merged['variant_classification'] = 
-    #mutation_merged['variant_classification'].replace("Missense", "Missense_Mutation", inplace=True)
+    
     mutation_merged.replace({'variant_classification': "Missense"}, "Missense_Mutation", inplace=True)
-    #mutation_merged['variant_classification'] = 
+    
     mutation_merged.replace({'variant_classification': "Splice_Donor"}, "Splice_Site", inplace=True)
     mutation_merged.replace({'variant_classification': "Splice_Acceptor"}, "Splice_Site", inplace=True)
     mutation_merged.replace({'variant_classification': "Nonsense"}, "Nonsense_Mutation", inplace=True)
@@ -69,12 +68,6 @@ def download_and_format_genomic_mutation(synLoginObject, genesTable, samplesTabl
     mutation_merged.replace({'variant_classification': "Frameshift"}, "Frameshift_Variant", inplace=True)
     mutation_merged.replace({'variant_classification': "intergenic_variant"}, "Silent", inplace=True)
 
-   # mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace("Nonsense", "Nonsense_Mutation", inplace=True)
-    #mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace('intron', 'Intron', inplace=True)
-    #mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace("synonymous", "Silent", inplace=True)
-    #mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace("Inframe_Del", "In_Frame_Del", inplace=True)
-    #mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace("5_prime_UTR", "5' UTR", inplace=True)
-    #mutation_merged['variant_classification'] = mutation_merged['variant_classification'].replace("intergenic_variant", "Silent", inplace=True)
     mutation_merged_select = mutation_merged[['entrez_id', 'Sample_ID_Tumor', 'Name', 'variant_classification']]
     #merge with improve_ids 
     samples['other_id_no_dash'] = samples['other_id'].str.replace("-2", "_2")
@@ -112,13 +105,9 @@ def download_and_format_genomic_mutation(synLoginObject, genesTable, samplesTabl
     if args.expression:
         download_and_format_transcriptomic(synObject, genes, samples).to_csv("/tmp/sarcpdo_transcriptomics.csv", index=False)
 
-   # if args.copy: 
-    #    download_and_format_copy_number(synObject, genes, samples).to_csv('sarcpdo_copynumber.csv', index=False)
-
+   
     if args.mutation:
-        download_and_format_genomic_mutation(synObject, genes, samples).to_csv('/tmp/sarcpdo_mutation.csv', index=False)
+        download_and_format_genomic_mutation(synObject, genes, samples).to_csv('/tmp/sarcpdo_mutations.csv', index=False)
 
-          # validate with: linkml validate -s coderdata/schema/coderdata.yaml ~/Downloads/sarcpdo_samples.csv
 
 
-    # command line testing: python3 01_createSarcPDOOmicsFiles.py -t $SYNAPSE_AUTH_TOKEN -s dev-environment/sarcpdo_samples.csv -g genes.csv -e

build/sarcpdo/02_createSarcPDODrugsFile.py

@@ -4,30 +4,38 @@
 import argparse
 import os
 
+#from utils.pubchem_retrieval import update_dataframe_and_write_tsv
 from pubchem_retrieval import update_dataframe_and_write_tsv
 
 
 def create_sarcpdo_drugs_file(synObject, prevDrugFilepath, outputPath):
     drug_query = synObject.tableQuery("select * from syn61892224")
     drug_data = drug_query.asDataFrame()
-    # check status of previous drug file
-    if not prevDrugFilepath:
-        # if sarcpdo_drugs.tsv is null, create the empty dataframe. 
-        empty_drugs = pd.DataFrame(columns = ['improve_drug_id', 'chem_name', 'pubchem_id', 'canSMILES', 'InChIKey', 'formula', 'weight'])
-        empty_drugs.to_csv('outputPath', sep='\t', index=False)
 
     # get unique drugs 
-    unique_drugs = drug_data['Drug_Name'].unique()
+    newdrugnames = drug_data['Drug_Name'].unique()
     # use helper functions in pubchem_retrieval.py 
-    update_dataframe_and_write_tsv(unique_drugs, output_filename=outputPath, # specify ignore_chems as null?
-                                   batch_size=1, isname=True, time_limit=48 * 60 * 60)
+    alldrugs = []
+    if prevDrugFilepath is not None and prevDrugFilepath is not "":
+        prevdrugs = [pd.read_csv(t,sep='\t') for t in prevDrugFilepath.split(',')]
+        alldrugs = pd.concat(prevdrugs).drop_duplicates()
+
+        imps = alldrugs[alldrugs.chem_name.isin(newdrugnames)]
+        newdrugs = alldrugs[alldrugs.improve_drug_id.isin(imps.improve_drug_id)]
+        
+        ##write drugs
+        newdrugs.to_csv(outputPath, sep='\t', index=False)
+
+    if len(alldrugs)==0 or len(newdrugnames)>len(set(newdrugs.improve_drug_id)): #we have more names we didn't match
+        print('Missing drugs in existing file, querying pubchem')
+        update_dataframe_and_write_tsv(newdrugnames,outputPath)
 
 
 if __name__ == "__main__":
-    print('in main')
+
     parser = argparse.ArgumentParser(description="This script handles downloading, processing and formatting of drug data files for the Sarcoma PDO project")
-    parser.add_argument('-d', '--prevDrugFilePath', help='Path to a previous drug file for sarcpdo',default=None)
-    parser.add_argument('-o', '--outputPath', help='Output path for updated sarcpdo drug file', default = None) 
+    parser.add_argument('-d', '--prevDrugFilePath', help='Path to a previous drug file for sarcpdo', default = None)
+    parser.add_argument('-o', '--outputPath', help='Output path for updated sarcpdo drug file', default = "/tmp/sarcpdo_drugs.tsv") 
     parser.add_argument('-t', '--token', help='Synapse token')
 
     args = parser.parse_args()
@@ -37,4 +45,3 @@ def create_sarcpdo_drugs_file(synObject, prevDrugFilepath, outputPath):
 
     create_sarcpdo_drugs_file(synObject, args.prevDrugFilePath, args.outputPath)
 
-    # command line testing: python3 02_createSarcPDODrugsFile.py -t $SYNAPSE_AUTH_TOKEN -d ../../../sarcpdo_drugs.csv -o sarcpdo_drugs.csv

build/sarcpdo/03_createSarcPDOExperimentFile.py

@@ -14,7 +14,7 @@
     parser.add_argument('-d', '--drugFile', nargs = "?", type=str, default = "", help = "Use this to provide previously generated drugs file for this dataset to link with to experiment data.")
 
     args = parser.parse_args()
-    print(args)
+    
     print("Logging into Synapse")
     PAT = args.token
     synObject = synapseclient.login(authToken=PAT)
@@ -38,10 +38,13 @@
 
     # inner merge with samples because there are samples without experiment info and many Sample_ID's in experiments data without sample info
     experiments = drug_data.merge(sarcpdo_drugs, how='left').merge(sarcpdo_samples, how='inner')
-
-    final_experiment = experiments[['improve_sample_id', 'improve_drug_id', 'Viability_Score']]
+    # drop rows corresponding to organoids
+    tumor_only = experiments[~experiments['model_type'].str.contains("organoid")]
+    # select relevant columns 
+    final_experiment = tumor_only[['improve_sample_id', 'improve_drug_id', 'Viability_Score']]
+    # add static info
     final_experiment.loc[:,['study']] = 'Landscape of Sarcoma'
-    final_experiment.loc[:,['source']] = 'pharmacoGX'
+    final_experiment.loc[:,['source']] = 'AlShihabietal2024'
     final_experiment.loc[:,['time']] = None
     final_experiment.loc[:,['time_unit']]= None
     final_experiment.loc[:,['dose_response_metric']] = 'published_auc' 
@@ -50,7 +53,3 @@
     toReturn = final_experiment[['source', 'improve_sample_id', 'improve_drug_id', 'study', 'time', 'time_unit', 'dose_response_metric', 'dose_response_value']]
 
     toReturn.to_csv('/tmp/sarcpdo_experiments.tsv', sep='\t', index=False)
-
-
-    # to test run
-    #  python3 03_createSarcPDOExperimentFile.py -t $SYNAPSE_AUTH_TOKEN -s sarcpdo_samples.csv -d sarcpdo_drugs.tsv

build/sarcpdo/build_drugs.sh

@@ -4,7 +4,11 @@ set -euo pipefail
 trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR
 
 echo "Running script with token and drugFile $1"
-python3 02_createSarcPDODrugsFile --token $SYNAPSE_AUTH_TOKEN -d $1 -o /tmp/sarcpdo_drugs.tsv
+# for running locally (from build directory):
+#python3 -m sarcpdo.02_createSarcPDODrugsFile --token $SYNAPSE_AUTH_TOKEN -d $1 -o /tmp/sarcpdo_drugs.tsv
+python3 02_createSarcPDODrugsFile.py --token $SYNAPSE_AUTH_TOKEN -d $1 -o /tmp/sarcpdo_drugs.tsv
 
 echo "Running build_drug_desc.py..."
+#for running locally: 
+#python3 utils/build_drug_desc.py --drugtable /tmp/sarcpdo_drugs.tsv --desctable /tmp/sarcpdo_drug_descriptors.tsv.gz
 python3 build_drug_desc.py --drugtable /tmp/sarcpdo_drugs.tsv --desctable /tmp/sarcpdo_drug_descriptors.tsv.gz