almost working version

Author: kcotto

scripts/compare_junctions_hist.py

@@ -4,7 +4,7 @@
 import pandas as pd
 from dfply import *
 import numpy as np
-import glob
+from scipy import stats
 import os
 
 tag = 'E'
@@ -43,6 +43,8 @@ def createkey(row):
     for line in reader:
         all_samples.append(line[0])
 
+num_of_samples = len(all_samples)
+
 ### read in all of the regtools cse output for this cohort ###
 # create list to hold each sample's df
 dfs = []
@@ -123,9 +125,10 @@ def createkey(row):
     samples_w_variant_df = samples_w_variant_df[['sample_y', 'variant_info_y', 'chrom', 'start', 'end', 'strand', 'anchor',
                                                  'info', 'genes', 'name_y', 'mean_norm_score_variant', 'sd_norm_score_variant',
                                                  'norm_score_y', 'score_y', 'junction', 'total_score_variant']]
-    samples_w_variant_df.columns = ['samples', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+    samples_w_variant_df.columns = ['junction_samples', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
                                     'info', 'genes', 'names', 'mean_norm_score_variant', 'sd_norm_score_variant',
                                     'norm_scores_variant', 'scores', 'junction', 'total_score_variant']
+    samples_w_variant_df['variant_samples'] = samples_w_variant_df['junction_samples']
     samples_w_variant_df = samples_w_variant_df[~samples_w_variant_df.astype(
         str).duplicated()]
 else:
@@ -144,12 +147,20 @@ def createkey(row):
     tmp_df = samples_w_variant_df.groupby('info')[['norm_score', 'score', 'sd_norm_score_variant',
                                                    'mean_norm_score_variant', 'sample', 'name']].aggregate(lambda x: x.tolist()).reset_index()
     samples_w_variant_df = pd.merge(samples_w_variant_df, tmp_df, on='info')
-    samples_w_variant_df = samples_w_variant_df[['sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+    samples_w_variant_df = samples_w_variant_df[['sample_x', 'sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
                                                  'info', 'genes', 'name_y', 'mean_norm_score_variant_y', 'sd_norm_score_variant_y',
                                                  'norm_score_y', 'score_y', 'junction', 'total_score_variant']]
-    samples_w_variant_df.columns = ['samples', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+    samples_w_variant_df.columns = ['sample_x', 'sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
                                     'info', 'genes', 'names', 'mean_norm_score_variant', 'sd_norm_score_variant',
                                     'norm_scores_variant', 'scores', 'junction', 'total_score_variant']
+    tmp_df = samples_w_variant_df.groupby('variant_info')[['sample_x']].aggregate(lambda x: set(x.tolist())).reset_index()
+    samples_w_variant_df = pd.merge(samples_w_variant_df, tmp_df, on='variant_info')
+    samples_w_variant_df = samples_w_variant_df[['sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+                                                 'info', 'genes', 'names', 'mean_norm_score_variant', 'sd_norm_score_variant',
+                                                 'norm_scores_variant', 'scores', 'junction', 'total_score_variant', 'sample_x_y']]
+    samples_w_variant_df.columns = ['junction_samples', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+                                    'info', 'genes', 'names', 'mean_norm_score_variant', 'sd_norm_score_variant',
+                                    'norm_scores_variant', 'scores', 'junction', 'total_score_variant', 'variant_samples']
     samples_w_variant_df = samples_w_variant_df[~samples_w_variant_df.astype(
         str).duplicated()]
 
@@ -171,7 +182,7 @@ def createkey(row):
 samples_wout_variant_df = samples_wout_variant_df.loc[samples_wout_variant_df['variant_info'].isin(
     all_splicing_variants['key'])]
 tmp_df = samples_wout_variant_df.groupby(
-    'info')[['norm_score']].aggregate(lambda x: x.tolist()).reset_index()
+    'info')[['norm_score', 'sample']].aggregate(lambda x: x.tolist()).reset_index()
 samples_wout_variant_df = pd.merge(samples_wout_variant_df, tmp_df, on='info')
 samples_wout_variant_df['samples_wout_variant_count'] = samples_wout_variant_df['norm_score_y'].astype(
     str).str.count(',') + 1
@@ -187,37 +198,90 @@ def createkey(row):
                            summarize(total_score_non=X.score.sum()) >>
                            outer_join(samples_wout_variant_df, by='info')
                            )
-    samples_wout_variant_df = samples_wout_variant_df[['sample', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
-                                                       'info', 'genes', 'name', 'norm_score_x_y', 'junction', 'total_score_non', 'samples_wout_variant_count']]
+    samples_wout_variant_df = samples_wout_variant_df[['sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+                                                       'info', 'genes', 'norm_score_x_y', 'junction', 'total_score_non', 'samples_wout_variant_count']]
 else:
     samples_wout_variant_df = (samples_wout_variant_df >>
                            group_by('info') >>
                            summarize(total_score_non=X.score.sum()) >>
                            outer_join(samples_wout_variant_df, by='info')
                            )
-    samples_wout_variant_df = samples_wout_variant_df[['sample', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
-                                                       'info', 'genes', 'name', 'norm_score_y', 'junction', 'total_score_non', 'samples_wout_variant_count']]
+    samples_wout_variant_df = samples_wout_variant_df[['sample_y', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
+                                                       'info', 'genes', 'norm_score_y', 'junction', 'total_score_non', 'samples_wout_variant_count']]
 samples_wout_variant_df.columns = ['sample', 'variant_info', 'chrom', 'start', 'end', 'strand', 'anchor',
-                                'info', 'genes', 'name', 'norm_scores_non', 'junction', 'total_score_non', 'samples_wout_variant_count']
+                                'info', 'genes', 'norm_scores_non', 'junction', 'total_score_non', 'samples_wout_variant_count']
 
 print('Merging dataframes')
-# samples_w_variant_df['samples'] = samples_w_variant_df['samples'].astype(str)
-# samples_w_variant_df['variant_info'] = samples_w_variant_df['variant_info'].astype(str)
-samples_w_variant_df['info'] = samples_w_variant_df['info'].astype(str)
-# samples_w_variant_df['names'] = samples_w_variant_df['names'].astype(str)
-# samples_w_variant_df['norm_scores_variant'] = samples_w_variant_df['norm_scores_variant'].astype(str)
-# samples_w_variant_df['scores'] = samples_w_variant_df['scores'].astype(str)
-# samples_wout_variant_df['norm_scores_non'] = samples_wout_variant_df['norm_scores_non'].astype(str)
-samples_wout_variant_df['info'] = samples_wout_variant_df['info'].astype(str)
-master_df = pd.merge(samples_w_variant_df, samples_wout_variant_df, how='outer' ,on='info')
+master_df = pd.merge(samples_w_variant_df, samples_wout_variant_df, how='left' ,on='info')
+master_df = master_df[-master_df.astype(
+        str).duplicated()]
 del(samples_wout_variant_df)
 del(samples_w_variant_df)
 
-master_df['samples_w_variant_count'] = master_df['norm_score_y'].astype(
+master_df['samples_w_variant_count'] = master_df['variant_samples'].astype(
     str).str.count(',') + 1
 
-samples_wout_variant_df = (samples_wout_variant_df >>
-                           group_by('info') >>
-                           summarize(mean_norm_score_non=X.norm_score.mean(), sd_norm_score_non=X.norm_score.std(), total_score_non=X.score.sum()) >>
-                           outer_join(samples_wout_variant_df, by='info')
-                           )
+tmp_df = master_df[['info', 'norm_scores_non', 'samples_wout_variant_count', 'samples_w_variant_count']]
+tmp_df = tmp_df.fillna(0)
+
+def add_zeros(row):
+    norm_scores = row[1]
+    if norm_scores == 0:
+        norm_scores = ['0']
+    samples_wout_variant = row[2]
+    samples_w_variant = row[3]    
+    num_of_zeros_toadd = num_of_samples - samples_wout_variant - samples_w_variant
+    zeros = np.repeat(0, num_of_zeros_toadd).tolist()
+    norm_scores = norm_scores + zeros
+    new_norm_score_value = (',').join(map(str, norm_scores))
+    return new_norm_score_value
+
+tmp_df['new_norm_scores'] = tmp_df.apply(lambda row: add_zeros(row), axis=1)
+master_df = pd.merge(master_df, tmp_df, how='left' ,on='info')
+del(tmp_df)
+
+def get_mean(row):
+    values = row[-1].split(',')
+    values = [float(i) for i in values]
+    mean = np.mean(values)
+    return mean
+
+master_df['mean_norm_score_non'] = master_df.apply(lambda row: get_mean(row), axis=1)
+
+def get_sd(row):
+    values = row[-2].split(',')
+    values = [float(i) for i in values]
+    std = np.std(values)
+    return std
+
+master_df['sd_norm_score_non'] = master_df.apply(lambda row: get_sd(row), axis=1)
+
+def get_min(row):
+    values = row[12]
+    values = [float(i) for i in values]
+    minimum = min(values)
+    return(minimum)
+
+master_df['min_norm_score_variant'] = master_df.apply(lambda row: get_min(row), axis=1)
+
+def get_pvalue_mean(row):
+    values = row[32].split(',')
+    values = [float(i) for i in values]
+    mean_value = row[10]
+    pvalue = stats.percentileofscore(values, mean_value)
+    pvalue = 1 - (pvalue/100.0)
+    return pvalue
+
+master_df['p_value_mean'] = master_df.apply(lambda row: get_pvalue_mean(row), axis=1)
+
+def get_pvalue_min(row):
+    values = row[32].split(',')
+    values = [float(i) for i in values]
+    mean_value = row[35]
+    pvalue = stats.percentileofscore(values, mean_value)
+    pvalue = 1 - (pvalue/100.0)
+    return pvalue
+    
+master_df['p_value_min'] = master_df.apply(lambda row: get_pvalue_mean(row), axis=1)
+
+# master_df = master_df[['samples', 'variant_info_x', ']]