pandasGEXpress/diff_gctoo.py: added median_norm functionality, documentation, and small tweaks

lev · lev · commit 74e0ec4213b9 · 2018-04-13T19:17:15.000-04:00
diff --git a/cmapPy/pandasGEXpress/diff_gctoo.py b/cmapPy/pandasGEXpress/diff_gctoo.py
@@ -1,46 +1,85 @@
 '''
 diff_gctoo.py
 
-Given a GCToo object calculates differential values (expression, viability etc.)
-Values can be made differential relative to all samples on a plate ("plate-control")
-or relative to negative control samples ("vehicle-control").
+Converts a matrix of values (e.g. gene expression, viability, etc.) into a
+matrix of differential values. Values can be made differential relative to all
+samples in the dataset ("plate-control") or relative to just negative control
+samples ("vehicle-control"). The method of computing the differential can be
+either a robust z-score ("robust_z") or simply median normalization
+("median_norm").
+
 '''
-import sys
 import cmapPy.math.robust_zscore as robust_zscore
 import cmapPy.pandasGEXpress.GCToo as GCToo
 
-def calc_differential(gctoo, plate_control=True, group_field='pert_type', group_val='ctl_vehicle',
-                      func = robust_zscore.calc_zscore, pos_diff_thresh=10, neg_diff_thresh=-10):
+possible_diff_methods = ["robust_z", "median_norm"]
+
+
+def diff_gctoo(gctoo, plate_control=True, group_field='pert_type', group_val='ctl_vehicle',
+               diff_method="robust_z", upper_diff_thresh=10, lower_diff_thresh=-10):
+    ''' Converts a matrix of values (e.g. gene expression, viability, etc.)
+    into a matrix of differential values.
 
-    '''
     Args:
-    df (pandas df): data on which to perform diff
-    plate_control (bool): True means calculate differential using plate control. False means vehicle control.
+    df (pandas df): data to make diff_gctoo
+    plate_control (bool): True means calculate diff_gctoo using plate control.
+        False means vehicle control.
     group_field (string): Metadata field in which to find group_val
     group_val (string): Value in group_field that indicates use in vehicle control
-    func (function): Function to apply to data for calculating diff, eg. zscore, fold change
-    pos_diff_thresh (float): Maximum value for diff data
-    neg_diff_thresh: Minimum value for diff data
+    diff_method (string): Method of computing differential data; currently only
+        support either "robust_z" or "median_norm"
+    upper_diff_thresh (float): Maximum value for diff data
+    lower_diff_thresh (float): Minimum value for diff data
 
     Returns:
-    diff_gctoo (pandas df): Diff data!
+    out_gctoo (GCToo object): GCToo with differential data values
     '''
+    assert diff_method in possible_diff_methods, (
+        "possible_diff_methods: {}, diff_method: {}".format(
+            possible_diff_methods, diff_method))
+
+    # Compute median and MAD using all samples in the dataset
+    if plate_control:
+
+        # Compute differential data
+        if diff_method == "robust_z":
+            diff_data = robust_zscore.robust_zscore(gctoo.data_df)
+
+        elif diff_method == "median_norm":
+            medians = gctoo.data_df.median(axis=1)
+            diff_data = gctoo.data_df.subtract(medians, axis='index')
+
+    # Compute median and MAD from negative controls, rather than all samples
+    else:
+
+        assert group_field in gctoo.col_metadata_df.columns.values, (
+            "group_field {} not present in column metadata. " +
+            "gctoo.col_metadata_df.columns.values: {}").format(
+            group_field, gctoo.col_metadata_df.columns.values)
+
+        assert sum(gctoo.col_metadata_df[group_field] == group_val) > 0, (
+            "group_val {} not present in the {} column.").format(
+            group_val, group_field)
+
+        # Find negative control samples
+        neg_ctl_samples = gctoo.col_metadata_df.index[gctoo.col_metadata_df[group_field] == group_val]
+        neg_ctl_df = gctoo.data_df[neg_ctl_samples]
 
-    if plate_control == False:
-        # If using only a subset of the plate for control (usually vehicle control) extract this df
-        neg_dex = gctoo.col_metadata_df[gctoo.col_metadata_df[group_field] == group_val].index.tolist()
-        neg_df = gctoo.data_df[neg_dex]
-        diff_data = func(gctoo.data_df, neg_df)
+        # Compute differential data
+        if diff_method == "robust_z":
+            diff_data = robust_zscore.robust_zscore(gctoo.data_df, neg_ctl_df)
 
-    elif plate_control == True:
-        diff_data = func(gctoo.data_df)
+        elif diff_method == "median_norm":
+            medians = gctoo.data_df.median(axis=1)
+            diff_data = gctoo.data_df.subtract(medians, axis='index')
 
-    row_metadata_df = gctoo.row_metadata_df
+    # Threshold differential data before returning
+    diff_data = diff_data.clip(lower=lower_diff_thresh, upper=upper_diff_thresh)
 
-    # Threshold zscore data before returning
-    diff_data[diff_data < neg_diff_thresh] = neg_diff_thresh
-    diff_data[diff_data > pos_diff_thresh] = pos_diff_thresh
+    # Construct output GCToo object
+    out_gctoo = GCToo.GCToo(data_df=diff_data,
+                            row_metadata_df=gctoo.row_metadata_df,
+                            col_metadata_df=gctoo.col_metadata_df)
 
-    diff_gctoo = GCToo.GCToo(data_df=diff_data, row_metadata_df=row_metadata_df, col_metadata_df=gctoo.col_metadata_df)
+    return out_gctoo
 
-    return diff_gctoo
diff --git a/cmapPy/pandasGEXpress/tests/test_diff_gctoo.py b/cmapPy/pandasGEXpress/tests/test_diff_gctoo.py
@@ -1,45 +1,76 @@
 import unittest
-import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger
 import logging
 import pandas as pd
-import sys
+import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger
 import cmapPy.pandasGEXpress.GCToo as GCToo
-import cmapPy.pandasGEXpress.diff_gctoo as differential
+import cmapPy.pandasGEXpress.diff_gctoo as diff_gctoo
 
 logger = logging.getLogger(setup_logger.LOGGER_NAME)
 
-test_mat = pd.DataFrame({'A':[4,2,3], 'B': [2,8,6], 'C': [6,5,9], 'D': [5,2,1], 'E':[8,8,6], 'F': [7,6,6]})
-test_col_meta = pd.DataFrame({'pert_type': ['trt_cp', 'trt_cp', 'trt_cp', 'trt_cp','ctl_vehicle','ctl_vehicle'],
-                              'pert_iname': ['bort', 'bort', 'DMSO', 'DMSO', 'bort', 'bort']},
-                             index=['A', 'B', 'C', 'D', 'E', 'F'])
-test_gctoo = GCToo.GCToo(data_df=test_mat, col_metadata_df=test_col_meta, row_metadata_df=pd.DataFrame(index=range(0,3)))
+test_mat = pd.DataFrame({'A':[4,2,3], 'B': [2,8,6], 'C': [6,5,9],
+                         'D': [5,2,1], 'E':[8,8,6], 'F': [7,6,6]})
+test_col_meta = pd.DataFrame(
+    {'pert_type': ['trt_cp', 'trt_cp', 'trt_cp',
+                   'trt_cp', 'ctl_vehicle', 'ctl_vehicle'],
+     'pert_iname': ['bort', 'bort', 'DMSO', 'DMSO', 'bort', 'bort']},
+    index=['A', 'B', 'C', 'D', 'E', 'F'])
+test_gctoo = GCToo.GCToo(data_df=test_mat,
+                         col_metadata_df=test_col_meta)
+
 
 class TestDifferential(unittest.TestCase):
-    def test_differential(self):
-        pc_zscores = differential.calc_differential(test_gctoo, True)
-        vc_zscores1 = differential.calc_differential(test_gctoo, False)
-        vc_zscores2 = differential.calc_differential(test_gctoo, False, 'pert_iname', 'DMSO')
+    def test_diff_gctoo_pc(self):
+        pc_zscores = diff_gctoo.diff_gctoo(test_gctoo, plate_control=True, lower_diff_thresh=-2)
         self.assertTrue(pc_zscores.data_df.shape == (3, 6))
+
+        pd.util.testing.assert_frame_equal(pc_zscores.data_df, pd.DataFrame(
+            {'A': [-0.6745, -0.9443, -1.349],
+             'C': [0.2248, -0.1349, 1.349],
+             'B': [-1.5738, 0.6745, 0.0], 'E': [1.1242, 0.6745, 0.0],
+             'D': [-0.2248, -0.9443, -2], # last val should be -2 bc of thresholding
+             'F': [0.6745, 0.1349, 0.0]}))
+
+        # test diff_method assertion
+        with self.assertRaises(AssertionError) as e:
+            diff_gctoo.diff_gctoo(test_gctoo, plate_control=True, diff_method="robust_zs")
+        self.assertIn("diff_method: robust_zs", str(e.exception))
+
+        # test median norm
+        pc_median_normed_df = diff_gctoo.diff_gctoo(test_gctoo, diff_method="median_norm")
+        self.assertEqual(pc_median_normed_df.data_df.iloc[0, 0], -1.5)
+        self.assertEqual(pc_median_normed_df.data_df.loc[2, "B"], 0)
+
+    def test_diff_gctoo_vc(self):
+        vc_zscores1 = diff_gctoo.diff_gctoo(test_gctoo, plate_control=False)
+        vc_zscores2 = diff_gctoo.diff_gctoo(test_gctoo, plate_control=False,
+                                            group_field='pert_iname',
+                                            group_val='DMSO')
         self.assertTrue(vc_zscores1.data_df.shape == (3, 6))
         self.assertTrue(vc_zscores2.data_df.shape == (3, 6))
-        pd.util.testing.assert_frame_equal(pc_zscores.data_df, pd.DataFrame({'A': [-0.6745, -0.9443, -1.349],
-                                                            'C': [0.2248, -0.1349, 1.349],
-                                                            'B': [-1.5738, 0.6745, 0.0], 'E': [1.1242, 0.6745, 0.0],
-                                                            'D': [-0.2248, -0.9443, -2.2483],
-                                                            'F': [0.6745, 0.1349, 0.0]}, index=[0,1,2]))
-
-        pd.util.testing.assert_frame_equal(vc_zscores1.data_df, pd.DataFrame({'A': [-4.7214, -3.3725, -10.0],
-                                                               'C': [-2.0235, -1.349, 10.0],
-                                                               'B': [-7.4194, 0.6745, 0.0],
-                                                               'E': [0.6745, 0.6745, 0.0],
-                                                               'D': [-3.3725, -3.3725, -10.0],
-                                                               'F': [-0.6745, -0.6745, 0.0]},
-                                                                index=[0,1,2]))
-
-        pd.util.testing.assert_frame_equal(vc_zscores2.data_df, pd.DataFrame({'A': [-2.0235, -0.6745, -0.3372],
-                                                               'C': [0.6745, 0.6745, 0.6745],
-                                                               'B': [-4.7214, 2.0235, 0.1686],
-                                                               'E': [3.3725, 2.0235, 0.1686],
-                                                               'D': [-0.6745, -0.6745, -0.6745],
-                                                               'F': [2.0235, 1.1242, 0.1686]},
-                                                              index=[0,1,2]))
+
+        pd.util.testing.assert_frame_equal(vc_zscores1.data_df, pd.DataFrame(
+            {'A': [-4.7214, -3.3725, -10.0], # check for thresholding
+             'C': [-2.0235, -1.349, 10.0],
+             'B': [-7.4194, 0.6745, 0.0],
+             'E': [0.6745, 0.6745, 0.0],
+             'D': [-3.3725, -3.3725, -10.0],
+             'F': [-0.6745, -0.6745, 0.0]}))
+
+        pd.util.testing.assert_frame_equal(vc_zscores2.data_df, pd.DataFrame(
+            {'A': [-2.0235, -0.6745, -0.3372],
+             'C': [0.6745, 0.6745, 0.6745],
+             'B': [-4.7214, 2.0235, 0.1686],
+             'E': [3.3725, 2.0235, 0.1686],
+             'D': [-0.6745, -0.6745, -0.6745],
+             'F': [2.0235, 1.1242, 0.1686]}))
+
+        # test group_val assertion
+        with self.assertRaises(AssertionError) as e:
+            diff_gctoo.diff_gctoo(test_gctoo, plate_control=False, group_val="dmso")
+        self.assertIn("dmso not present", str(e.exception))
+
+
+if __name__ == "__main__":
+    setup_logger.setup(verbose=True)
+    unittest.main()
+
