Merge branch 'master' into math

Author: lev

.gitignore

@@ -1,2 +1,5 @@
 __pycache__/
 *.pyc
+cmapPy.egg-info/
+.vscode
+.gitignore

.travis.yml

@@ -13,6 +13,7 @@ install:
 env: 
   - TEST_DIR=cmapPy/pandasGEXpress/tests
   - TEST_DIR=cmapPy/set_io/tests
+  - TEST_DIR=cmapPy/math/tests
 
 # run all tests in each test dir 
 script: cd $TEST_DIR && python -m unittest discover -p "test_*.py"

README.rst

@@ -17,7 +17,7 @@
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 **Connectivity Map, Broad Institute of MIT and Harvard**
 
-Documentation: `<http://cmappy.readthedocs.io/en/latest/>`_
+Documentation: `<https://clue.io/cmapPy/index.html>`_
 
 For questions/problems, please add an issue (that includes code/files that reproduce your problem) to the repository. 
 
@@ -29,6 +29,7 @@ We welcome contributors! For your pull requests, please include the following:
 * Sample code/file that reproducibly causes the bug/issue
 * Documented code providing fix
 * Unit tests evaluating added/modified methods. 
+ 
 
 Citation
 ====================

cmapPy/clue_api_client/__init__.py

@@ -1 +0,0 @@
-from .clue_api_client import ClueApiClient

cmapPy/math/fast_corr.py

@@ -0,0 +1,57 @@
+import logging
+import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger
+import numpy
+import cmapPy.math.fast_cov as fast_cov
+import pandas
+
+
+logger = logging.getLogger(setup_logger.LOGGER_NAME)
+
+
+def fast_corr(x, y=None):
+    """calculate the pearson correlation matrix for the columns of x (MxN), or optionally, the correlaton matrix between x and y (OxP).
+    In the language of statistics the columns are the variables and the rows are the observations.
+
+    Args:
+        x (numpy array-like) MxN in shape
+        y (optional, numpy array-like) OxP in shape
+
+        returns (numpy array-like) array of the covariance values
+            for defaults (y=None), shape is NxN
+            if y is provied, shape is NxP
+    """
+    if y is None:
+        y = x
+
+    cov_mat = fast_cov.fast_cov(x, y)
+    
+    std_x = numpy.std(x, axis=0, ddof=1)
+    std_y = numpy.std(y, axis=0, ddof=1)
+    
+    std_outer = numpy.outer(std_x, std_y)
+
+    return cov_mat / std_outer
+
+
+def fast_spearman(x, y=None):
+    """calculate the spearnab correlation matrix for the columns of x (MxN), or optionally, the spearmancorrelaton matrix between x and y (OxP).
+    In the language of statistics the columns are the variables and the rows are the observations.
+
+    Args:
+        x (numpy array-like) MxN in shape
+        y (optional, numpy array-like) OxP in shape
+
+        returns:
+            (numpy array-like) array of the covariance values
+                for defaults (y=None), shape is NxN
+                if y is provied, shape is NxP
+    """
+    logger.debug("x.shape:  {}".format(x.shape))
+    if hasattr(y, "shape"):  
+        logger.debug("y.shape:  {}".format(y.shape))
+
+    x_ranks = pandas.DataFrame(x).rank(method="average").values
+    logger.debug("some min and max ranks of x_ranks:\n{}\n{}".format(numpy.min(x_ranks[:10], axis=0), numpy.max(x_ranks[:10], axis=0)))
+    y_ranks = pandas.DataFrame(y).rank(method="average").values if y is not None else None
+
+    return fast_corr(x_ranks, y_ranks)

cmapPy/math/fast_cov.py

@@ -0,0 +1,60 @@
+import logging
+import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger
+import numpy
+
+
+logger = logging.getLogger(setup_logger.LOGGER_NAME)
+
+
+def fast_cov(x, y=None):
+    """calculate the covariance matrix for the columns of x (MxN), or optionally, the covariance matrix between the
+    columns of x and and the columns of y (MxP).  (In the language of statistics, the columns are variables, the rows
+    are observations).
+
+    Args:
+        x (numpy array-like) MxN in shape
+        y (numpy array-like) MxP in shape
+
+        returns (numpy array-like) array of the covariance values
+            for defaults (y=None), shape is NxN
+            if y is provided, shape is NxP
+    """
+    validate_x_y(x, y)
+
+    if y is None:
+        y = x
+
+    mean_x = numpy.mean(x, axis=0)    
+    mean_y = numpy.mean(y, axis=0)
+
+    mean_centered_x = x - mean_x
+    mean_centered_y = y - mean_y
+    
+    dotprod = numpy.dot(mean_centered_x.T, mean_centered_y)
+
+    denom = x.shape[0] - 1
+
+    return dotprod / denom
+
+
+def validate_x_y(x, y):
+    error_msg = ""
+
+    if not hasattr(x, "shape"):
+        error_msg += "x needs to be numpy array-like but it does not have \"shape\" attribute - type(x):  {}\n".format(type(x))
+    
+    if y is not None:
+        if not hasattr(y, "shape"):
+            error_msg += "y needs to be numpy array-like but it does not have \"shape\" attribute - type(y):  {}\n".format(type(y))
+        elif x.shape[0] != y.shape[0]:
+            error_msg += "the number of rows in the x and y matrices must be the same".format(x.shape, y.shape)
+
+    if error_msg != "":
+        raise CmapPyMathFastCovInvalidInputXY(error_msg)
+
+
+class CmapPyMathFastCovInvalidInputAxis(Exception):
+    pass
+
+class CmapPyMathFastCovInvalidInputXY(Exception):
+    pass

cmapPy/math/tests/test_fast_corr.py

@@ -0,0 +1,153 @@
+import unittest
+import logging
+import cmapPy.pandasGEXpress.setup_GCToo_logger as setup_logger
+import cmapPy.math.fast_corr as fast_corr
+import numpy
+import pandas
+
+
+logger = logging.getLogger(setup_logger.LOGGER_NAME)
+
+num_iterations_functional_tests = 20
+max_dimension_functional_tests = 10
+multiplier_max_functional_tests = 100
+
+
+class TestFastCorr(unittest.TestCase):
+    @staticmethod
+    def build_standard_x_y():
+        x = numpy.array([[1,7,2], [5,3,11]])
+        logger.debug("x:  {}".format(x))
+        logger.debug("x.shape:  {}".format(x.shape))
+
+        y = numpy.array([[13, 17, 19], [23, 31, 29]])
+        logger.debug("y:  {}".format(y))
+        logger.debug("y.shape:  {}".format(y.shape))
+
+        return x, y
+
+    def test_fast_corr_just_x(self):
+        logger.debug("*************happy path just x")
+        x, _ = TestFastCorr.build_standard_x_y()
+
+        ex = numpy.corrcoef(x, rowvar=False)
+        logger.debug("expected ex:  {}".format(ex))
+
+        r = fast_corr.fast_corr(x)
+        logger.debug("r:  {}".format(r))
+        
+        self.assertTrue(numpy.allclose(ex, r))
+
+        #happy path just x, other direction
+        ex = numpy.corrcoef(x, rowvar=True)
+        logger.debug("happy path just x, other direction, expected ex:  {}".format(ex))
+        r = fast_corr.fast_corr(x.T)
+        logger.debug("r:  {}".format(r))
+        self.assertTrue(numpy.allclose(ex, r))
+
+    def test_fast_corr_x_and_y(self):
+        logger.debug("*************happy path x and y")
+        x, y = TestFastCorr.build_standard_x_y()
+
+        combined = numpy.hstack([x, y])
+        logger.debug("combined:  {}".format(combined))
+        logger.debug("combined.shape:  {}".format(combined.shape))
+
+        off_diag_ind = combined.shape[1] / 2
+
+        raw_ex = numpy.corrcoef(combined, rowvar=False)
+        logger.debug("raw expected produced from numpy.cov on full combined - raw_ex:  {}".format(raw_ex))
+        ex = raw_ex[:off_diag_ind, off_diag_ind:]
+        logger.debug("expected ex:  {}".format(ex))
+
+        r = fast_corr.fast_corr(x, y)
+        logger.debug("r:  {}".format(r))
+        self.assertTrue(numpy.allclose(ex, r))
+
+        #happy path x and y, other direction
+        combined = numpy.hstack([x.T, y.T])
+        logger.debug("*************happy path x and y, other direction - combined:  {}".format(combined))
+        logger.debug("combined.shape:  {}".format(combined.shape))
+
+        off_diag_ind = combined.shape[1] / 2
+
+        raw_ex = numpy.corrcoef(combined, rowvar=False)
+        logger.debug("raw expected produced from numpy.cov on full combined - raw_ex:  {}".format(raw_ex))
+        ex = raw_ex[:off_diag_ind, off_diag_ind:]
+        logger.debug("expected ex:  {}".format(ex))
+
+        r = fast_corr.fast_corr(x.T, y.T)
+        logger.debug("r:  {}".format(r))
+        self.assertTrue(numpy.allclose(ex, r))
+    
+    def test_fast_corr_x_and_y_different_shapes(self):
+        logger.debug("*************happy path x and y different shapes")
+        x, _ = TestFastCorr.build_standard_x_y()
+        y = numpy.array([[13, 17, 19, 41, 23], [23, 29, 31, 37, 43]])
+        logger.debug("y.shape:  {}".format(y.shape))
+        logger.debug("y:\n{}".format(y))
+
+        combined = numpy.hstack([x, y])
+        logger.debug("combined:  {}".format(combined))
+        logger.debug("combined.shape:  {}".format(combined.shape))
+
+        raw_ex = numpy.corrcoef(combined, rowvar=False)
+        logger.debug("raw expected produced from numpy.cov on full combined - raw_ex:  {}".format(raw_ex))
+        logger.debug("raw_ex.shape:  {}".format(raw_ex.shape))
+
+        ex = raw_ex[:x.shape[1], -y.shape[1]:]
+        logger.debug("expected ex:  {}".format(ex))
+        logger.debug("ex.shape:  {}".format(ex.shape))
+
+        r = fast_corr.fast_corr(x, y)
+        logger.debug("r:  {}".format(r))
+        self.assertTrue(numpy.allclose(ex, r))
+
+    def test_fast_corr_functional(self):
+        logger.debug("*************happy path functional test using randomly generated matrices")
+
+        for i in xrange(num_iterations_functional_tests):
+            #the dimension containing the observations must have at least size 2
+            x_shape = [numpy.random.randint(2, max_dimension_functional_tests),
+                numpy.random.randint(1, max_dimension_functional_tests)]
+            logger.debug("x_shape:  {}".format(x_shape))
+
+            x = numpy.random.rand(x_shape[0], x_shape[1]) * numpy.random.randint(1, multiplier_max_functional_tests, size=1)
+            logger.debug("x:\n{}".format(x))
+
+            y_other_shape = numpy.random.randint(1, max_dimension_functional_tests, size=1)[0]
+            y_shape = (x_shape[0], y_other_shape)
+            logger.debug("y_shape:  {}".format(y_shape))
+            y = numpy.random.rand(y_shape[0], y_shape[1]) * numpy.random.randint(1, multiplier_max_functional_tests, size=1)
+            logger.debug("y:\n{}".format(y))
+
+            combined = numpy.hstack([x, y])
+
+            raw_ex = numpy.corrcoef(combined, rowvar=False)
+            logger.debug("raw_ex.shape:  {}".format(raw_ex.shape))
+
+            ex = raw_ex[:x.shape[1], -y.shape[1]:]
+            logger.debug("ex:\n{}".format(ex))
+            logger.debug("ex.shape:  {}".format(ex.shape))
+
+            r = fast_corr.fast_corr(x, y)
+            logger.debug("r:\n{}".format(r))
+            logger.debug("r.shape:  {}".format(r.shape))
+
+            self.assertTrue(numpy.allclose(ex, r))
+
+    def test_fast_spearman(self):
+        x, y = TestFastCorr.build_standard_x_y()
+
+        ex = numpy.array([[1.0, 1.0, 1.0], [-1.0, -1.0, -1.0], [1.0, 1.0, 1.0]])
+
+        r = fast_corr.fast_spearman(x, y)
+        logger.debug("r:  {}".format(r))
+
+        self.assertTrue(numpy.allclose(ex, r))
+
+
+if __name__ == "__main__":
+    setup_logger.setup(verbose=True)
+
+    unittest.main()