Linkage funcs extracted and cached.

This only works for input data with hashable values for now. This gives
a tremendous speed boost at the cost of memory. The memory cost can
however be improved by removing "clusters inside other clusters".

Author: exhuma

cluster/linkage.py

@@ -0,0 +1,101 @@
+from __future__ import division
+from functools import wraps
+
+
+def cached(fun):
+    """
+    memoizing decorator for linkage functions.
+
+    Parameters have been hardcoded (no ``*args``, ``**kwargs`` magic), because,
+    the way this is coded (interchangingly using sets and frozensets) is true
+    for this specific case. For other cases that is not necessarily guaranteed.
+    """
+    return fun
+
+    _cache = {}
+
+    @wraps(fun)
+    def newfun(a, b, distance_function):
+        frozen_a = frozenset(a)
+        frozen_b = frozenset(b)
+        if (frozen_a, frozen_b) not in _cache:
+            result = fun(a, b, distance_function)
+            _cache[(frozen_a, frozen_b)] = result
+        return _cache[(frozen_a, frozen_b)]
+    return newfun
+
+
+@cached
+def single(a, b, distance_function):
+    """
+    Given two collections ``a`` and ``b``, this will return the distance of the
+    points which are closest togetger.  ``distance_function`` is used to
+    determine the distance between two elements.
+
+    Example::
+
+        >>> single([1, 2], [3, 4], lambda x, y: abs(x-y))
+        1  # (distance between 2 and 3)
+    """
+    left_a, right_a = min(a), max(a)
+    left_b, right_b = min(b), max(b)
+    result = min(distance_function(left_a, right_b),
+                 distance_function(left_b, right_a))
+    return result
+
+
+@cached
+def complete(a, b, distance_function):
+    """
+    Given two collections ``a`` and ``b``, this will return the distance of the
+    points which are farthest apart.  ``distance_function`` is used to determine
+    the distance between two elements.
+
+    Example::
+
+        >>> single([1, 2], [3, 4], lambda x, y: abs(x-y))
+        3  # (distance between 1 and 4)
+    """
+    left_a, right_a = min(a), max(a)
+    left_b, right_b = min(b), max(b)
+    result = max(distance_function(left_a, right_b),
+                 distance_function(left_b, right_a))
+    return result
+
+
+@cached
+def average(a, b, distance_function):
+    """
+    Given two collections ``a`` and ``b``, this will return the mean of all
+    distances. ``distance_function`` is used to determine the distance between
+    two elements.
+
+    Example::
+
+        >>> single([1, 2], [3, 100], lambda x, y: abs(x-y))
+        26
+    """
+    distances = [distance_function(x, y)
+                 for x in a for y in b]
+    return sum(distances) / len(distances)
+
+
+@cached
+def uclus(a, b, distance_function):
+    """
+    Given two collections ``a`` and ``b``, this will return the *median* of all
+    distances. ``distance_function`` is used to determine the distance between
+    two elements.
+
+    Example::
+
+        >>> single([1, 2], [3, 100], lambda x, y: abs(x-y))
+        2.5
+    """
+    distances = sorted([distance_function(x, y)
+                        for x in a for y in b])
+    midpoint, rest = len(distances) // 2, len(distances) % 2
+    if not rest:
+        return sum(distances[midpoint-1:midpoint+1]) / 2
+    else:
+        return distances[midpoint]

cluster/method/hierarchical.py

@@ -15,12 +15,14 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 #
 
+from functools import partial
 import logging
 
 from cluster.cluster import Cluster
 from cluster.matrix import Matrix
 from cluster.method.base import BaseClusterMethod
 from cluster.util import median, mean, fullyflatten
+from cluster.linkage import single, complete, average, uclus
 
 
 logger = logging.getLogger(__name__)
@@ -58,7 +60,7 @@ class HierarchicalClustering(BaseClusterMethod):
 
     def __init__(self, data, distance_function, linkage=None, num_processes=1):
         if not linkage:
-            linkage = 'single'
+            linkage = single
         logger.info("Initializing HierarchicalClustering object with linkage "
                     "method %s", linkage)
         BaseClusterMethod.__init__(self, sorted(data), distance_function)
@@ -74,131 +76,19 @@ def set_linkage_method(self, method):
             ``'single'``, ``'complete'``, ``'average'`` or ``'uclus'``.
         """
         if method == 'single':
-            self.linkage = self.single_linkage_distance
+            self.linkage = single
         elif method == 'complete':
-            self.linkage = self.complete_linkage_distance
+            self.linkage = complete
         elif method == 'average':
-            self.linkage = self.average_linkage_distance
+            self.linkage = average
         elif method == 'uclus':
-            self.linkage = self.uclus_distance
+            self.linkage = uclus
+        elif hasattr(method, '__call__'):
+            self.linkage = method
         else:
             raise ValueError('distance method must be one of single, '
                              'complete, average of uclus')
 
-    def uclus_distance(self, x, y):
-        """
-        The method to determine the distance between one cluster an another
-        item/cluster. The distance equals to the *average* (median) distance
-        from any member of one cluster to any member of the other cluster.
-
-        :param x: first cluster/item.
-        :param y: second cluster/item.
-        """
-        # create a flat list of all the items in <x>
-        if not isinstance(x, Cluster):
-            x = [x]
-        else:
-            x = fullyflatten(x.items)
-
-        # create a flat list of all the items in <y>
-        if not isinstance(y, Cluster):
-            y = [y]
-        else:
-            y = fullyflatten(y.items)
-
-        distances = []
-        for k in x:
-            for l in y:
-                distances.append(self.distance(k, l))
-        return median(distances)
-
-    def average_linkage_distance(self, x, y):
-        """
-        The method to determine the distance between one cluster an another
-        item/cluster. The distance equals to the *average* (mean) distance
-        from any member of one cluster to any member of the other cluster.
-
-        :param x: first cluster/item.
-        :param y: second cluster/item.
-        """
-        # create a flat list of all the items in <x>
-        if not isinstance(x, Cluster):
-            x = [x]
-        else:
-            x = fullyflatten(x.items)
-
-        # create a flat list of all the items in <y>
-        if not isinstance(y, Cluster):
-            y = [y]
-        else:
-            y = fullyflatten(y.items)
-
-        distances = []
-        for k in x:
-            for l in y:
-                distances.append(self.distance(k, l))
-        return mean(distances)
-
-    def complete_linkage_distance(self, x, y):
-        """
-        The method to determine the distance between one cluster an another
-        item/cluster. The distance equals to the *longest* distance from any
-        member of one cluster to any member of the other cluster.
-
-        :param x: first cluster/item.
-        :param y: second cluster/item.
-        """
-
-        # create a flat list of all the items in <x>
-        if not isinstance(x, Cluster):
-            x = [x]
-        else:
-            x = fullyflatten(x.items)
-
-        # create a flat list of all the items in <y>
-        if not isinstance(y, Cluster):
-            y = [y]
-        else:
-            y = fullyflatten(y.items)
-
-        # retrieve the minimum distance (single-linkage)
-        maxdist = self.distance(x[0], y[0])
-        for k in x:
-            for l in y:
-                maxdist = max(maxdist, self.distance(k, l))
-
-        return maxdist
-
-    def single_linkage_distance(self, x, y):
-        """
-        The method to determine the distance between one cluster an another
-        item/cluster. The distance equals to the *shortest* distance from any
-        member of one cluster to any member of the other cluster.
-
-        :param x: first cluster/item.
-        :param y: second cluster/item.
-        """
-
-        # create a flat list of all the items in <x>
-        if not isinstance(x, Cluster):
-            x = [x]
-        else:
-            x = fullyflatten(x.items)
-
-        # create a flat list of all the items in <y>
-        if not isinstance(y, Cluster):
-            y = [y]
-        else:
-            y = fullyflatten(y.items)
-
-        # retrieve the minimum distance (single-linkage)
-        mindist = self.distance(x[0], y[0])
-        for k in x:
-            for l in y:
-                mindist = min(mindist, self.distance(k, l))
-
-        return mindist
-
     def cluster(self, matrix=None, level=None, sequence=None):
         """
         Perform hierarchical clustering.
@@ -217,10 +107,11 @@ def cluster(self, matrix=None, level=None, sequence=None):
             matrix = []
 
         # if the matrix only has two rows left, we are done
+        linkage = partial(self.linkage, distance_function=self.distance)
         while len(matrix) > 2 or matrix == []:
 
             item_item_matrix = Matrix(self._data,
-                                      self.linkage,
+                                      linkage,
                                       True,
                                       0)
             item_item_matrix.genmatrix(self.num_processes)