Further coverage improvements

Author: lmcinnes

hdbscan/hdbscan_.py

@@ -18,14 +18,7 @@
 from sklearn.externals.joblib import Memory
 from sklearn.externals import six
 from warnings import warn
-
-#Try and work around older sklearn api
-try:
-    from sklearn.utils import check_array
-except ImportError:
-    from sklearn.utils import check_arrays
-
-    check_array = check_arrays
+from sklearn.utils import check_array
 
 from scipy.sparse import csgraph
 
@@ -527,7 +520,7 @@ def hdbscan(X, min_cluster_size=5, min_samples=None, alpha=1.0,
             if X.shape[1] > 60:
                 (single_linkage_tree,
                  result_min_span_tree) = \
-                    memory.cache(_hdbscan_prims_kdtree)(X, min_samples, alpha,
+                    memory.cache(_hdbscan_prims_balltree)(X, min_samples, alpha,
                                                         metric, p, leaf_size,
                                                         gen_min_span_tree, **kwargs)
             else:
@@ -764,8 +757,12 @@ def outlier_scores_(self):
         if self._outlier_scores is not None:
             return self._outlier_scores
         else:
-            self._outlier_scores = outlier_scores(self._condensed_tree)
-            return self._outlier_scores
+            if self._condensed_tree is not None:
+                self._outlier_scores = outlier_scores(self._condensed_tree)
+                return self._outlier_scores
+            else:
+                warn('No condensed tree was generated; try running fit first.')
+                return None
 
     @property
     def condensed_tree_(self):
@@ -794,7 +791,7 @@ def minimum_spanning_tree_(self):
                     'No minimum spanning tree will be provided without raw data.')
                 return None
         else:
-            warn('No minimum spanning tree was generated. \n'
-                 'This may be due to optimized algorithm variations that skip\n'
-                 'explicit generation of the spanning tree.')
+            warn('No minimum spanning tree was generated.'
+                'This may be due to optimized algorithm variations that skip'
+                'explicit generation of the spanning tree.')
             return None

hdbscan/robust_single_linkage_.py

@@ -14,13 +14,7 @@
 
 from sklearn.externals.joblib import Memory
 from sklearn.externals import six
-
-try:
-    from sklearn.utils import check_array
-except ImportError:
-    from sklearn.utils import check_arrays
-
-    check_array = check_arrays
+from sklearn.utils import check_array
 
 from ._hdbscan_linkage import mst_linkage_core, mst_linkage_core_vector, label
 from ._hdbscan_boruvka import KDTreeBoruvkaAlgorithm, BallTreeBoruvkaAlgorithm
@@ -116,6 +110,12 @@ def _rsl_prims_balltree(X, k=5, alpha=1.4142135623730951, metric='minkowski', p=
 
 def _rsl_boruvka_kdtree(X, k=5, alpha=1.0,
                         metric='minkowski', p=2, leaf_size=40):
+    if metric == 'minkowski':
+        if p is None:
+            raise TypeError('Minkowski metric given but no p value supplied!')
+        if p < 0:
+            raise ValueError('Minkowski metric with negative p value is not defined!')
+
     dim = X.shape[0]
     min_samples = min(dim - 1, k)
 
@@ -132,6 +132,12 @@ def _rsl_boruvka_kdtree(X, k=5, alpha=1.0,
 
 def _rsl_boruvka_balltree(X, k=5, alpha=1.0,
                           metric='minkowski', p=2, leaf_size=40):
+    if metric == 'minkowski':
+        if p is None:
+            raise TypeError('Minkowski metric given but no p value supplied!')
+        if p < 0:
+            raise ValueError('Minkowski metric with negative p value is not defined!')
+
     dim = X.shape[0]
     min_samples = min(dim - 1, k)
 
@@ -148,7 +154,7 @@ def _rsl_boruvka_balltree(X, k=5, alpha=1.0,
 
 def robust_single_linkage(X, cut, k=5, alpha=1.4142135623730951,
                           gamma=5, metric='minkowski', p=2, algorithm='best',
-                          memory=Memory(cachedir=None, verbose=0)):
+                          memory=Memory(cachedir=None, verbose=0), leaf_size=40):
     """Perform robust single linkage clustering from a vector array
     or distance matrix.
 
@@ -202,6 +208,10 @@ def robust_single_linkage(X, cut, k=5, alpha=1.4142135623730951,
         By default, no caching is done. If a string is given, it is the
         path to the caching directory.
 
+    leaf_size : int, optional
+        Leaf size for trees responsible for fast nearest
+        neighbour queries. (default 40)
+
     Returns
     -------
     labels : array [n_samples]
@@ -229,6 +239,9 @@ def robust_single_linkage(X, cut, k=5, alpha=1.4142135623730951,
     if type(gamma) is not int or gamma < 1:
         raise ValueError('gamma must be an integer greater than zero!')
 
+    if type(leaf_size) is not int or leaf_size < 1:
+        raise ValueError('Leaf size must be at least one!')
+
     X = check_array(X, accept_sparse='csr')
     if isinstance(memory, six.string_types):
         memory = Memory(cachedir=memory, verbose=0)
@@ -245,10 +258,10 @@ def robust_single_linkage(X, cut, k=5, alpha=1.4142135623730951,
                 memory.cache(_rsl_prims_balltree)(X, k, alpha, metric, p)
         elif algorithm == 'boruvka_kdtree':
             single_linkage_tree = \
-                memory.cache(_rsl_boruvka_kdtree)(X, k, alpha, metric, p)
+                memory.cache(_rsl_boruvka_kdtree)(X, k, alpha, metric, p, leaf_size)
         elif algorithm == 'boruvka_balltree':
             single_linkage_tree = \
-                memory.cache(_rsl_boruvka_balltree)(X, k, alpha, metric, p)
+                memory.cache(_rsl_boruvka_balltree)(X, k, alpha, metric, p, leaf_size)
         else:
             raise TypeError('Unknown algorithm type %s specified' % algorithm)
     else:

hdbscan/tests/test_hdbscan.py

@@ -30,6 +30,8 @@
 
 from sklearn import datasets
 
+import warnings
+
 n_clusters = 3
 # X = generate_clustered_data(n_clusters=n_clusters, n_samples_per_cluster=50)
 X, y = make_blobs(n_samples=50, random_state=0)
@@ -110,7 +112,7 @@ def test_hdbscan_sparse_distance_matrix():
     n_clusters_1 = len(set(labels)) - int(-1 in labels)  # ignore noise
     assert_equal(n_clusters_1, n_clusters)
 
-    labels = HDBSCAN(metric="precomputed").fit(D).labels_
+    labels = HDBSCAN(metric="precomputed", gen_min_span_tree=True).fit(D).labels_
     n_clusters_2 = len(set(labels)) - int(-1 in labels)
     assert_equal(n_clusters_2, n_clusters)
 
@@ -124,6 +126,89 @@ def test_hdbscan_feature_vector():
     n_clusters_2 = len(set(labels)) - int(-1 in labels)
     assert_equal(n_clusters_2, n_clusters)
 
+def test_hdbscan_prims_kdtree():
+    labels, p, persist, ctree, ltree, mtree = hdbscan(X, algorithm='prims_kdtree')
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(algorithm='prims_kdtree', gen_min_span_tree=True).fit(X).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+    assert_raises(ValueError,
+                  hdbscan,
+                  X,
+                  algorithm='prims_kdtree',
+                  metric='russelrao')
+
+def test_hdbscan_prims_balltree():
+    labels, p, persist, ctree, ltree, mtree = hdbscan(X, algorithm='prims_balltree')
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(algorithm='prims_balltree', gen_min_span_tree=True).fit(X).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+    assert_raises(ValueError,
+                  hdbscan,
+                  X,
+                  algorithm='prims_balltree',
+                  metric='cosine')
+
+def test_hdbscan_boruvka_kdtree():
+    labels, p, persist, ctree, ltree, mtree = hdbscan(X, algorithm='boruvka_kdtree')
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(algorithm='boruvka_kdtree', gen_min_span_tree=True).fit(X).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+    assert_raises(ValueError,
+                  hdbscan,
+                  X,
+                  algorithm='boruvka_kdtree',
+                  metric='russelrao')
+
+def test_hdbscan_boruvka_balltree():
+    labels, p, persist, ctree, ltree, mtree = hdbscan(X, algorithm='boruvka_balltree')
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(algorithm='boruvka_balltree', gen_min_span_tree=True).fit(X).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+    assert_raises(ValueError,
+                  hdbscan,
+                  X,
+                  algorithm='boruvka_balltree',
+                  metric='cosine')
+
+
+def test_hdbscan_high_dimensional():
+    H, y = make_blobs(n_samples=50, random_state=0, n_features=64)
+    # H, y = shuffle(X, y, random_state=7)
+    H = StandardScaler().fit_transform(H)
+    labels, p, persist, ctree, ltree, mtree = hdbscan(H)
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(algorithm='best', metric='seuclidean', V=np.ones(H.shape[1])).fit(H).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+def test_hdbscan_best_balltree_metric():
+    labels, p, persist, ctree, ltree, mtree = hdbscan(X, metric='seuclidean', V=np.ones(X.shape[1]))
+    n_clusters_1 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_1, n_clusters)
+
+    labels = HDBSCAN(metric='seuclidean', V=np.ones(X.shape[1])).fit(X).labels_
+    n_clusters_2 = len(set(labels)) - int(-1 in labels)
+    assert_equal(n_clusters_2, n_clusters)
+
+
 def test_hdbscan_no_clusters():
     labels, p, persist, ctree, ltree, mtree = hdbscan(X, min_cluster_size=len(X)+1)
     n_clusters_1 = len(set(labels)) - int(-1 in labels)
@@ -212,6 +297,42 @@ def test_tree_networkx_output_formats():
     if_networkx(clusterer.single_linkage_tree_.to_networkx)()
     if_networkx(clusterer.minimum_spanning_tree_.to_networkx)()
 
+def test_hdbscan_outliers():
+    clusterer = HDBSCAN(gen_min_span_tree=True).fit(X)
+    scores = clusterer.outlier_scores_
+    assert(scores is not None)
+
+def test_hdbscan_unavailable_attributes():
+    clusterer = HDBSCAN(gen_min_span_tree=False)
+    with warnings.catch_warnings(record=True) as w:
+        tree = clusterer.condensed_tree_
+        assert(len(w) > 0)
+        assert(tree is None)
+    with warnings.catch_warnings(record=True) as w:
+        tree = clusterer.single_linkage_tree_
+        assert(len(w) > 0)
+        assert(tree is None)
+    with warnings.catch_warnings(record=True) as w:
+        scores = clusterer.outlier_scores_
+        assert(len(w) > 0)
+        assert(scores is None)
+    with warnings.catch_warnings(record=True) as w:
+        tree = clusterer.minimum_spanning_tree_
+        assert(len(w) > 0)
+        assert(tree is None)
+
+def test_hdbscan_min_span_tree_availability():
+    clusterer = HDBSCAN().fit(X)
+    tree = clusterer.minimum_spanning_tree_
+    assert(tree is None)
+    D = distance.squareform(distance.pdist(X))
+    D /= np.max(D)
+    HDBSCAN(metric='precomputed').fit(D)
+    tree = clusterer.minimum_spanning_tree_
+    assert(tree is None)
+
+
+
 def test_hdbscan_badargs():
     assert_raises(ValueError,
                   hdbscan,
@@ -246,6 +367,18 @@ def test_hdbscan_badargs():
     assert_raises(ValueError,
                   hdbscan,
                   X, metric='minkowski', p=-1, algorithm='boruvka_balltree')
+    assert_raises(ValueError,
+                  hdbscan,
+                  X, metric='precomputed', algorithm='boruvka_kdtree')
+    assert_raises(ValueError,
+                  hdbscan,
+                  X, metric='precomputed', algorithm='prims_kdtree')
+    assert_raises(ValueError,
+                  hdbscan,
+                  X, metric='precomputed', algorithm='prims_balltree')
+    assert_raises(ValueError,
+                  hdbscan,
+                  X, metric='precomputed',algorithm='boruvka_balltree')
     assert_raises(ValueError,
                   hdbscan,
                   X, alpha=-1)
@@ -258,6 +391,9 @@ def test_hdbscan_badargs():
     assert_raises(TypeError,
                   hdbscan,
                   X, metric='minkowski', p=None)
+    assert_raises(ValueError,
+                  hdbscan,
+                  X, leaf_size=0)
 
 def test_hdbscan_sparse():
 

hdbscan/tests/test_rsl.py

@@ -96,6 +96,65 @@ def test_rsl_boruvka_balltree():
 #     n_clusters_2 = len(set(labels)) - int(-1 in labels)
 #     assert_equal(n_clusters_2, n_clusters)
 
+def test_rsl_badargs():
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  'fail', 0.4)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  None, 0.4)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, k='fail')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, k=-1)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='imperial')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric=None)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='minkowski', p=-1)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='minkowski', p=-1, algorithm='prims_kdtree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='minkowski', p=-1, algorithm='prims_balltree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='minkowski', p=-1, algorithm='boruvka_balltree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='precomputed', algorithm='boruvka_kdtree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='precomputed', algorithm='prims_kdtree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='precomputed', algorithm='prims_balltree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, metric='precomputed',algorithm='boruvka_balltree')
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, alpha=-1)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, alpha='fail')
+    assert_raises(Exception,
+                  robust_single_linkage,
+                  X, 0.4, algorithm='something_else')
+    assert_raises(TypeError,
+                  robust_single_linkage,
+                  X, 0.4, metric='minkowski', p=None)
+    assert_raises(ValueError,
+                  robust_single_linkage,
+                  X, 0.4, leaf_size=0)
+
 def test_rsl_is_sklearn_estimator():
 
     check_estimator(RobustSingleLinkage)

requirements.txt

@@ -1,5 +1,5 @@
 cython>=0.22
 numpy>=1.9
 scipy >= 0.9
-scikit-learn>=0.16
+scikit-learn>=0.17