remove_unused_values: Support sparse matrices

Author: nikicc

Orange/preprocess/remove.py

@@ -1,9 +1,11 @@
 from collections import namedtuple
+
 import numpy as np
 
-from .preprocess import Preprocess
 from Orange.data import Domain, DiscreteVariable, Table
 from Orange.preprocess.transformation import Lookup
+from Orange.statistics.util import nanunique
+from .preprocess import Preprocess
 
 __all__ = ["Remove"]
 
@@ -234,10 +236,7 @@ def remove_unused_values(var, data):
         Domain([var]),
         data
     )
-    array = column_data.X.ravel()
-    mask = np.isfinite(array)
-    unique = np.array(np.unique(array[mask]), dtype=int)
-
+    unique = nanunique(column_data.X).astype(int)
     if len(unique) == len(var.values):
         return var
 

Orange/tests/test_remove.py

@@ -4,6 +4,8 @@
 import unittest
 
 import numpy as np
+import scipy.sparse as sp
+
 from Orange.data import Table
 from Orange.preprocess import Remove
 from Orange.tests import test_filename
@@ -133,3 +135,19 @@ def test_remove_unused_values_metas(self):
         self.assertEqual(res.domain["b"].values, res.domain["c"].values)
         self.assertEqual(res.domain["d"].values, ["1", "2"])
         self.assertEqual(res.domain["f"].values, ['1', 'hey'])
+
+    def test_remove_unused_values_attr_sparse(self):
+        data = self.test8
+        data = data[1:]
+        data.X = sp.csr_matrix(data.X)
+        remover = Remove(Remove.RemoveUnusedValues)
+        new_data = remover(data)
+        attr_res = remover.attr_results
+
+        self.assertEqual((new_data.X != data.X).nnz, 0)
+        self.assertEqual([a.values for a in new_data.domain.attributes
+                          if a.is_discrete], [['1'], ['4']])
+        self.assertEqual([c.values for c in new_data.domain.class_vars
+                          if c.is_discrete], [['1', '2', '3'], ['2']])
+        self.assertDictEqual(attr_res,
+                             {'removed': 0, 'reduced': 1, 'sorted': 0})