OWFile: Add test for loading sparse data

Author: nikicc

Orange/widgets/data/tests/test_owfile.py

@@ -2,8 +2,12 @@
 # pylint: disable=missing-docstring
 from os import path, remove
 from unittest.mock import Mock
+import pickle
+import tempfile
+
 
 import numpy as np
+import scipy.sparse as sp
 
 from AnyQt.QtCore import QMimeData, QPoint, Qt, QUrl
 from AnyQt.QtGui import QDragEnterEvent, QDropEvent
@@ -195,3 +199,19 @@ def test_check_datetime_disabled(self):
             for i in range(4):
                 vartype_delegate.setEditorData(combo, idx(i))
                 self.assertEqual(combo.count(), counts[i])
+
+    def test_domain_edit_on_sparse_data(self):
+        iris = Table("iris")
+        iris.X = sp.csr_matrix(iris.X)
+
+        f = tempfile.NamedTemporaryFile(suffix='.pickle', delete=False)
+        pickle.dump(iris, f)
+        f.close()
+
+        self.widget.add_path(f.name)
+        self.widget.load_data()
+
+        output = self.get_output("Data")
+        self.assertIsInstance(output, Table)
+        self.assertEqual(iris.X.shape, output.X.shape)
+        self.assertTrue(sp.issparse(output.X))

Orange/widgets/utils/domaineditor.py

@@ -198,6 +198,35 @@ def __init__(self, widget):
         self.place_delegate = PlaceDelegate(self, VarTableModel.places)
         self.setItemDelegateForColumn(Column.place, self.place_delegate)
 
+    @staticmethod
+    def _iter_vals(x):
+        """Iterate over values of sparse or dense arrays."""
+        for i in range(x.shape[0]):
+            yield x[i, 0]
+
+    @staticmethod
+    def _to_column(x, to_sparse, dtype=None):
+        """Transform list of values to sparse/dense column array."""
+        x = np.array(x, dtype=dtype).reshape(-1, 1)
+        if to_sparse:
+            if dtype is not None:
+                raise ValueError('Cannot set dtype on sparse matrix.')
+            x = sp.csc_matrix(x)
+        return x
+
+    @staticmethod
+    def _merge(cols, assure_dense=False):
+        if len(cols) == 0:
+            return None
+        if assure_dense and any(sp.issparse(c) for c in cols):
+            cols = [c.toarray() if sp.issparse(c) else c for c in cols]
+        if not any(sp.issparse(c) for c in cols):
+            return np.hstack(cols)
+        if not all(sp.issparse(c) for c in cols):
+            cols = [c if sp.issparse(c) else sp.csc_matrix(c)
+                    for c in cols]
+        return sp.hstack(cols).tocsr()
+
     def get_domain(self, domain, data):
         """Create domain (and dataset) from changes made in the widget.
 
@@ -217,20 +246,6 @@ def get_domain(self, domain, data):
         def is_missing(x):
             return str(x) in ("nan", "")
 
-        def iter_vals(x):
-            """Iterate over values of sparse or dense arrays."""
-            for i in range(x.shape[0]):
-                yield x[i, 0]
-
-        def to_column(x, to_sparse, dtype=None):
-            """Transform list of values to sparse/dense column array."""
-            x = np.array(x, dtype=dtype).reshape(-1, 1)
-            if to_sparse:
-                if dtype is not None:
-                    raise ValueError('Cannot set dtype on sparse matrix.')
-                x = sp.csc_matrix(x)
-            return x
-
         for (name, tpe, place, _, _), (orig_var, orig_plc) in \
                 zip(variables,
                         chain([(at, Place.feature) for at in domain.attributes],
@@ -255,35 +270,23 @@ def to_column(x, to_sparse, dtype=None):
                 values = list(str(i) for i in unique(col_data) if not is_missing(i))
                 var = tpe(name, values)
                 col_data = [np.nan if is_missing(x) else values.index(str(x))
-                            for x in iter_vals(col_data)]
-                col_data = to_column(col_data, is_sparse)
+                            for x in self._iter_vals(col_data)]
+                col_data = self._to_column(col_data, is_sparse)
             elif tpe == StringVariable and type(orig_var) == DiscreteVariable:
                 var = tpe(name)
                 col_data = [orig_var.repr_val(x) if not np.isnan(x) else ""
-                            for x in iter_vals(col_data)]
-                col_data = to_column(col_data, is_sparse, dtype=object)
+                            for x in self._iter_vals(col_data)]
+                col_data = self._to_column(col_data, is_sparse, dtype=object)
             else:
                 var = tpe(name)
             places[place].append(var)
             cols[place].append(col_data)
 
         # merge columns for X, Y and metas
-        def merge(cols, assure_dense=False):
-            if len(cols) == 0:
-                return None
-            if assure_dense and any(sp.issparse(c) for c in cols):
-                cols = [c.toarray() if sp.issparse(c) else c for c in cols]
-            if not any(sp.issparse(c) for c in cols):
-                return np.hstack(cols)
-            if not all(sp.issparse(c) for c in cols):
-                cols = [c if sp.issparse(c) else sp.csc_matrix(c)
-                        for c in cols]
-            return sp.hstack(cols).tocsr()
-
         feats = cols[Place.feature]
-        X = merge(feats) if feats else np.empty((len(data), 0))
-        Y = merge(cols[Place.class_var], assure_dense=True)
-        m = merge(cols[Place.meta], assure_dense=True)
+        X = self._merge(feats) if feats else np.empty((len(data), 0))
+        Y = self._merge(cols[Place.class_var], assure_dense=True)
+        m = self._merge(cols[Place.meta], assure_dense=True)
         domain = Domain(*places)
         return domain, [X, Y, m]