DomainEditor: Support sparse data

Author: nikicc

Orange/widgets/data/owfile.py

@@ -403,11 +403,6 @@ def apply_domain_edit(self):
         if self.data is not None:
             domain, cols = self.domain_editor.get_domain(self.data.domain, self.data)
             X, y, m = cols
-            X = np.array(X).T if len(X) else np.empty((len(self.data), 0))
-            y = np.array(y).T if len(y) else None
-            dtpe = object if any(isinstance(m, StringVariable)
-                                 for m in domain.metas) else float
-            m = np.array(m, dtype=dtpe).T if len(m) else None
             table = Table.from_numpy(domain, X, y, m, self.data.W)
             table.name = self.data.name
             table.ids = np.array(self.data.ids)

Orange/widgets/utils/domaineditor.py

@@ -1,13 +1,15 @@
 from itertools import chain
 
 import numpy as np
+import scipy.sparse as sp
 
 from AnyQt.QtCore import Qt, QAbstractTableModel
 from AnyQt.QtGui import QColor
 from AnyQt.QtWidgets import QComboBox, QTableView, QSizePolicy
 
 from Orange.data import DiscreteVariable, ContinuousVariable, StringVariable, \
     TimeVariable, Domain
+from Orange.statistics.util import unique
 from Orange.widgets import gui
 from Orange.widgets.gui import HorizontalGridDelegate
 from Orange.widgets.settings import ContextSetting
@@ -215,6 +217,20 @@ def get_domain(self, domain, data):
         def is_missing(x):
             return str(x) in ("nan", "")
 
+        def iter_vals(x):
+            """Iterate over values of sparse or dense arrays."""
+            for i in range(x.shape[0]):
+                yield x[i, 0]
+
+        def to_column(x, to_sparse, dtype=None):
+            """Transform list of values to sparse/dense column array."""
+            x = np.array(x, dtype=dtype).reshape(-1, 1)
+            if to_sparse:
+                if dtype is not None:
+                    raise ValueError('Cannot set dtype on sparse matrix.')
+                x = sp.csc_matrix(x)
+            return x
+
         for (name, tpe, place, _, _), (orig_var, orig_plc) in \
                 zip(variables,
                         chain([(at, Place.feature) for at in domain.attributes],
@@ -225,31 +241,51 @@ def is_missing(x):
             if orig_plc == Place.meta:
                 col_data = data[:, orig_var].metas
             elif orig_plc == Place.class_var:
-                col_data = data[:, orig_var].Y
+                col_data = data[:, orig_var].Y.reshape(-1, 1)
             else:
                 col_data = data[:, orig_var].X
-            col_data = col_data.ravel()
+            is_sparse = sp.issparse(col_data)
             if name == orig_var.name and tpe == type(orig_var):
                 var = orig_var
             elif tpe == type(orig_var):
                 # change the name so that all_vars will get the correct name
                 orig_var.name = name
                 var = orig_var
             elif tpe == DiscreteVariable:
-                values = list(str(i) for i in np.unique(col_data) if not is_missing(i))
+                values = list(str(i) for i in unique(col_data) if not is_missing(i))
                 var = tpe(name, values)
                 col_data = [np.nan if is_missing(x) else values.index(str(x))
-                            for x in col_data]
+                            for x in iter_vals(col_data)]
+                col_data = to_column(col_data, is_sparse)
             elif tpe == StringVariable and type(orig_var) == DiscreteVariable:
                 var = tpe(name)
                 col_data = [orig_var.repr_val(x) if not np.isnan(x) else ""
-                            for x in col_data]
+                            for x in iter_vals(col_data)]
+                col_data = to_column(col_data, is_sparse, dtype=object)
             else:
                 var = tpe(name)
             places[place].append(var)
             cols[place].append(col_data)
+
+        # merge columns for X, Y and metas
+        def merge(cols, assure_dense=False):
+            if len(cols) == 0:
+                return None
+            if assure_dense and any(sp.issparse(c) for c in cols):
+                cols = [c.toarray() if sp.issparse(c) else c for c in cols]
+            if not any(sp.issparse(c) for c in cols):
+                return np.hstack(cols)
+            if not all(sp.issparse(c) for c in cols):
+                cols = [c if sp.issparse(c) else sp.csc_matrix(c)
+                        for c in cols]
+            return sp.hstack(cols).tocsr()
+
+        feats = cols[Place.feature]
+        X = merge(feats) if feats else np.empty((len(data), 0))
+        Y = merge(cols[Place.class_var], assure_dense=True)
+        m = merge(cols[Place.meta], assure_dense=True)
         domain = Domain(*places)
-        return domain, cols
+        return domain, [X, Y, m]
 
     def set_domain(self, domain):
         self.variables = self.parse_domain(domain)