WiP: Nomogram support on sparse

Author: nikicc

Orange/data/util.py

@@ -2,9 +2,58 @@
 Data-manipulation utilities.
 """
 import numpy as np
+import scipy.sparse as sp
 import bottleneck as bn
 
 
+def _nan_min_max(x, axis=0, func=None):
+    if not sp.issparse(x):
+        return func(x, axis=axis)
+    else:
+        r = []
+        if axis == 0:
+            x = x.T
+
+        # TODO check & transform to correct format
+
+        for row in x:
+            values = row.data
+            have_zeros = np.prod(row.shape) != values.size
+            extreme = func(values)
+            if have_zeros:
+                extreme = func([0, extreme])
+            r.append(extreme)
+        return np.array(r)
+
+
+def nan_min(x, axis):
+    return _nan_min_max(x, axis, np.nanmin)
+
+
+def nan_max(x, axis):
+    return _nan_min_max(x, axis, np.nanmax)
+
+
+def nan_average(x):
+    if not sp.issparse(x):
+        return np.average(x)
+    else:
+        n_values = np.prod(x.shape) - np.sum(np.isnan(x.data))
+        return np.nansum(x.data) / n_values
+
+
+def unique(x, return_counts=True):
+    if not sp.issparse(x):
+        return np.unique(x, return_counts=return_counts)
+    else:
+        n_zeros = np.prod(x.shape) - x.data.size
+        r = np.unique(x.data, return_counts=return_counts)
+        if return_counts:
+            return np.insert(r[0], 0, 0), np.insert(r[1], 0, n_zeros)
+        else:
+            return np.insert(r, 0, 0)
+
+
 def one_hot(values, dtype=float):
     """Return a one-hot transform of values
 

Orange/widgets/visualize/ownomogram.py

@@ -1,5 +1,6 @@
 import time
 from enum import IntEnum
+from collections import OrderedDict
 
 import numpy as np
 
@@ -12,6 +13,7 @@
 from AnyQt.QtCore import Qt, QEvent, QRectF, QSize
 
 from Orange.data import Table, Domain
+from Orange.data.util import nan_min, nan_max, nan_average, unique
 from Orange.classification import Model
 from Orange.classification.naive_bayes import NaiveBayesModel
 from Orange.classification.logistic_regression import \
@@ -865,11 +867,13 @@ def calculate_log_reg_coefficients(self):
         self.log_reg_coeffs = [coeffs[:, ranges[i]] for i in range(len(attrs))]
         self.log_reg_coeffs_orig = self.log_reg_coeffs.copy()
 
-        for i in range(len(self.log_reg_coeffs)):
+        min_values = nan_min(self.data.X, axis=0)
+        max_values = nan_max(self.data.X, axis=0)
+
+        for i, min_t, max_t in zip(range(len(self.log_reg_coeffs)),
+                                   min_values, max_values):
             if self.log_reg_coeffs[i].shape[1] == 1:
                 coef = self.log_reg_coeffs[i]
-                min_t = np.nanmin(self.data.X, axis=0)[i]
-                max_t = np.nanmax(self.data.X, axis=0)[i]
                 self.log_reg_coeffs[i] = np.hstack((coef * min_t, coef * max_t))
                 self.log_reg_cont_data_extremes.append(
                     [sorted([min_t, max_t], reverse=(c < 0)) for c in coef])
@@ -1076,10 +1080,10 @@ def _init_feature_marker_values(self):
             value, feature_val = 0, None
             if len(self.log_reg_coeffs):
                 if attr.is_discrete:
-                    ind, n = np.unique(self.data.X[:, i], return_counts=True)
+                    ind, n = unique(self.data.X[:, i], return_counts=True)
                     feature_val = np.nan_to_num(ind[np.argmax(n)])
                 else:
-                    feature_val = np.average(self.data.X[:, i])
+                    feature_val = nan_average(self.data.X[:, i])
             inst_in_dom = instances and attr in instances.domain
             if inst_in_dom and not np.isnan(instances[0][attr]):
                 feature_val = instances[0][attr]
@@ -1104,13 +1108,15 @@ def send_report(self):
 
     @staticmethod
     def reconstruct_domain(original, preprocessed):
-        attrs = []
+        # abuse dict to make "in" comparisons faster
+        attrs = OrderedDict()
         for attr in preprocessed.attributes:
             cv = attr._compute_value.variable._compute_value
             var = cv.variable if cv else original[attr.name]
-            if var in attrs:
+            if var in attrs:    # the reason for OrderedDict
                 continue
-            attrs.append(var)
+            attrs[var] = None   # we only need keys
+        attrs = list(attrs.keys())
         return Domain(attrs, original.class_var, original.metas)
 
     @staticmethod