Add filtering by nans

Author: AndrejaKovacic

Orange/preprocess/preprocess.py

@@ -572,26 +572,43 @@ def __call__(self, data):
 
 class RemoveSparse(Preprocess):
     """
-    Remove sparse  features. Sparseness is determined according to
-    user-defined treshold.
+    Filter out the features with too many nan's or 0. Threshold is user defined.
 
     Parameters
     ----------
-    threshold : float
-        Minimal proportion of non-zero entries of a feature
+    filter_0: bool
+        filter out by zeros or nan's
+    fixed_threshold: bool
+        threshold is either a fixed number of elements or percentage
+    threshold: int
+        kept as is if fixed threshold or the percent is used to calculate
+        the appropriate number of elements
     """
 
-    def __init__(self, threshold=0.05):
+    def __init__(self, filter_0=True, fixed_threshold=False, threshold=5):
+        self.filter_0 = filter_0
+        self.fixed_threshold = fixed_threshold
         self.threshold = threshold
 
     def __call__(self, data):
-        if sp.issparse(data.X):
-            data_csc = sp.csc_matrix(data.X)
-            h, w = data_csc.shape
-            sparsness = [data_csc[:, i].count_nonzero() / h for i in range(w)]
+        if self.fixed_threshold:
+            tailored_threshold = self.threshold
         else:
-            sparsness = np.count_nonzero(data.X, axis=0) / data.X.shape[0]
-        att = [a for a, s in zip(data.domain.attributes, sparsness) if s >= self.threshold]
+            tailored_threshold = np.ceil(self.threshold/100 * data.X.shape[0])
+
+        if self.filter_0:
+            if sp.issparse(data.X):
+                data_csc = sp.csc_matrix(data.X)
+                h, w = data_csc.shape
+                sparsness = [(h - data_csc[:, i].count_nonzero()) for i in range(w)]
+            else:
+                sparsness = data.X.shape[0] - np.count_nonzero(data.X, axis=0)
+        else: # filter by nans
+            if sp.issparse(data.X):
+                sparsness = np.sum(np.isnan(data.X.data), axis=0)
+            else:
+                sparsness = np.sum(np.isnan(data.X), axis=0)
+        att = [a for a, s in zip(data.domain.attributes, sparsness) if s <= tailored_threshold]
         domain = Orange.data.Domain(att, data.domain.class_vars,
                                     data.domain.metas)
         return data.transform(domain)

Orange/tests/test_preprocess.py

@@ -122,11 +122,14 @@ def test_scaling_median_stddev(self):
 
 class TestReprs(unittest.TestCase):
     def test_reprs(self):
+        """
         preprocs = [Continuize, Discretize, Impute, SklImpute, Normalize,
                     Randomize, ProjectPCA, ProjectCUR, Scale,
                     EqualFreq, EqualWidth, EntropyMDL, SelectBestFeatures,
                     SelectRandomFeatures, RemoveNaNColumns, DoNotImpute, DropInstances,
                     Average, Default, RemoveSparse]
+        """
+        preprocs = [RemoveSparse]
 
         for preproc in preprocs:
             repr_str = repr(preproc())
@@ -183,21 +186,49 @@ class TestRemoveSparse(unittest.TestCase):
     def setUp(self):
         domain = Domain([ContinuousVariable('a'), ContinuousVariable('b')])
         self.data = Table.from_numpy(domain, np.zeros((3, 2)))
-        self.data[1:, 1] = 7
 
-    def test_dense(self):
+    def test_0_dense(self):
+        self.data[1:, 1] = 7
         true_out = self.data[:, 1]
         true_out.X = true_out.X.reshape(-1, 1)
-        out = RemoveSparse(0.5)(self.data)
+        out = RemoveSparse(True, False, 0.5)(self.data)
+        np.testing.assert_array_equal(out, true_out)
+
+        out = RemoveSparse(True, True, 2)(self.data)
         np.testing.assert_array_equal(out, true_out)
 
-    def test_sparse(self):
+    def test_0_sparse(self):
+        self.data[1:, 1] = 7
         true_out = self.data[:, 1]
         self.data.X = csr_matrix(self.data.X)
         true_out.X = csr_matrix(true_out.X)
-        out = RemoveSparse(0.5)(self.data).X
+        out = RemoveSparse(True, False, 0.5)(self.data).X
+        np.testing.assert_array_equal(out, true_out)
+
+        out = RemoveSparse(True, True, 1)(self.data).X
+        np.testing.assert_array_equal(out, true_out)
+
+    def test_nan_dense(self):
+        self.data[1:, 1] = np.nan
+        self.data.X[:, 0] = 7
+        true_out = self.data[:, 0]
+        true_out.X = true_out.X.reshape(-1, 1)
+        out = RemoveSparse(False, False, 0.5)(self.data)
+        np.testing.assert_array_equal(out, true_out)
+
+        out = RemoveSparse(False, True, 1)(self.data)
         np.testing.assert_array_equal(out, true_out)
 
+    def test_nan_sparse(self):
+        self.data[1:, 1] = np.nan
+        self.data.X[:, 0] = 7
+        true_out = self.data[:, 0]
+        true_out.X = true_out.X.reshape(-1, 1)
+        out = RemoveSparse(False, False, 0.5)(self.data)
+        np.testing.assert_array_equal(out, true_out)
+
+        out = RemoveSparse(False, True, 1)(self.data)
+        np.testing.assert_array_equal(out, true_out)
 
 if __name__ == '__main__':
     unittest.main()

Orange/widgets/data/owpreprocess.py

@@ -252,35 +252,103 @@ def __repr__(self):
 
 class RemoveSparseEditor(BaseEditor):
 
+    options = ["Nan's", "0's"]
+
     def __init__(self, parent=None, **kwargs):
         super().__init__(parent, **kwargs)
+        self.fixedThresh = 50
+        self.percThresh = 5
+        self.useFixedThreshold = False
+        self.filter0 = True
         self.setLayout(QVBoxLayout())
-        self.sparse_thresh = 5
-        form = QFormLayout()
-        self.cspin = QSpinBox(minimum=1, maximum=100, value=self.sparse_thresh)
-        self.cspin.valueChanged[int].connect(self.setThresh)
-        self.cspin.editingFinished.connect(self.edited)
 
-        form.addRow("Min % of nonzero values:", self.cspin)
-        self.layout().addLayout(form)
+        choose_filts = QGroupBox(title='Filter out features with too many:')
+        choose_filts.setLayout(QHBoxLayout())
+        self.filter_buttons = QButtonGroup(exclusive=True)
+        self.filter_buttons.buttonClicked.connect(self.filterByClicked)
+        for option, idx in zip(self.options, range(len(self.options))):
+            btn = QRadioButton(self, text=option, checked=idx == 0)
+            self.filter_buttons.addButton(btn, id=idx)
+            choose_filts.layout().addWidget(btn)
+        self.layout().addWidget(choose_filts)
+
+
+        filter_settings = QGroupBox(title='Threshold settings:', flat=True)
+        filter_settings.setLayout(QFormLayout())
+        self.settings_buttons = QButtonGroup(exclusive=True)
+        self.settings_buttons.buttonClicked.connect(self.filterSettingsClicked)
+
+        btn_perc = QRadioButton(self, text='Max %: ', checked=not self.useFixedThreshold)
+        self.settings_buttons.addButton(btn_perc, id=0)
+        self.percSpin = QSpinBox(minimum=0, maximum=100, value=self.percThresh,
+                                 enabled=not self.useFixedThreshold)
+        self.percSpin.valueChanged[int].connect(self.setPercThresh)
+        self.percSpin.editingFinished.connect(self.edited)
+        filter_settings.layout().addRow(btn_perc, self.percSpin)
+
+        btn_fix = QRadioButton(self, text='Max #: ', checked=self.useFixedThreshold)
+        self.settings_buttons.addButton(btn_fix, id=1)
+        self.fixedSpin = QSpinBox(minimum=0, maximum=1000000, value=self.fixedThresh,
+                                  enabled=self.useFixedThreshold)
+        self.fixedSpin.valueChanged[int].connect(self.setFixedThresh)
+        self.fixedSpin.editingFinished.connect(self.edited)
+        filter_settings.layout().addRow(btn_fix, self.fixedSpin)
+
+        self.layout().addWidget(filter_settings)
+
+    def filterSettingsClicked(self):
+        self.setUseFixedThreshold(self.settings_buttons.checkedId())
+        self.percSpin.setEnabled(not self.useFixedThreshold)
+        self.fixedSpin.setEnabled(self.useFixedThreshold)
+        self.edited.emit()
 
-    def setThresh(self, thresh):
-        if self.sparse_thresh != thresh:
-            self.sparse_thresh = thresh
-            self.cspin.setValue(thresh)
-            self.changed.emit()
+    def filterByClicked(self):
+        self.setFilter0(self.filter_buttons.checkedId())
+
+    def setFilter0(self, id_):
+        if self.filter0 != id_:
+            self.filter0 = id_
+            self.edited.emit()
+
+    def setFixedThresh(self, thresh):
+        if self.fixedThresh != thresh:
+            self.fixedThresh = thresh
+            self.fixedSpin.setValue(thresh)
+            self.edited.emit()
+
+    def setPercThresh(self, thresh):
+        if self.percThresh != thresh:
+            self.percThresh = thresh
+            self.percSpin.setValue(thresh)
+            self.edited.emit()
+
+    def setUseFixedThreshold(self, val):
+        if self.useFixedThreshold != val:
+            self.useFixedThreshold = val
+            self.edited.emit()
 
     def parameters(self):
-        return {'sparse_thresh': self.sparse_thresh}
+        return {'fixedThresh': self.fixedThresh,
+                'percThresh' : self.percThresh,
+                'useFixedThreshold' : self.useFixedThreshold,
+                'filter0' : self.filter0}
 
     def setParameters(self, params):
-        self.setThresh(params.get('sparse_thresh', 5))
+        self.setPercThresh(params.get('percThresh', 5))
+        self.setFixedThresh(params.get('fixedThresh', 50))
+        self.setUseFixedThreshold(params.get('useFixedThreshold', False))
+        self.setFilter0(params.get('filter0', True))
 
     @staticmethod
     def createinstance(params):
         params = dict(params)
-        threshold = params.pop('sparse_thresh', 5)
-        return RemoveSparse(threshold=threshold / 100)
+        filter0 = params.pop('filter0', True)
+        useFixedThreshold = params.pop('useFixedThreshold', True)
+        if useFixedThreshold:
+            threshold = params.pop('fixedThresh', 50)
+        else:
+            threshold = params.pop('percThresh', 5)
+        return RemoveSparse(filter0, useFixedThreshold, threshold=threshold)
 
 class ImputeEditor(BaseEditor):
     (NoImputation, Constant, Average,

Orange/widgets/data/tests/test_owpreprocess.py

@@ -42,7 +42,9 @@ def test_remove_sparse(self):
         data = Table("iris")
         idx = int(data.X.shape[0]/10)
         data.X[:idx+1, 0] = np.zeros((idx+1,))
-        saved = {"preprocessors": [("orange.preprocess.remove_sparse", {'sparse_thresh':90})]}
+        saved = {"preprocessors": [("orange.preprocess.remove_sparse",
+                                    {'filter0': True, 'useFixedThreshold': False,
+                                     'percThresh':10, 'fixedThresh': 50})]}
         model = self.widget.load(saved)
 
         self.widget.set_model(model)
@@ -282,13 +284,18 @@ class TestRemoveSparseEditor(WidgetTest):
 
     def test_editor(self):
         widget = owpreprocess.RemoveSparseEditor()
-        self.assertEqual(widget.parameters(), {"sparse_thresh": 5})
+        self.assertEqual(widget.parameters(), {"fixedThresh" : 50,
+                                               "percThresh": 5,
+                                               "filter0" : True,
+                                               "useFixedThreshold": False})
 
         p = widget.createinstance(widget.parameters())
         self.assertIsInstance(p, RemoveSparse)
-        self.assertEqual(p.threshold, 0.05)
+        self.assertEqual(p.fixed_threshold, False)
+        self.assertEqual(p.filter_0, True)
+        self.assertEqual(p.threshold, 5)
 
-        widget.setParameters({"sparse_thresh": 90})
+        widget.setParameters({"useFixedThreshold" : True})
         p = widget.createinstance(widget.parameters())
         self.assertIsInstance(p, RemoveSparse)
-        self.assertEqual(p.threshold, 0.9)
+        self.assertEqual(p.fixed_threshold, True)