Merge pull request #5102 from janezd/impute-constant-all

[ENH] Impute: Allow setting a default value for all numeric and time variables

Author: lanzagar

Orange/preprocess/impute.py

@@ -7,7 +7,7 @@
 from .transformation import Transformation, Lookup
 
 __all__ = ["ReplaceUnknowns", "Average", "DoNotImpute", "DropInstances",
-           "Model", "AsValue", "Random", "Default"]
+           "Model", "AsValue", "Random", "Default", "FixedValueByType"]
 
 
 class ReplaceUnknowns(Transformation):
@@ -113,6 +113,10 @@ def __call__(self, data, variable, value=None):
         a.to_sql = ImputeSql(variable, value)
         return a
 
+    @staticmethod
+    def supports_variable(variable):
+        return variable.is_primitive()
+
 
 class ImputeSql(Reprable):
     def __init__(self, var, default):
@@ -124,7 +128,7 @@ def __call__(self):
 
 
 class Default(BaseImputeMethod):
-    name = "Value"
+    name = "Fixed value"
     short_name = "value"
     description = ""
     columns_only = True
@@ -142,6 +146,32 @@ def copy(self):
         return Default(self.default)
 
 
+class FixedValueByType(BaseImputeMethod):
+    name = "Fixed value"
+    short_name = "Fixed Value"
+    format = "{var.name}"
+
+    def __init__(self,
+                 default_discrete=np.nan, default_continuous=np.nan,
+                 default_string=None, default_time=np.nan):
+        # If you change the order of args or in dict, also fix method copy
+        self.defaults = {
+            Orange.data.DiscreteVariable: default_discrete,
+            Orange.data.ContinuousVariable: default_continuous,
+            Orange.data.StringVariable: default_string,
+            Orange.data.TimeVariable: default_time
+        }
+
+    def __call__(self, data, variable, *, default=None):
+        variable = data.domain[variable]
+        if default is None:
+            default = self.defaults[type(variable)]
+        return variable.copy(compute_value=ReplaceUnknowns(variable, default))
+
+    def copy(self):
+        return FixedValueByType(*self.defaults.values())
+
+
 class ReplaceUnknownsModel(Reprable):
     """
     Replace unknown values with predicted values using a `Orange.base.Model`
@@ -272,6 +302,9 @@ def __call__(self, data, variable):
         else:
             raise TypeError(type(variable))
 
+    @staticmethod
+    def supports_variable(variable):
+        return variable.is_primitive()
 
 class ReplaceUnknownsRandom(Transformation):
     """
@@ -354,3 +387,7 @@ def __call__(self, data, variable):
             dist[1, :] += 1 / dist.shape[1]
         return variable.copy(
             compute_value=ReplaceUnknownsRandom(variable, dist))
+
+    @staticmethod
+    def supports_variable(variable):
+        return variable.is_primitive()

Orange/preprocess/tests/test_impute.py

@@ -1,7 +1,12 @@
 import unittest
 
-from Orange.data import DiscreteVariable, ContinuousVariable
-from Orange.preprocess.impute import ReplaceUnknownsRandom, ReplaceUnknowns
+import numpy as np
+
+from Orange.data import \
+    Domain, Table, \
+    DiscreteVariable, ContinuousVariable, TimeVariable, StringVariable
+from Orange.preprocess.impute import ReplaceUnknownsRandom, ReplaceUnknowns, \
+    FixedValueByType
 from Orange.statistics.distribution import Discrete
 
 
@@ -52,5 +57,59 @@ def test_equality(self):
         self.assertNotEqual(hash(t1), hash(t1a))
 
 
+class TestFixedValuesByType(unittest.TestCase):
+    def setUp(self):
+        domain = Domain(
+            [DiscreteVariable("d", values=tuple("abc")),
+             ContinuousVariable("c"),
+             TimeVariable("t")],
+            [],
+            [StringVariable("s")]
+        )
+        n = np.nan
+        self.data = Table(
+            domain,
+            np.array([[1, n, 15], [n, 42, n]]),
+            np.empty((2, 0)),
+            np.array([["foo"], [""]]))
+
+    def test_none_defined(self):
+        d, c, t = self.data.domain.attributes
+        s,  = self.data.domain.metas
+
+        imputer = FixedValueByType()
+        for var in (d, c, t):
+            imp = imputer(self.data, var)
+            self.assertIsInstance(imp.compute_value, ReplaceUnknowns)
+            self.assertTrue(np.isnan(imp.compute_value.value))
+        imp = imputer(self.data, s)
+        self.assertIsInstance(imp.compute_value, ReplaceUnknowns)
+        self.assertIsNone(imp.compute_value.value)
+
+    def test_all_defined(self):
+        d, c, t = self.data.domain.attributes
+        s, = self.data.domain.metas
+
+        imputer = FixedValueByType(
+            default_discrete=1, default_continuous=42,
+            default_string="foo", default_time=3.14)
+
+        self.assertEqual(imputer(self.data, d).compute_value.value, 1)
+        self.assertEqual(imputer(self.data, c).compute_value.value, 42)
+        self.assertEqual(imputer(self.data, t).compute_value.value, 3.14)
+        self.assertEqual(imputer(self.data, s).compute_value.value, "foo")
+
+    def test_with_default(self):
+        s, = self.data.domain.metas
+
+        imputer = FixedValueByType(
+            default_discrete=1, default_continuous=42,
+            default_string="foo", default_time=3.14)
+
+        self.assertEqual(
+            imputer(self.data, s, default="bar").compute_value.value,
+            "bar")
+
+
 if __name__ == "__main__":
     unittest.main()

Orange/widgets/data/owimpute.py

@@ -11,10 +11,11 @@
 from AnyQt.QtWidgets import (
     QGroupBox, QRadioButton, QPushButton, QHBoxLayout, QGridLayout,
     QVBoxLayout, QStackedWidget, QComboBox,
-    QButtonGroup, QStyledItemDelegate, QListView, QDoubleSpinBox
-)
-from AnyQt.QtCore import Qt, QThread, QModelIndex
+    QButtonGroup, QStyledItemDelegate, QListView, QDoubleSpinBox, QLabel)
+from AnyQt.QtCore import Qt, QThread, QModelIndex, QDateTime, QLocale
 from AnyQt.QtCore import pyqtSlot as Slot
+from AnyQt.QtGui import QDoubleValidator
+
 from orangewidget.utils.listview import ListViewSearch
 
 import Orange.data
@@ -154,6 +155,8 @@ class Warning(OWWidget.Warning):
     _variable_imputation_state = settings.ContextSetting({})  # type: VariableState
 
     autocommit = settings.Setting(True)
+    default_numeric = settings.Setting("")
+    default_time = settings.Setting(0)
 
     want_main_area = False
     resizing_enabled = False
@@ -171,11 +174,13 @@ def __init__(self):
         main_layout.setContentsMargins(10, 10, 10, 10)
         self.controlArea.layout().addLayout(main_layout)
 
-        box = QGroupBox(title=self.tr("Default Method"), flat=False)
-        box_layout = QGridLayout(box)
-        box_layout.setContentsMargins(5, 0, 0, 0)
+        box = gui.vBox(None, "Default Method")
         main_layout.addWidget(box)
 
+        box_layout = QGridLayout(box)
+        box_layout.setSpacing(8)
+        box.layout().addLayout(box_layout)
+
         button_group = QButtonGroup()
         button_group.buttonClicked[int].connect(self.set_default_method)
 
@@ -186,6 +191,41 @@ def __init__(self):
             button_group.addButton(button, method)
             box_layout.addWidget(button, i % 3, i // 3)
 
+        def set_to_fixed_value():
+            self.set_default_method(Method.Default)
+
+        def set_default_time(datetime):
+            self.default_time = datetime.toSecsSinceEpoch()
+            if self.default_method_index != Method.Default:
+                set_to_fixed_value()
+            else:
+                self._invalidate()
+
+        hlayout = QHBoxLayout()
+        box.layout().addLayout(hlayout)
+        button = QRadioButton("Fixed values; numeric variables:")
+        button_group.addButton(button, Method.Default)
+        button.setChecked(Method.Default == self.default_method_index)
+        hlayout.addWidget(button)
+
+        locale = QLocale()
+        locale.setNumberOptions(locale.NumberOption.RejectGroupSeparator)
+        validator = QDoubleValidator()
+        validator.setLocale(locale)
+        le = gui.lineEdit(
+            None, self, "default_numeric",
+            validator=validator, alignment=Qt.AlignRight,
+            callback=self._invalidate, focusInCallback=set_to_fixed_value)
+        hlayout.addWidget(le)
+
+        hlayout.addWidget(QLabel(", time:"))
+
+        self.time_widget = gui.DateTimeEditWCalendarTime(self)
+        self.time_widget.setContentsMargins(0, 0, 0, 0)
+        self.default_time = QDateTime.currentDateTime().toSecsSinceEpoch()
+        self.time_widget.dateTimeChanged.connect(set_default_time)
+        hlayout.addWidget(self.time_widget)
+
         self.default_button_group = button_group
 
         box = QGroupBox(title=self.tr("Individual Attribute Settings"),
@@ -267,6 +307,17 @@ def create_imputer(self, method, *args):
             m = AsDefault()
             m.method = default
             return m
+        elif method == Method.Default and not args:  # global default values
+            if self.default_numeric == "":
+                default_num = np.nan
+            else:
+                default_num, ok = QLocale().toDouble(self.default_numeric)
+                if not ok:
+                    default_num = np.nan
+            return impute.FixedValueByType(
+                default_continuous=default_num,
+                default_time=self.default_time or np.nan
+            )
         else:
             return METHODS[method](*args)
 
@@ -302,6 +353,8 @@ def set_data(self, data):
         if data is not None:
             self.varmodel[:] = data.domain.variables
             self.openContext(data.domain)
+            self.time_widget.set_datetime(
+                QDateTime.fromSecsSinceEpoch(self.default_time))
             # restore per variable imputation state
             self._restore_state(self._variable_imputation_state)
 
@@ -660,5 +713,19 @@ def storeSpecificSettings(self):
         super().storeSpecificSettings()
 
 
+def __sample_data():  # pragma: no cover
+    domain = Orange.data.Domain(
+        [Orange.data.ContinuousVariable(f"c{i}") for i in range(3)]
+        + [Orange.data.TimeVariable(f"t{i}") for i in range(3)],
+        [])
+    n = np.nan
+    x = np.array([
+        [1, 2, n, 1000, n, n],
+        [2, n, 1, n, 2000, 2000]
+    ])
+    return Orange.data.Table(domain, x, np.empty((2, 0)))
+
+
 if __name__ == "__main__":  # pragma: no cover
+    # WidgetPreview(OWImpute).run(__sample_data())
     WidgetPreview(OWImpute).run(Orange.data.Table("brown-selected"))

Orange/widgets/data/tests/test_owimpute.py

@@ -3,10 +3,10 @@
 from unittest.mock import Mock
 import numpy as np
 
-from AnyQt.QtCore import Qt, QItemSelection
+from AnyQt.QtCore import Qt, QItemSelection, QLocale
 from AnyQt.QtTest import QTest
 
-from Orange.data import Table, Domain
+from Orange.data import Table, Domain, ContinuousVariable, TimeVariable
 from Orange.preprocess import impute
 from Orange.widgets.data.owimpute import OWImpute, AsDefault, Learner, Method
 from Orange.widgets.tests.base import WidgetTest
@@ -119,6 +119,31 @@ def test_select_method(self):
         self.assertIsInstance(widget.get_method_for_column(0), AsDefault)
         self.assertIsInstance(widget.get_method_for_column(2), AsDefault)
 
+    def test_overall_default(self):
+        domain = Domain(
+            [ContinuousVariable(f"c{i}") for i in range(3)]
+            + [TimeVariable(f"t{i}") for i in range(3)],
+            [])
+        n = np.nan
+        x = np.array([
+            [1, 2, n, 1000, n, n],
+            [2, n, 1, n, 2000, 2000]
+        ])
+        data = Table(domain, x, np.empty((2, 0)))
+
+        widget = self.widget
+        widget.default_numeric = QLocale().toString(3.14)
+        widget.default_time = 42
+        widget.default_method_index = Method.Default
+
+        self.send_signal(self.widget.Inputs.data, data)
+        imp_data = self.get_output(self.widget.Outputs.data)
+        np.testing.assert_almost_equal(
+            imp_data.X,
+            [[1, 2, 3.14, 1000, 42, 42],
+             [2, 3.14, 1, 42, 2000, 2000]]
+        )
+
     def test_value_edit(self):
         data = Table("heart_disease")[::10]
         self.send_signal(self.widget.Inputs.data, data)