Merge pull request #4302 from janezd/save-addon-formats

[ENH] Allow add-ons to register file format for the Save widget

Author: markotoplak

Orange/data/io.py

@@ -395,6 +395,8 @@ def write_file(cls, filename, data):
     # the sort order in file open/save combo boxes. Lower is better.
     PRIORITY = 10000
     OPTIONAL_TYPE_ANNOTATIONS = False
+    SUPPORT_COMPRESSED = False
+    SUPPORT_SPARSE_DATA = False
 
     def __init__(self, filename):
         """

Orange/widgets/data/owsave.py

@@ -1,8 +1,8 @@
 import os.path
 
 from Orange.data.table import Table
-from Orange.data.io import TabReader, CSVReader, PickleReader, ExcelReader, \
-    XlsReader
+from Orange.data.io import \
+    TabReader, CSVReader, PickleReader, ExcelReader, XlsReader, FileFormat
 from Orange.widgets import gui, widget
 from Orange.widgets.widget import Input
 from Orange.widgets.settings import Setting
@@ -22,14 +22,6 @@ class OWSave(OWSaveBase):
 
     settings_version = 2
 
-    writers = [TabReader, CSVReader, PickleReader, ExcelReader, XlsReader]
-    filters = {
-        **{f"{w.DESCRIPTION} (*{w.EXTENSIONS[0]})": w
-           for w in writers},
-        **{f"Compressed {w.DESCRIPTION} (*{w.EXTENSIONS[0]}.gz)": w
-           for w in writers if w.SUPPORT_COMPRESSED}
-    }
-
     class Inputs:
         data = Input("Data", Table)
 
@@ -38,6 +30,8 @@ class Error(OWSaveBase.Error):
 
     add_type_annotations = Setting(True)
 
+    builtin_order = [TabReader, CSVReader, PickleReader, ExcelReader, XlsReader]
+
     def __init__(self):
         super().__init__(2)
 
@@ -53,6 +47,21 @@ def __init__(self):
         self.grid.setRowMinimumHeight(1, 8)
         self.adjustSize()
 
+    @classmethod
+    def get_filters(cls):
+        writers = [format for format in FileFormat.formats
+                   if getattr(format, 'write_file', None)
+                   and getattr(format, "EXTENSIONS", None)]
+        writers.sort(key=lambda writer: cls.builtin_order.index(writer)
+                     if writer in cls.builtin_order else 99)
+
+        return {
+            **{f"{w.DESCRIPTION} (*{w.EXTENSIONS[0]})": w
+               for w in writers},
+            **{f"Compressed {w.DESCRIPTION} (*{w.EXTENSIONS[0]}.gz)": w
+               for w in writers if w.SUPPORT_COMPRESSED}
+        }
+
     @Inputs.data
     def dataset(self, data):
         self.data = data
@@ -95,21 +104,21 @@ def migrate_settings(cls, settings, version=0):
         def migrate_to_version_2():
             # Set the default; change later if possible
             settings.pop("compression", None)
-            settings["filter"] = next(iter(cls.filters))
+            settings["filter"] = next(iter(cls.get_filters()))
             filetype = settings.pop("filetype", None)
             if filetype is None:
                 return
 
             ext = cls._extension_from_filter(filetype)
             if settings.pop("compress", False):
-                for afilter in cls.filters:
+                for afilter in cls.get_filters():
                     if ext + ".gz" in afilter:
                         settings["filter"] = afilter
                         return
                 # If not found, uncompressed may have been erroneously set
                 # for a writer that didn't support if (such as .xlsx), so
                 # fall through to uncompressed
-            for afilter in cls.filters:
+            for afilter in cls.get_filters():
                 if ext in afilter:
                     settings["filter"] = afilter
                     return
@@ -128,20 +137,18 @@ def initial_start_dir(self):
 
     def valid_filters(self):
         if self.data is None or not self.data.is_sparse():
-            return self.filters
+            return self.get_filters()
         else:
-            return {filt: writer for filt, writer in self.filters.items()
+            return {filt: writer for filt, writer in self.get_filters().items()
                     if writer.SUPPORT_SPARSE_DATA}
 
     def default_valid_filter(self):
+        valid = self.valid_filters()
         if self.data is None or not self.data.is_sparse() \
-                or self.filters[self.filter].SUPPORT_SPARSE_DATA:
+                or (self.filter in valid
+                        and valid[self.filter].SUPPORT_SPARSE_DATA):
             return self.filter
-        for filt, writer in self.filters.items():
-            if writer.SUPPORT_SPARSE_DATA:
-                return filt
-        # This shouldn't happen and it will trigger an error in tests
-        return None   # pragma: no cover
+        return next(iter(valid))
 
 
 if __name__ == "__main__":  # pragma: no cover

Orange/widgets/data/tests/test_owsave.py

@@ -8,7 +8,7 @@
 from AnyQt.QtWidgets import QFileDialog
 
 from Orange.data import Table
-from Orange.data.io import TabReader, PickleReader, ExcelReader
+from Orange.data.io import TabReader, PickleReader, ExcelReader, FileFormat
 from Orange.tests import named_file
 from Orange.widgets.data.owsave import OWSave, OWSaveBase
 from Orange.widgets.utils.save.tests.test_owsavebase import \
@@ -23,6 +23,15 @@ def _w(s):  # pylint: disable=invalid-name
     return s.replace("/", os.sep)
 
 
+class MockFormat(FileFormat):
+    EXTENSIONS = ('.mock',)
+    DESCRIPTION = "Mock file format"
+
+    @staticmethod
+    def write_file(filename, data):
+        pass
+
+
 class OWSaveTestBase(WidgetTest, SaveWidgetsTestBaseMixin):
     def setUp(self):
         with open_widget_classes():
@@ -91,7 +100,7 @@ def test_initial_start_dir(self):
     @patch("Orange.widgets.utils.save.owsavebase.QFileDialog.getSaveFileName")
     def test_save_file_sets_name(self, _filedialog):
         widget = self.widget
-        filters = iter(widget.filters)
+        filters = iter(widget.get_filters())
         filter1 = next(filters)
         filter2 = next(filters)
 
@@ -259,20 +268,20 @@ def test_valid_filters_for_sparse(self):
         widget = self.widget
 
         widget.data = None
-        self.assertEqual(widget.filters, widget.valid_filters())
+        self.assertEqual(widget.get_filters(), widget.valid_filters())
 
         widget.data = self.iris
-        self.assertEqual(widget.filters, widget.valid_filters())
+        self.assertEqual(widget.get_filters(), widget.valid_filters())
 
         widget.data.X = sp.csr_matrix(widget.data.X)
         valid = widget.valid_filters()
-        self.assertNotEqual(widget.filters, {})
+        self.assertNotEqual(widget.get_filters(), {})
         # false positive, pylint: disable=no-member
         self.assertTrue(all(v.SUPPORT_SPARSE_DATA for v in valid.values()))
 
     def test_valid_default_filter(self):
         widget = self.widget
-        for widget.filter, writer in widget.filters.items():
+        for widget.filter, writer in widget.get_filters().items():
             if not writer.SUPPORT_SPARSE_DATA:
                 break
 
@@ -284,13 +293,19 @@ def test_valid_default_filter(self):
 
         widget.data.X = sp.csr_matrix(widget.data.X)
         self.assertTrue(
-            widget.filters[widget.default_valid_filter()].SUPPORT_SPARSE_DATA)
+            widget.get_filters()[widget.default_valid_filter()]
+            .SUPPORT_SPARSE_DATA)
+
+    def test_add_on_writers(self):
+        # test adding file formats after registering the widget
+        self.assertIn(MockFormat, self.widget.valid_filters().values())
+        # this test doesn't call it - test_save_uncompressed does
 
     def test_send_report(self):
         widget = self.widget
 
         widget.report_items = Mock()
-        for writer in widget.filters.values():
+        for writer in widget.get_filters().values():
             widget.writer = writer
             for widget.add_type_annotations in (False, True):
                 widget.filename = f"foo.{writer.EXTENSIONS[0]}"
@@ -355,14 +370,14 @@ def test_migration_to_version_2(self):
         settings = {**const_settings,
                     'compress': True, 'compression': 'lzma (.xz)'}
         OWSave.migrate_settings(settings)
-        self.assertTrue(settings["filter"] in OWSave.filters)
+        self.assertTrue(settings["filter"] in OWSave.get_filters())
 
         # Unsupported file format (is this possible?)
         settings = {**const_settings,
                     'compress': True, 'compression': 'lzma (.xz)',
                     'filetype': 'Bar file (.bar)'}
         OWSave.migrate_settings(settings)
-        self.assertTrue(settings["filter"] in OWSave.filters)
+        self.assertTrue(settings["filter"] in OWSave.get_filters())
 
 
 class TestFunctionalOWSave(WidgetTest):
@@ -377,7 +392,7 @@ def test_save_uncompressed(self):
         spiris = Table("iris")
         spiris.X = sp.csr_matrix(spiris.X)
 
-        for selected_filter, writer in widget.filters.items():
+        for selected_filter, writer in widget.get_filters().items():
             widget.write = writer
             ext = writer.EXTENSIONS[0]
             with named_file("", suffix=ext) as filename:
@@ -386,20 +401,22 @@ def test_save_uncompressed(self):
 
                 self.send_signal(widget.Inputs.data, self.iris)
                 widget.save_file_as()
-                self.assertEqual(len(Table(filename)), 150)
+                if hasattr(writer, "read"):
+                    self.assertEqual(len(Table(filename)), 150)
 
                 if writer.SUPPORT_SPARSE_DATA:
                     self.send_signal(widget.Inputs.data, spiris)
                     widget.save_file()
-                    self.assertEqual(len(Table(filename)), 150)
+                    if hasattr(writer, "read"):
+                        self.assertEqual(len(Table(filename)), 150)
 
 
 @unittest.skipUnless(sys.platform == "linux", "Tests for dialog on Linux")
 class TestOWSaveLinuxDialog(OWSaveTestBase):
     def test_get_save_filename_linux(self):
         widget = self.widget
         widget.initial_start_dir = lambda: "baz"
-        widget.filters = dict.fromkeys("abc")
+        widget.get_filters = lambda: dict.fromkeys("abc")
         widget.filter = "b"
         dlg = widget.SaveFileDialog = Mock()  # pylint: disable=invalid-name
         instance = dlg.return_value
@@ -413,7 +430,7 @@ def test_get_save_filename_linux(self):
         instance.exec.return_value = QFileDialog.Rejected
         self.assertEqual(widget.get_save_filename(), ("", ""))
 
-    @patch.object(OWSaveBase, "filters", OWSave.filters)
+    @patch.object(OWSaveBase, "filters", OWSave.get_filters())
     def test_save_file_dialog_enforces_extension_linux(self):
         dialog = OWSave.SaveFileDialog(
             OWSave, None, "Save File", "foo.bar",
@@ -468,7 +485,8 @@ def remove_star(filt):
     def test_get_save_filename_darwin(self, dlg):
         widget = self.widget
         widget.initial_start_dir = lambda: "baz"
-        widget.filters = dict.fromkeys(("aa (*.a)", "bb (*.b)", "cc (*.c)"))
+        widget.get_filters = \
+            lambda: dict.fromkeys(("aa (*.a)", "bb (*.b)", "cc (*.c)"))
         widget.filter = "bb (*.b)"
         instance = dlg.return_value
         instance.exec.return_value = dlg.Accepted = QFileDialog.Accepted
@@ -489,10 +507,10 @@ def test_get_save_filename_darwin(self, dlg):
     def test_save_file_dialog_enforces_extension_darwin(self, dlg):
         widget = self.widget
         filter1 = ""  # prevent pylint warning 'undefined-loop-variable'
-        for filter1 in widget.filters:
+        for filter1 in widget.get_filters():
             if OWSaveBase._extension_from_filter(filter1) == ".tab":
                 break
-        for filter2 in widget.filters:
+        for filter2 in widget.get_filters():
             if OWSaveBase._extension_from_filter(filter2) == ".csv.gz":
                 break
 
@@ -536,7 +554,7 @@ def selected_files():
         widget = self.widget
         widget.initial_start_dir = lambda: "baz"
         filter1 = ""  # prevent pylint warning 'undefined-loop-variable'
-        for filter1 in widget.filters:
+        for filter1 in widget.get_filters():
             if OWSaveBase._extension_from_filter(filter1) == ".tab":
                 break
 

Orange/widgets/utils/save/owsavebase.py

@@ -23,8 +23,9 @@ class OWSaveBase(widget.OWWidget, openclass=True):
       - calls `self.on_new_input`.
 
     - a class attribute `filters` with a list of filters or a dictionary whose
-      keys are filters
-    - method `do_save` that saves `self.data` into `self.filename`.
+      keys are filters OR a class method `get_filters` that returns such a
+      list or dictionary
+    - method `do_save` that saves `self.data` into `self.filename`
 
     Alternatively, instead of defining `do_save` a derived class can make
     `filters` a dictionary whose keys are classes that define a method `write`
@@ -70,7 +71,7 @@ def __init__(self, start_row=0):
         self.data = None
         # This cannot be done outside because `filters` is defined by subclass
         if not self.filter:
-            self.filter = next(iter(self.filters))
+            self.filter = next(iter(self.get_filters()))
 
         self.grid = grid = QGridLayout()
         gui.widgetBox(self.controlArea, orientation=grid)
@@ -89,6 +90,10 @@ def __init__(self, start_row=0):
         self.adjustSize()
         self.update_messages()
 
+    @classmethod
+    def get_filters(cls):
+        return cls.filters
+
     @property
     def writer(self):
         """
@@ -98,7 +103,7 @@ def writer(self):
         corresponding to the filter. Derived classes (e.g. OWSave) may also use
         it elsewhere.
         """
-        return self.filters[self.filter]
+        return self.get_filters()[self.filter]
 
     def on_new_input(self):
         """
@@ -160,8 +165,8 @@ def do_save(self):
         Do the saving.
 
         Default implementation calls the write method of the writer
-        corresponding to the current filter. This requires that class attribute
-        filters is a dictionary whose keys are classes.
+        corresponding to the current filter. This requires that get_filters()
+        returns is a dictionary whose keys are classes.
 
         Derived classes may simplify this by providing a list of filters and
         override do_save. This is particularly handy if the widget supports only
@@ -217,7 +222,7 @@ def _replace_extension(cls, filename, extension):
         function removes anything that can appear anywhere.
         """
         known_extensions = set()
-        for filt in cls.filters:
+        for filt in cls.get_filters():
             known_extensions |= set(cls._extension_from_filter(filt).split("."))
         if "" in known_extensions:
             known_extensions.remove("")
@@ -233,7 +238,7 @@ def _extension_from_filter(selected_filter):
         return re.search(r".*\(\*?(\..*)\)$", selected_filter).group(1)
 
     def valid_filters(self):
-        return self.filters
+        return self.get_filters()
 
     def default_valid_filter(self):
         return self.filter

Orange/widgets/utils/save/tests/test_owsavebase.py

@@ -36,10 +36,11 @@ def test_input_handler(self):
         widget.on_new_input.assert_called()
 
     def test_filters(self):
-        self.assertGreaterEqual(len(self.widget.filters), 1,
+        filters = self.widget.get_filters()
+        self.assertGreaterEqual(len(filters), 1,
                                 msg="Widget defines no filters")
         if type(self.widget).do_save is OWSaveBase.do_save:
-            self.assertIsInstance(self.widget.filters, collections.abc.Mapping)
+            self.assertIsInstance(filters, collections.abc.Mapping)
 
 
 class TestOWSaveBaseWithWriters(WidgetTest):
@@ -92,7 +93,7 @@ def test_base_methods(self):
         self.assertEqual(widget.initial_start_dir(),
                          os.path.expanduser(f"~{os.sep}"))
         self.assertEqual(widget.suggested_name(), "")
-        self.assertIs(widget.valid_filters(), widget.filters)
+        self.assertIs(widget.valid_filters(), widget.get_filters())
         self.assertIs(widget.default_valid_filter(), widget.filter)
 
 
@@ -133,7 +134,7 @@ def test_base_methods(self):
         self.assertEqual(widget.initial_start_dir(),
                          os.path.expanduser(f"~{os.sep}"))
         self.assertEqual(widget.suggested_name(), "")
-        self.assertIs(widget.valid_filters(), widget.filters)
+        self.assertIs(widget.valid_filters(), widget.get_filters())
         self.assertIs(widget.default_valid_filter(), widget.filter)