Merge pull request #386 from mpsonntag/dictParseIgnore

achilleas-k · web-flow · commit 1083db44a4e2 · 2020-04-27T13:47:49.000+02:00
Implement ignore_errors for DictParser.Reader

LGTM
diff --git a/odml/format.py b/odml/format.py
@@ -82,7 +82,8 @@ def revmap(self, name):
             else:
                 for k, val in self._map.items():
                     self._rev_map[val] = k
-        return self._rev_map.get(name, name)
+
+        return self._rev_map.get(name)
 
     def __iter__(self):
         """ Iterates each python property name """
diff --git a/odml/tools/dict_parser.py b/odml/tools/dict_parser.py
@@ -2,11 +2,15 @@
 The dict_parser module provides access to the DictWriter and DictReader class.
 Both handle the conversion of odML documents from and to Python dictionary objects.
 """
+import sys
 
 from .. import format as odmlfmt
 from ..info import FORMAT_VERSION
 from .parser_utils import InvalidVersionException, ParserException, odml_tuple_export
 
+LABEL_ERROR = "Error"
+LABEL_WARNING = "Warning"
+
 
 def parse_cardinality(vals):
     """
@@ -169,16 +173,22 @@ class DictReader:
     A reader to parse dictionaries with odML content into an odml.Document.
     """
 
-    def __init__(self, show_warnings=True):
+    def __init__(self, show_warnings=True, ignore_errors=False):
         """
         :param show_warnings: Toggle whether to print warnings to the command line.
                               Any warnings can be accessed via the Reader's class
                               warnings attribute after parsing is done.
+        :param ignore_errors: To allow loading and fixing of invalid odml files
+                              encountered errors can be converted to warnings
+                              instead. Such a document can only be saved when
+                              all errors have been addressed though.
         """
         self.parsed_doc = None  # Python dictionary object equivalent
-        self.show_warnings = show_warnings
         self.warnings = []
 
+        self.show_warnings = show_warnings
+        self.ignore_errors = ignore_errors
+
     def is_valid_attribute(self, attr, fmt):
         """
         Checks whether a provided attribute is valid for a provided odml class
@@ -196,13 +206,40 @@ def is_valid_attribute(self, attr, fmt):
         if fmt.revmap(attr):
             return attr
 
-        msg = "Invalid element <%s> inside <%s> tag" % (attr, fmt.__class__.__name__)
-        self.warnings.append(msg)
-        if self.show_warnings:
-            print(msg)
+        msg = "Invalid element '%s' inside <%s> tag" % (attr, fmt.__class__.__name__)
+        self.error(msg)
 
         return None
 
+    def error(self, msg):
+        """
+        If the parsers ignore_errors property is set to False, a ParserException
+        will be raised. Otherwise the message is passed to the parsers warning
+        method.
+
+        :param msg: Error message.
+        """
+        if self.ignore_errors:
+            return self.warn(msg, LABEL_ERROR)
+
+        raise ParserException(msg)
+
+    def warn(self, msg, label=LABEL_WARNING):
+        """
+        Adds a message to the parsers warnings property. If the parsers show_warnings
+        property is set to True, an additional error message will be written
+        to sys.stderr.
+
+        :param msg: Warning message.
+        :param label: Defined message level, can be 'Error' or 'Warning'. Default is 'Warning'.
+        """
+        msg = "%s: %s" % (label, msg)
+
+        self.warnings.append(msg)
+
+        if self.show_warnings:
+            sys.stderr.write("Parser%s\n" % msg)
+
     def to_odml(self, parsed_doc):
         """
         Parses a Python dictionary object containing an odML document to an odml.Document.
@@ -280,14 +317,19 @@ def parse_sections(self, section_list):
                     # Make sure to always use the correct odml format attribute name
                     sec_attrs[odmlfmt.Section.map(attr)] = content
 
-            sec = odmlfmt.Section.create(**sec_attrs)
-            for prop in sec_props:
-                sec.append(prop)
+            try:
+                sec = odmlfmt.Section.create(**sec_attrs)
+
+                for prop in sec_props:
+                    sec.append(prop)
 
-            for child_sec in children_secs:
-                sec.append(child_sec)
+                for child_sec in children_secs:
+                    sec.append(child_sec)
 
-            odml_sections.append(sec)
+                odml_sections.append(sec)
+            except Exception as exc:
+                msg = "Section not created (%s)\n  %s" % (sec_attrs, str(exc))
+                self.error(msg)
 
         return odml_sections
 
@@ -316,7 +358,11 @@ def parse_properties(self, props_list):
                     # Make sure to always use the correct odml format attribute name
                     prop_attrs[odmlfmt.Property.map(attr)] = content
 
-            prop = odmlfmt.Property.create(**prop_attrs)
-            odml_props.append(prop)
+            try:
+                prop = odmlfmt.Property.create(**prop_attrs)
+                odml_props.append(prop)
+            except Exception as exc:
+                msg = "Property not created (%s)\n%s" % (prop_attrs, str(exc))
+                self.error(msg)
 
         return odml_props
diff --git a/odml/tools/odmlparser.py b/odml/tools/odmlparser.py
@@ -183,7 +183,8 @@ def from_file(self, file, doc_format=None):
                     print(err)
                     return None
 
-            par = DictReader(show_warnings=self.show_warnings)
+            par = DictReader(ignore_errors=True,
+                             show_warnings=self.show_warnings)
             self.doc = par.to_odml(self.parsed_doc)
             # Provide original file name via the in memory document
             self.doc.origin_file_name = basename(file)
diff --git a/test/test_parser_json.py b/test/test_parser_json.py
@@ -1,18 +1,140 @@
+"""
+This module supplies basic tests for the odml.dict_parser.DictReader
+reading from json files.
+"""
+
 import json
 import os
+import tempfile
 import unittest
 
 from odml.tools import dict_parser
 from odml.tools.parser_utils import ParserException, InvalidVersionException
 
 
+_INVALID_ATTRIBUTE_HANDLING_DOC = """
+{
+  "Document": {
+    "id": "6af2a3ec-9f3a-42d6-a59d-95f3ccbaa383",
+    "%s": "i_do_not_exist_on_doc_level",
+    "sections": [
+      {
+        "id": "51f3c79c-a7d7-471e-be16-e085caa851fb",
+        "%s": "i_do_not_exist_on_sec_level",
+        "type": "test",
+        "name": "sec",
+        "sections": [],
+        "properties": [
+          {
+            "id": "c5aed35a-d9ff-437d-82d6-68f0cda8ea94",
+            "%s": "i_do_not_exist_on_prop_level",
+            "name": "prop",
+            "value": [
+              1,
+              2,
+              3
+            ],
+            "type": "int"
+          }
+        ]
+      }
+    ]
+  },
+  "odml-version": "1.1"
+}
+""".strip()
+
+_SEC_CREATION_ERROR_DOC = """
+{
+  "Document": {
+    "id": "6af2a3ec-9f3a-42d6-a59d-95f3ccbaa383",
+    "sections": [
+      {
+        "id": "1113c79c-a7d7-471e-be16-e085caa851fb",
+        "name": "sec",
+        "type": "test",
+        "sections": [
+          {
+            "id": "1213c79c-a7d7-471e-be16-e085caa851fb",
+            "name": "%s",
+            "type": "test"
+          },
+          {
+            "id": [
+              "I-am-so-kaputt"
+            ],
+            "name": "%s",
+            "type": "test"
+          }
+        ]
+      }
+    ]
+  },
+  "odml-version": "1.1"
+}
+""".strip()
+
+_PROP_CREATION_ERROR_DOC = """
+{
+  "Document": {
+    "id": "6af2a3ec-9f3a-42d6-a59d-95f3ccbaa383",
+    "sections": [
+      {
+        "id": "51f3c79c-a7d7-471e-be16-e085caa851fb",
+        "type": "test",
+        "name": "sec",
+        "properties": [
+          {
+            "id": "121ed35a-d9ff-437d-82d6-68f0cda8ea94",
+            "name": "valid_prop"
+          },
+          {
+            "id": "122ed35a-d9ff-437d-82d6-68f0cda8ea94",
+            "name": "invalid_prop",
+            "value": [
+              "a",
+              "b"
+            ],
+            "type": "int"
+          }
+        ]
+      }
+    ]
+  },
+  "odml-version": "1.1"
+}
+""".strip()
+
+
 class TestJSONParser(unittest.TestCase):
 
     def setUp(self):
         dir_path = os.path.dirname(os.path.realpath(__file__))
         self.basepath = os.path.join(dir_path, "resources")
 
-        self.json_reader = dict_parser.DictReader()
+        self.json_reader = dict_parser.DictReader(show_warnings=False)
+
+        dir_name = os.path.basename(os.path.splitext(__file__)[0])
+        tmp_base_path = os.path.join(tempfile.gettempdir(), "odml_test")
+        if not os.path.exists(tmp_base_path):
+            os.mkdir(tmp_base_path)
+
+        tmp_dir_path = os.path.join(tmp_base_path, dir_name)
+        if not os.path.exists(tmp_dir_path):
+            os.mkdir(tmp_dir_path)
+
+        self.tmp_dir_path = tmp_dir_path
+
+    def _prepare_doc(self, file_name, file_content):
+        file_path = os.path.join(self.tmp_dir_path, file_name)
+
+        with open(file_path, "w") as dump_file:
+            dump_file.write(file_content)
+
+        with open(file_path) as raw_data:
+            parsed_doc = json.load(raw_data)
+
+        return parsed_doc
 
     def test_missing_root(self):
         filename = "missing_root.json"
@@ -46,3 +168,83 @@ def test_invalid_version(self):
 
         with self.assertRaises(InvalidVersionException):
             _ = self.json_reader.to_odml(parsed_doc)
+
+    def test_invalid_attribute_handling(self):
+        filename = "invalid_attributes.yaml"
+
+        inv_doc_attr = "invalid_doc_attr"
+        inv_sec_attr = "invalid_sec_attr"
+        inv_prop_attr = "invalid_prop_attr"
+
+        file_content = _INVALID_ATTRIBUTE_HANDLING_DOC % (inv_doc_attr, inv_sec_attr, inv_prop_attr)
+        parsed_doc = self._prepare_doc(filename, file_content)
+
+        # Test ParserException on default parse
+        exc_msg = "Invalid element"
+        with self.assertRaises(ParserException) as exc:
+            _ = self.json_reader.to_odml(parsed_doc)
+        self.assertIn(exc_msg, str(exc.exception))
+
+        # Test success on ignore_error parse
+        self.json_reader.ignore_errors = True
+        doc = self.json_reader.to_odml(parsed_doc)
+
+        self.assertEqual(len(doc.sections), 1)
+        self.assertEqual(len(doc.sections["sec"].properties), 1)
+
+        self.assertEqual(len(self.json_reader.warnings), 3)
+        for msg in self.json_reader.warnings:
+            self.assertIn("Error", msg)
+            self.assertTrue(inv_doc_attr in msg or inv_sec_attr in msg or inv_prop_attr in msg)
+
+    def test_sec_creation_error(self):
+        filename = "broken_section.yaml"
+
+        valid = "valid_sec"
+        invalid = "invalid_sec"
+
+        file_content = _SEC_CREATION_ERROR_DOC % (valid, invalid)
+        parsed_doc = self._prepare_doc(filename, file_content)
+
+        # Test ParserException on default parse
+        exc_msg = "Section not created"
+        with self.assertRaises(ParserException) as exc:
+            _ = self.json_reader.to_odml(parsed_doc)
+        self.assertIn(exc_msg, str(exc.exception))
+
+        # Test success on ignore_error parse
+        self.json_reader.ignore_errors = True
+        doc = self.json_reader.to_odml(parsed_doc)
+
+        self.assertEqual(len(doc.sections["sec"].sections), 1)
+        self.assertIn(valid, doc.sections["sec"].sections)
+        self.assertNotIn(invalid, doc.sections["sec"].sections)
+
+        self.assertEqual(len(self.json_reader.warnings), 1)
+        for msg in self.json_reader.warnings:
+            self.assertIn("Error", msg)
+            self.assertIn(exc_msg, msg)
+
+    def test_prop_creation_error(self):
+        filename = "broken_property.yaml"
+
+        parsed_doc = self._prepare_doc(filename, _PROP_CREATION_ERROR_DOC)
+
+        # Test ParserException on default parse
+        exc_msg = "Property not created"
+        with self.assertRaises(ParserException) as exc:
+            _ = self.json_reader.to_odml(parsed_doc)
+        self.assertIn(exc_msg, str(exc.exception))
+
+        # Test success on ignore_error parse
+        self.json_reader.ignore_errors = True
+        doc = self.json_reader.to_odml(parsed_doc)
+
+        self.assertEqual(len(doc.sections["sec"].properties), 1)
+        self.assertIn("valid_prop", doc.sections["sec"].properties)
+        self.assertNotIn("invalid_prop", doc.sections["sec"].properties)
+
+        self.assertEqual(len(self.json_reader.warnings), 1)
+        for msg in self.json_reader.warnings:
+            self.assertIn("Error", msg)
+            self.assertIn(exc_msg, msg)
diff --git a/test/test_parser_yaml.py b/test/test_parser_yaml.py
diff --git a/test/test_terminology.py b/test/test_terminology.py
