SAC-28668: update schema generation v6 (#180)

* use `anyOf` when multiple types are found

* fix test

* Update schema generation and bump version for v6 deploy

Co-authored-by: Bryant Gray <[email protected]>
Co-authored-by: Andres Pineda <[email protected]>

* Grab error list changes from v5

Co-authored-by: Bryant Gray <[email protected]>
Co-authored-by: Andres Pineda <[email protected]>

* Fix linting error

Co-authored-by: Bryant Gray <[email protected]>

---------

Co-authored-by: Bryant Gray <[email protected]>
Co-authored-by: Andres Pineda <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 6.2.2
+  * Updates json schema generation to not emit dates
+  * Handle multiple schemas with anyOf and emit them in a specific order
+  * Do not emit error messages when checking multiple schemas and a subsequent schema passes
+  * [#179](https://github.com/singer-io/singer-python/pull/179)
+
 ## 6.2.1
   * Fixes json schema generation to not treat numbers as dates
   * Fixes json schema generation to handle empty arrays

setup.py

@@ -4,7 +4,7 @@
 import subprocess
 
 setup(name="singer-python",
-      version='6.2.1',
+      version='6.2.2',
       description="Singer.io utility library",
       author="Stitch",
       classifiers=['Programming Language :: Python :: 3 :: Only'],

singer/schema_generation.py

@@ -1,6 +1,3 @@
-import dateutil.parser
-
-
 def add_observation(acc, path):
 
     node = acc
@@ -37,13 +34,6 @@ def add_observations(acc, path, data):
             return acc
         except (ValueError, TypeError):
             pass
-        try:
-            # If the string parses as a date, add an observation that it's a date
-            dateutil.parser.parse(data)
-            add_observation(acc, path + ["date"])
-            return acc
-        except (dateutil.parser.ParserError, OverflowError):
-            pass
         add_observation(acc, path + ["string"])
     elif isinstance(data, bool):
         add_observation(acc, path + ["boolean"])
@@ -59,9 +49,13 @@ def add_observations(acc, path, data):
     return acc
 
 def to_json_schema(obs):
-    result = {'type': ['null']}
+    types = []
+    # add schema types in a specific order to anyOf list
+    for key in ['array', 'object', 'number', 'integer', 'boolean', 'string', 'null']:
+        if key not in obs:
+            continue
 
-    for key in obs:
+        result = {'type': ['null']}
 
         if key == 'object':
             result['type'] += ['object']
@@ -74,9 +68,6 @@ def to_json_schema(obs):
             result['type'] += ['array']
             result['items'] = to_json_schema(obs['array'])
 
-        elif key == 'date':
-            result['type'] += ['string']
-            result['format'] = 'date-time'
         elif key == 'string':
             result['type'] += ['string']
 
@@ -97,7 +88,15 @@ def to_json_schema(obs):
         else:
             raise Exception("Unexpected data type " + key)
 
-    return result
+        types.append(result)
+
+    if len(types) == 0:
+        return {'type': ['null', 'string']}
+
+    if len(types) == 1:
+        return types[0]
+
+    return {'anyOf': types}
 
 def generate_schema(records):
     obs = {}

singer/transform.py

@@ -185,6 +185,8 @@ def _transform_anyof(self, data, schema, path):
             success, transformed_data = self.transform_recur(data, subschema, path)
             if success:
                 return success, transformed_data
+            else:
+                self.errors.pop()
         else: # pylint: disable=useless-else-on-loop
             # exhaused all schemas and didn't return, so we failed :-(
             self.errors.append(Error(path, data, schema, logging_level=LOGGER.level))

tests/test_schema_generation.py

@@ -10,7 +10,7 @@ def test_simple_schema(self):
                 'a': {'type': ['null', 'integer']},
                 'b': {'type': ['null', 'string']},
                 'c': {'type': ['null', 'boolean']},
-                'dt': {'type': ['null', 'string'], 'format': 'date-time'}
+                'dt': {'type': ['null', 'string']}
             }
         }
         self.assertEqual(expected_schema, generate_schema(records))
@@ -23,24 +23,20 @@ def test_mix_n_match_records_schema(self):
         ]
         expected_schema = {
             'type': ['null', 'object'],
-            'properties': {
-                'a': {'type': {'null', 'integer', 'string', 'boolean'}},
-                'b': {'type': ['null', 'string']},
-                'c': {'type': {'null', 'integer', 'string'}, 'format': 'singer.decimal'},
-                'd': {
-                    'type': {'null', 'array', 'object'},
-                    'items': {'type': {'null', 'integer', 'string'}},
-                    'properties': {'one': {'type': ['null', 'integer']},
-                                   'two': {'type': ['null', 'string']}}
-
-                }
-            }
+            'properties': {'a': {'anyOf': [{'type': ['null', 'integer']},
+                                           {'type': ['null', 'boolean']},
+                                           {'type': ['null', 'string']}]},
+                           'b': {'type': ['null', 'string']},
+                           'c': {'anyOf': [{'type': ['null', 'string'], 'format': 'singer.decimal'},
+                                           {'type': ['null', 'integer']}]},
+                'd': {'anyOf': [{'type': ['null', 'array'],
+                                 'items': {'anyOf': [{'type': ['null', 'integer']},
+                                                     {'type': ['null', 'string']}]}},
+                                {'type': ['null', 'object'],
+                                 'properties': {'one': {'type': ['null', 'integer']},
+                                                'two': {'type': ['null', 'string']}}}]}}
         }
         actual_schema = generate_schema(records)
-        actual_schema['properties']['a']['type'] = set(actual_schema['properties']['a']['type'])
-        actual_schema['properties']['c']['type'] = set(actual_schema['properties']['c']['type'])
-        actual_schema['properties']['d']['type'] = set(actual_schema['properties']['d']['type'])
-        actual_schema['properties']['d']['items']['type'] = set(actual_schema['properties']['d']['items']['type'])
         self.assertEqual(expected_schema, actual_schema)
 
     def test_nested_structue_schema(self):