Sac 28668 fix transform and schema (#177)

* handle empty arrays and fields that could be either formatted or nested

* remove ipdb

* bump version and add changelog entry

* handle string parsing similar to existin tap-s3 logic

* fix syntax error

* fix bad tests

Author: skuttleman

CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## 6.2.1
+  * Fixes json schema generation to not treat numbers as dates
+  * Fixes json schema generation to handle empty arrays
+  * Fixes record transformation to handle fields that could be either formatted string or nested data structure
+  * [#177](https://github.com/singer-io/singer-python/pull/177)
+
 ## 6.2.0
   * Adds json schema generation [#175](https://github.com/singer-io/singer-python/pull/175)
 

setup.py

@@ -4,7 +4,7 @@
 import subprocess
 
 setup(name="singer-python",
-      version='6.2.0',
+      version='6.2.1',
       description="Singer.io utility library",
       author="Stitch",
       classifiers=['Programming Language :: Python :: 3 :: Only'],

singer/schema_generation.py

@@ -18,19 +18,33 @@ def add_observations(acc, path, data):
         for key in data:
             add_observations(acc, path + ["object", key], data[key])
     elif isinstance(data, list):
+        if len(data) == 0:
+            add_observations(acc, path + ["array"], None)
         for item in data:
             add_observations(acc, path + ["array"], item)
     elif isinstance(data, str):
-        # If the string parses as a date, add an observation that its a date
         try:
-            data = dateutil.parser.parse(data)
-        except (dateutil.parser.ParserError, OverflowError):
-            data = None
-        if data:
+            # If the string parses as a int, add an observation that it's a integer
+            int(data)
+            add_observation(acc, path + ["integer"])
+            return acc
+        except (ValueError, TypeError):
+            pass
+        try:
+            # If the string parses as a float, add an observation that it's a number
+            float(data)
+            add_observation(acc, path + ["number"])
+            return acc
+        except (ValueError, TypeError):
+            pass
+        try:
+            # If the string parses as a date, add an observation that it's a date
+            dateutil.parser.parse(data)
             add_observation(acc, path + ["date"])
-        else:
-            add_observation(acc, path + ["string"])
-
+            return acc
+        except (dateutil.parser.ParserError, OverflowError):
+            pass
+        add_observation(acc, path + ["string"])
     elif isinstance(data, bool):
         add_observation(acc, path + ["boolean"])
     elif isinstance(data, int):

singer/transform.py

@@ -266,13 +266,13 @@ def _transform(self, data, typ, schema, path):
             else:
                 return False, None
 
-        elif schema.get("format") == "date-time":
+        elif typ == "string" and schema.get("format") == "date-time":
             data = self._transform_datetime(data)
             if data is None:
                 return False, None
 
             return True, data
-        elif schema.get("format") == "singer.decimal":
+        elif typ == "string" and schema.get("format") == "singer.decimal":
             if data is None:
                 return False, None
 

tests/test_transform.py

@@ -25,7 +25,7 @@ def test_nested_transform(self):
 
     def test_multi_type_object_transform(self):
         schema =  {"type": ["null", "object", "string"],
-                   "properties": {"whatever": {"type": "date-time",
+                   "properties": {"whatever": {"type": "string",
                                                "format": "date-time"}}}
         data = {"whatever": "2017-01-01"}
         expected = {"whatever": "2017-01-01T00:00:00.000000Z"}
@@ -36,7 +36,7 @@ def test_multi_type_object_transform(self):
 
     def test_multi_type_array_transform(self):
         schema =  {"type": ["null", "array", "integer"],
-                   "items": {"type": "date-time", "format": "date-time"}}
+                   "items": {"type": "string", "format": "date-time"}}
         data = ["2017-01-01"]
         expected = ["2017-01-01T00:00:00.000000Z"]
         self.assertEqual(expected, transform(data, schema))