Type bug fix and regression test for types (#65)

* Adds a conversion function for json.dumps that handles unsupported types.

* Adds regression tests for all types.

Author: loganripplinger

README.md

@@ -128,15 +128,15 @@ TABLESCHEMA_TO_SQLALCHEMY_TYPES = {
     'integer': Integer,
     'boolean': Boolean,
     'object': String,
-    'array': String,
+    'array': String, # Not supported yet
     'date': Date,
-    'time': DateTime,
+    'time': DateTime, # Not supported yet
     'datetime': DateTime,
     'year': Integer,
-    'yearmonth': Integer,
+    'yearmonth': Integer, # Not supported yet
     'duration': Integer,
     'geopoint': String,
-    'geojson': String,
+    'geojson': String, # Not supported yet
     'any': String
 }
 ```

data_resource_api/app/data_managers/data_resource_manager.py

@@ -6,12 +6,13 @@
 """
 from threading import Thread
 from time import sleep
-from flask import Flask
+from flask import Flask, make_response
 from flask_restful import Api, Resource
 from data_resource_api.factories import DataResourceFactory
 from data_resource_api.db import Base, Session, Checksum
 from data_resource_api.app.utils.exception_handler import handle_errors
 from data_resource_api.app.utils.descriptor import Descriptor
+from data_resource_api.app.utils.json_converter import safe_json_dumps
 from data_resource_api.utils import exponential_backoff
 from data_resource_api.app.data_managers.data_manager import DataManager
 
@@ -156,6 +157,12 @@ def create_app(self):
                               '/', endpoint='all_services_ep')
         self.app.register_error_handler(Exception, handle_errors)
 
+        @self.api.representation('application/json')
+        def output_json(data, code, headers=None):
+            resp = make_response(safe_json_dumps(data), code)
+            resp.headers.extend(headers or {})
+            return resp
+
         return self.app
 
     def process_descriptor(self, descriptor: Descriptor):

data_resource_api/app/utils/json_converter.py

@@ -0,0 +1,15 @@
+import json
+import datetime
+
+
+def unknown_field_json_converter(o):
+    if isinstance(o, datetime.datetime):
+        return str(o.isoformat()) + "Z"
+    if isinstance(o, datetime.date):
+        return str(o.isoformat())
+
+
+def safe_json_dumps(json_dict):
+    """This will add explicit conversions for types to json.dumps
+    """
+    return json.dumps(json_dict, default=unknown_field_json_converter)

tests/conftest.py

@@ -12,7 +12,8 @@
     skills_descriptor,
     credentials_descriptor,
     programs_descriptor,
-    json_descriptor)
+    json_descriptor,
+    everything_descriptor)
 
 from sqlalchemy.ext.declarative import declarative_base
 from data_resource_api.logging import LogFactory
@@ -209,11 +210,18 @@ def json_client():
 
 
 @pytest.fixture(scope='module')
-def no_db_dmm():
-    dmm = DataModelManagerSync(
-        use_local_dirs=False,
-        descriptors=None)
-    yield dmm
+def everything_client():
+    client = Client([everything_descriptor])
+    yield client.run_and_return_test_client()
+    client.stop_container()
+
+
+# @pytest.fixture(scope='module')
+# def no_db_dmm():
+#     dmm = DataModelManagerSync(
+#         use_local_dirs=False,
+#         descriptors=None)
+#     yield dmm
 
 
 @pytest.fixture

tests/end_to_end/test_all_types.py

@@ -0,0 +1,201 @@
+import pytest
+import json
+from tests.service import ApiHelper
+from expects import expect, be_an, raise_error, have_property, equal, be_empty
+
+
+ROUTE = '/alltypes'
+
+
+def run_query(client, key, value, expected_value=None):
+    if not expected_value:
+        expected_value = value
+    post_body = {
+        key: value
+    }
+    id_ = ApiHelper.everything_post(ROUTE, client, post_body)
+    resp_data = ApiHelper.everything_get(ROUTE, client, id_)
+    resp = json.loads(resp_data)
+
+    # response = json.dumps(resp, sort_keys=True)
+    # expected_output = json.dumps(expected_string, sort_keys=True)
+    expect(resp[key]).to(equal(expected_value))
+    assert type(resp[key]) == type(expected_value)
+
+
+def test_string(everything_client):
+    # {
+    #     "name": "string",
+    #     "title": "string",
+    #     "type": "string",
+    #     "required": False
+    # },
+    run_query(everything_client, "string", "asdf1234")
+
+
+# @pytest.mark.xfail
+def test_number(everything_client):
+    # {
+    #     "name": "number",
+    #     "title": "number",
+    #     "type": "number",
+    #     "required": False
+    # },
+    run_query(everything_client, "number", 1234.0)
+    run_query(everything_client, "number", 1234, 1234.0)
+
+
+# @pytest.mark.xfail
+def test_integer(everything_client):
+    # {
+    #     "name": "integer",
+    #     "title": "integer",
+    #     "type": "integer",
+    #     "required": False
+    # },
+    run_query(everything_client, "integer", 1234)
+
+
+# @pytest.mark.xfail
+def test_boolean(everything_client):
+    # {
+    #     "name": "boolean",
+    #     "title": "boolean",
+    #     "type": "boolean",
+    #     "required": False
+    # },
+    run_query(everything_client, "boolean", False)
+    run_query(everything_client, "boolean", True)
+
+
+# @pytest.mark.xfail
+def test_object(everything_client):
+    # {
+    #     "name": "object",
+    #     "title": "object",
+    #     "type": "object",
+    #     "required": False
+    # },
+    run_query(everything_client, "object", {"json": "test"})
+
+
+@pytest.mark.skip  # Unsure how this should return
+def test_array(everything_client):
+    # {
+    #     "name": "array",
+    #     "title": "array",
+    #     "type": "array",
+    #     "required": False
+    # },
+    run_query(everything_client, "array", ['one', 'two', 'three'])
+
+
+# @pytest.mark.xfail
+def test_date(everything_client):
+    # {
+    #     "name": "date",
+    #     "title": "date",
+    #     "type": "date",
+    #     "required": False
+    # },
+    run_query(everything_client, "date", "2012-04-23")
+
+
+@pytest.mark.xfail  # TODO cannot save to database
+def test_time(everything_client):
+    # {
+    #     "name": "time",
+    #     "title": "time",
+    #     "type": "time",
+    #     "required": False
+    # },
+    run_query(everything_client, "time", "18:25:43.511Z")
+
+
+# @pytest.mark.xfail
+def test_datetime(everything_client):
+    # {
+    #     "name": "datetime",
+    #     "title": "datetime",
+    #     "type": "datetime",
+    #     "required": False
+    # },
+    run_query(everything_client, "datetime", "2012-04-23T18:25:43Z")
+
+
+@pytest.mark.xfail
+def test_datetime_with_miliseconds(everything_client):
+    # {
+    #     "name": "datetime",
+    #     "title": "datetime",
+    #     "type": "datetime",
+    #     "required": False
+    # },
+    run_query(everything_client, "datetime", "2012-04-23T18:25:43.511Z")
+
+
+# @pytest.mark.xfail
+def test_year(everything_client):
+    # {
+    #     "name": "year",
+    #     "title": "year",
+    #     "type": "year",
+    #     "required": False
+    # },
+    run_query(everything_client, "year", 2012)
+
+
+@pytest.mark.xfail  # TODO does not save to database
+def test_yearmonth(everything_client):
+    # {
+    #     "name": "yearmonth",
+    #     "title": "yearmonth",
+    #     "type": "yearmonth",
+    #     "required": False
+    # },
+    run_query(everything_client, "yearmonth", "2012-11")
+
+
+# @pytest.mark.xfail
+def test_duration(everything_client):
+    # {
+    #     "name": "duration",
+    #     "title": "duration",
+    #     "type": "duration",
+    #     "required": False
+    # },
+    run_query(everything_client, "duration", 11)
+
+
+# @pytest.mark.xfail
+def test_geopoint(everything_client):
+    # {
+    #     "name": "geopoint",
+    #     "title": "geopoint",
+    #     "type": "geopoint",
+    #     "required": False
+    # },
+    run_query(everything_client, "geopoint", "41.12,-71.34")
+
+
+# @pytest.mark.xfail
+def test_geojson(everything_client):
+    # {
+    #     "name": "geojson",
+    #     "title": "geojson",
+    #     "type": "geojson",
+    #     "required": False
+    # },
+    run_query(everything_client, "geopoint", "41.12,-71.34")
+    # https://geojson.org/ # TODO we don't support this
+
+
+# @pytest.mark.xfail
+def test_any(everything_client):
+    # {
+    #     "name": "any",
+    #     "title": "any",
+    #     "type": "any",
+    #     "required": False
+    # }
+    run_query(everything_client, "any", "asdf1234")