feat: Update openapi transformers

Author: yevgenypats

cloudquery/sdk/schema/column.py

@@ -23,6 +23,23 @@ def __init__(
         self.not_null = not_null
         self.incremental_key = incremental_key
         self.unique = unique
+    
+    def __str__(self) -> str:
+        return f"Column(name={self.name}, type={self.type}, description={self.description}, primary_key={self.primary_key}, not_null={self.not_null}, incremental_key={self.incremental_key}, unique={self.unique})"
+    
+    def __repr__(self) -> str:
+        return f"Column(name={self.name}, type={self.type}, description={self.description}, primary_key={self.primary_key}, not_null={self.not_null}, incremental_key={self.incremental_key}, unique={self.unique})"
+    
+    def __eq__(self, __value: object) -> bool:
+        if type(__value) == Column:
+            return (self.name == __value.name 
+                    and self.type == __value.type 
+                    and self.description == __value.description 
+                    and self.primary_key == __value.primary_key 
+                    and self.not_null == __value.not_null 
+                    and self.incremental_key == __value.incremental_key 
+                    and self.unique == __value.unique)
+        return False
 
     def to_arrow_field(self):
         metadata = {

cloudquery/sdk/transformers/__init__.py

@@ -1 +1,2 @@
 from .transformers import transform_list_of_dict
+from .openapi import oapi_definition_to_columns

cloudquery/sdk/transformers/openapi.py

@@ -1,5 +1,6 @@
 from typing import Dict, List
 import pyarrow as pa
+from cloudquery.sdk.types import JSONType
 from cloudquery.sdk.schema import Column
 
 
@@ -9,16 +10,25 @@ def oapi_type_to_arrow_type(field) -> pa.DataType:
         return pa.string()
     elif oapi_type == "number":
         return pa.int64()
+    elif oapi_type == "integer":
+        return pa.int64()
     elif oapi_type == "boolean":
         return pa.bool_()
+    elif oapi_type == "array":
+        return JSONType()
+    elif oapi_type == "object":
+        return JSONType()
+    elif oapi_type is None and "$ref" in field:
+        return JSONType()
     else:
         return pa.string()
 
 
-def oapi_properties_to_columns(properties: Dict) -> List[Column]:
+def oapi_definition_to_columns(definition: Dict) -> List[Column]:
     columns = []
-    for key, value in properties.items():
+    for key, value in definition["properties"].items():
+        column_type = oapi_type_to_arrow_type(value)
         columns.append(
-            Column(name=key, type=value, description=value.get("description"))
+            Column(name=key, type=column_type, description=value.get("description"))
         )
     return columns

tests/transformers/__init__.py

@@ -0,0 +1,57 @@
+import pyarrow as pa
+from cloudquery.sdk.transformers import oapi_definition_to_columns
+from cloudquery.sdk.schema import Column
+from cloudquery.sdk.types import JSONType
+
+OAPI_SPEC = {
+    "swagger": "2.0",
+    "info": {
+        "version": "2.0",
+        "title": "Test API",
+        "description": "Unit tests APIs",
+    },
+    "host": "cloudquery.io",
+    "schemes": ["https"],
+    "consumes": ["application/json"],
+    "produces": ["application/json"],
+    "paths": {},
+    "definitions": {
+        "TestDefinition": {
+            "type": "object",
+            "properties": {
+                "string": {
+                    "type": "string",
+                },
+                "number": {
+                    "type": "number",
+                },
+                "integer": {
+                    "type": "integer",
+                },
+                "boolean": {
+                    "type": "boolean",
+                },
+                "object": {
+                    "$ref": "#/definitions/SomeDefinition",
+                },
+                "array": {
+                    "type": "array",
+                    "items": {"$ref": "#/definitions/SomeDefinition"},  
+                },
+            },
+        },
+    },
+}
+
+
+def test_oapi_properties_to_columns():
+    expected_columns = [
+        Column("string", pa.string() , description=None),
+        Column("number", pa.int64() , description=None),
+        Column("integer", pa.int64() , description=None),
+        Column("boolean", pa.bool_() , description=None),
+        Column("object", JSONType() , description=None),
+        Column("array", JSONType() , description=None),
+    ]
+    columns = oapi_definition_to_columns(OAPI_SPEC["definitions"]["TestDefinition"])
+    assert expected_columns == columns