Fix: Expand default decimal size to (38, 9), resolve issues with large number values (#194)

Author: aaronsteers

airbyte/_processors/sql/duckdb.py

@@ -94,12 +94,13 @@ def _write_files_to_new_table(
             "\n, ".join(
                 [
                     self._quote_identifier(self.normalizer.normalize(prop_name))
-                    + ": "
+                    + ': "'
                     + str(
                         self._get_sql_column_definitions(stream_name)[
                             self.normalizer.normalize(prop_name)
                         ]
                     )
+                    + '"'
                     for prop_name in columns_list
                 ]
             ),

airbyte/types.py

@@ -13,7 +13,7 @@
 CONVERSION_MAP = {
     "string": sqlalchemy.types.VARCHAR,
     "integer": sqlalchemy.types.BIGINT,
-    "number": sqlalchemy.types.DECIMAL,
+    "number": sqlalchemy.types.DECIMAL(38, 9),
     "boolean": sqlalchemy.types.BOOLEAN,
     "date": sqlalchemy.types.DATE,
     "timestamp_with_timezone": sqlalchemy.types.TIMESTAMP,
@@ -116,11 +116,18 @@ def to_sql_type(
         """Convert a value to a SQL type."""
         try:
             airbyte_type, _ = _get_airbyte_type(json_schema_property_def)
-            return self.conversion_map[airbyte_type]()
+            sql_type = self.conversion_map[airbyte_type]
         except SQLTypeConversionError:
             print(f"Could not determine airbyte type from JSON schema: {json_schema_property_def}")
         except KeyError:
             print(f"Could not find SQL type for airbyte type: {airbyte_type}")
+        else:
+            # No exceptions were raised, so we can return the SQL type.
+            if isinstance(sql_type, type):
+                # This is a class. Call its constructor.
+                sql_type = sql_type()
+
+            return sql_type
 
         json_schema_type = json_schema_property_def.get("type", None)
         json_schema_format = json_schema_property_def.get("format", None)

tests/integration_tests/fixtures/source-test/source_test/run.py

@@ -33,6 +33,7 @@
                         "Column1": {"type": "string"},
                         "Column2": {"type": "number"},
                         "empty_column": {"type": "string"},
+                        "big_number": {"type": "number"},
                     },
                 },
             },
@@ -102,7 +103,12 @@
 sample_record_stream2 = {
     "type": "RECORD",
     "record": {
-        "data": {"Column1": "value1", "Column2": 1},
+        "data": {
+            "Column1": "value1",
+            "Column2": 1,
+            "empty_column": None,
+            "big_number": 1234567890123456,
+        },
         "stream": "stream2",
         "emitted_at": 1704067200,
     },

tests/integration_tests/test_source_test_fixture.py

@@ -113,7 +113,12 @@ def expected_test_stream_data() -> dict[str, list[dict[str, str | int]]]:
             {"column1": "value2", "column2": 2},
         ],
         "stream2": [
-            {"column1": "value1", "column2": 1, "empty_column": None},
+            {
+                "column1": "value1",
+                "column2": 1,
+                "empty_column": None,
+                "big_number": 1234567890123456
+            },
         ],
         "always-empty-stream": [],
     }