fix data types for solidgate (#309)

Author: karakanb

ingestr/src/solidgate/__init__.py

@@ -86,16 +86,12 @@
         "transaction_datetime_provider": {"data_type": "timestamp"},
         "transaction_datetime_utc": {"data_type": "timestamp"},
         "accounting_date": {"data_type": "date"},
-        "amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "amount_in_major_units": {"data_type": "decimal", "precision": 18, "scale": 4},
+        "amount": {"data_type": "double"},
+        "amount_in_major_units": {"data_type": "double"},
         "currency": {"data_type": "text"},
         "currency_minor_units": {"data_type": "bigint"},
-        "payout_amount": {"data_type": "decimal", "precision": 18, "scale": 4},
-        "payout_amount_in_major_units": {
-            "data_type": "decimal",
-            "precision": 18,
-            "scale": 4,
-        },
+        "payout_amount": {"data_type": "double"},
+        "payout_amount_in_major_units": {"data_type": "double"},
         "payout_currency": {"data_type": "text"},
         "payout_currency_minor_units": {"data_type": "bigint"},
         "record_type_key": {"data_type": "text"},

ingestr/src/solidgate/helpers.py

@@ -1,4 +1,5 @@
 import base64
+import csv
 import hashlib
 import hmac
 import json
@@ -123,11 +124,18 @@ def fetch_financial_entry_data(
                 except json.JSONDecodeError:
                     try:
                         csv_data = get_response.content.decode("utf-8")
-                        df = pd.read_csv(StringIO(csv_data))
-                        df["created_at"] = df["created_at"].apply(
-                            lambda x: pendulum.parse(x)
-                        )
-                        return df
+                        reader = csv.DictReader(StringIO(csv_data))
+                        rows = []
+                        for row in reader:
+                            if row["created_at"]:
+                                row["created_at"] = pendulum.parse(row["created_at"])
+                            else:
+                                row["created_at"] = None
+
+                            row2 = {k: v for k, v in row.items() if v != ''}
+                            rows.append(row2)
+                        
+                        return rows
                     except Exception as e:
                         raise Exception(f"Error reading CSV: {e}")
             else: