Merge pull request #104 from awslabs/aurora

Improving Aurora tests

Author: igorborgest

awswrangler/aurora.py

@@ -201,6 +201,7 @@ def _get_load_sql(path: str, schema_name: str, table_name: str, engine: str, reg
             sql: str = ("-- AWS DATA WRANGLER\n"
                         "SELECT aws_s3.table_import_from_s3(\n"
                         f"'{schema_name}.{table_name}',\n"
+                        "'',\n"
                         "'(FORMAT CSV, DELIMITER '','', QUOTE ''\"'', ESCAPE ''\"'')',\n"
                         f"'({bucket},{key},{region})')")
         elif "mysql" in engine.lower():

testing/test_awswrangler/test_pandas.py

@@ -1925,10 +1925,12 @@ def test_aurora_postgres_load_special(bucket, postgres_parameters):
     df = pd.DataFrame({
         "id": [1, 2, 3, 4],
         "value": ["foo", "boo", "bar", "abc"],
-        "special": ["\\", "\"", "\\\\\\\\", "\"\"\"\""]
+        "slashes": ["\\", "\"", "\\\\\\\\", "\"\"\"\""],
+        "floats": [1.0, 2.0, 3.0, 4.0],
+        "decimals": [Decimal((0, (1, 9, 9), -2)), Decimal((0, (1, 9, 9), -2)), Decimal((0, (1, 9, 0), -2)), Decimal((0, (3, 1, 2), -2))]
     })
 
-    path = f"s3://{bucket}/test_aurora_postgres_slash"
+    path = f"s3://{bucket}/test_aurora_postgres_special"
     wr.pandas.to_aurora(dataframe=df,
                         connection="aws-data-wrangler-postgres",
                         schema="public",
@@ -1958,14 +1960,25 @@ def test_aurora_postgres_load_special(bucket, postgres_parameters):
         assert rows[1][2] == "\""
         assert rows[2][2] == "\\\\\\\\"
         assert rows[3][2] == "\"\"\"\""
+        assert (rows[0][3] == 1.0) and (str(type(rows[0][3])) == str(type(1.0)))
+        assert rows[1][3] == 2.0
+        assert rows[2][3] == 3.0
+        assert rows[3][3] == 4.0
+        assert rows[0][4] == Decimal((0, (1, 9, 9), -2))
+        assert rows[1][4] == Decimal((0, (1, 9, 9), -2))
+        assert rows[2][4] == Decimal((0, (1, 9, 0), -2))
+        assert rows[3][4] == Decimal((0, (3, 1, 2), -2))
     conn.close()
 
 
 def test_aurora_mysql_load_special(bucket, mysql_parameters):
     df = pd.DataFrame({
         "id": [1, 2, 3, 4],
         "value": ["foo", "boo", "bar", "abc"],
-        "special": ["\\", "\"", "\\\\\\\\", "\"\"\"\""]
+        "slashes": ["\\", "\"", "\\\\\\\\", "\"\"\"\""],
+        "floats": [1.0, 2.0, 3.0, 4.0],
+        "decimals": [Decimal((0, (1, 9, 9), -2)), Decimal((0, (1, 9, 9), -2)), Decimal((0, (1, 9, 0), -2)),
+                     Decimal((0, (3, 1, 2), -2))]
     })
 
     path = f"s3://{bucket}/test_aurora_mysql_special"
@@ -1998,6 +2011,14 @@ def test_aurora_mysql_load_special(bucket, mysql_parameters):
         assert rows[1][2] == "\""
         assert rows[2][2] == "\\\\\\\\"
         assert rows[3][2] == "\"\"\"\""
+        assert (rows[0][3] == 1.0) and (str(type(rows[0][3])) == str(type(1.0)))
+        assert rows[1][3] == 2.0
+        assert rows[2][3] == 3.0
+        assert rows[3][3] == 4.0
+        assert rows[0][4] == Decimal((0, (1, 9, 9), -2))
+        assert rows[1][4] == Decimal((0, (1, 9, 9), -2))
+        assert rows[2][4] == Decimal((0, (1, 9, 0), -2))
+        assert rows[3][4] == Decimal((0, (3, 1, 2), -2))
     conn.close()