Bumping version to 0.2.0

Author: igorborgest

README.md

@@ -2,7 +2,7 @@
 
 > Utility belt to handle data on AWS.
 
-[![Release](https://img.shields.io/badge/release-0.1.4-brightgreen.svg)](https://pypi.org/project/awswrangler/)
+[![Release](https://img.shields.io/badge/release-0.2.0-brightgreen.svg)](https://pypi.org/project/awswrangler/)
 [![Downloads](https://img.shields.io/pypi/dm/awswrangler.svg)](https://pypi.org/project/awswrangler/)
 [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7-brightgreen.svg)](https://pypi.org/project/awswrangler/)
 [![Documentation Status](https://readthedocs.org/projects/aws-data-wrangler/badge/?version=latest)](https://aws-data-wrangler.readthedocs.io/en/latest/?badge=latest)

awswrangler/__version__.py

@@ -1,4 +1,4 @@
 __title__ = "awswrangler"
 __description__ = "Utility belt to handle data on AWS."
-__version__ = "0.1.4"
+__version__ = "0.2.0"
 __license__ = "Apache License 2.0"

awswrangler/pandas.py

@@ -1524,7 +1524,13 @@ def to_aurora(self,
                           engine=engine,
                           region=region)
 
-        self._session.s3.delete_objects(path=temp_s3_path, procs_io_bound=procs_io_bound)
+        if "postgres" in engine.lower():
+            self._session.s3.delete_listed_objects(objects_paths=load_paths, procs_io_bound=procs_io_bound)
+        elif "mysql" in engine.lower():
+            self._session.s3.delete_listed_objects(objects_paths=load_paths + [manifest_path],
+                                                   procs_io_bound=procs_io_bound)
+        else:
+            raise InvalidEngine(f"{engine} is not a valid engine. Please use 'mysql' or 'postgres'!")
 
     def read_sql_aurora(self,
                         sql: str,

testing/test_awswrangler/test_glue.py

@@ -114,15 +114,15 @@ def test_get_tables_search(session, table):
 
 
 def test_get_tables_prefix(session, table):
-    tables = list(session.glue.get_tables(prefix=table[:-1]))
+    tables = list(session.glue.get_tables(name_prefix=table[:-1]))
     assert len(tables) > 0
     for tbl in tables:
         if tbl["Name"] == table:
             assert tbl["TableType"] == "EXTERNAL_TABLE"
 
 
 def test_get_tables_suffix(session, table):
-    tables = list(session.glue.get_tables(suffix=table[1:]))
+    tables = list(session.glue.get_tables(name_suffix=table[1:]))
     assert len(tables) > 0
     for tbl in tables:
         if tbl["Name"] == table:

testing/test_awswrangler/test_pandas.py

@@ -1830,7 +1830,8 @@ def test_aurora_mysql_load_append(bucket, mysql_parameters):
 
 
 def test_aurora_postgres_load_append(bucket, postgres_parameters):
-    df = pd.DataFrame({"id": [1, 2, 3], "value": ["foo", "boo", "bar"]})
+    n: int = 10_000
+    df = pd.DataFrame({"id": list((range(n))), "value": list(["foo" if i % 2 == 0 else "boo" for i in range(n)])})
     conn = Aurora.generate_connection(database="postgres",
                                       host=postgres_parameters["PostgresAddress"],
                                       port=3306,

testing/test_awswrangler/test_sagemaker.py

@@ -70,7 +70,7 @@ def model(bucket):
 def model_empty(bucket):
     model_path = "output_empty/model.tar.gz"
 
-    with tarfile.open("model.tar.gz", "w:gz") as tar:
+    with tarfile.open("model.tar.gz", "w:gz"):
         pass
 
     s3 = boto3.resource("s3")

testing/test_awswrangler/test_spark.py

@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime, date
+from time import sleep
 
 import pytest
 import boto3
@@ -88,6 +89,8 @@ def test_read_csv(session, bucket, sample_name):
     [("snappy", []), ("gzip", ["date", "value"]), ("none", ["time"])],
 )
 def test_create_glue_table_parquet(session, bucket, database, compression, partition_by):
+    s3_path = f"s3://{bucket}/test"
+    session.s3.delete_objects(path=s3_path)
     path = "data_samples/nano.csv"
     schema = "id INTEGER, name STRING, value DOUBLE, date DATE, time TIMESTAMP"
     timestamp_format = "yyyy-MM-dd"
@@ -100,12 +103,12 @@ def test_create_glue_table_parquet(session, bucket, database, compression, parti
         .withColumn("my_array", array(lit(0), lit(1))) \
         .withColumn("my_struct", struct(lit("text").alias("a"), lit(1).alias("b"))) \
         .withColumn("my_map", create_map(lit("k0"), lit(1.0), lit("k1"), lit(2.0)))
-    s3_path = f"s3://{bucket}/test"
     dataframe.write \
         .mode("overwrite") \
         .format("parquet") \
         .partitionBy(partition_by) \
         .save(compression=compression, path=s3_path)
+    sleep(10)
     session.spark.create_glue_table(dataframe=dataframe,
                                     file_format="parquet",
                                     partition_by=partition_by,
@@ -119,6 +122,7 @@ def test_create_glue_table_parquet(session, bucket, database, compression, parti
     assert pandas_df.iloc[0]["counter"] == 5
     query = "select my_array[1] as foo, my_struct.a as boo, my_map['k0'] as bar from test limit 1"
     pandas_df = session.pandas.read_sql_athena(sql=query, database=database)
+    session.s3.delete_objects(path=s3_path)
     assert pandas_df.iloc[0]["foo"] == 0
     assert pandas_df.iloc[0]["boo"] == "text"
     assert pandas_df.iloc[0]["bar"] == 1.0