YAPF (formatting)

Author: igorborgest

awswrangler/pandas.py

@@ -1851,19 +1851,19 @@ def read_csv_prefix(
         paths: List[str] = self._session.s3.list_objects(path=path_prefix)
         paths = [p for p in paths if not p.endswith("/")]
         return self.read_csv_list(paths=paths,
-                                max_result_size=max_result_size,
-                                header=header,
-                                names=names,
-                                usecols=usecols,
-                                dtype=dtype,
-                                sep=sep,
-                                thousands=thousands,
-                                decimal=decimal,
-                                lineterminator=lineterminator,
-                                quotechar=quotechar,
-                                quoting=quoting,
-                                escapechar=escapechar,
-                                parse_dates=parse_dates,
-                                infer_datetime_format=infer_datetime_format,
-                                encoding=encoding,
-                                converters=converters)
+                                  max_result_size=max_result_size,
+                                  header=header,
+                                  names=names,
+                                  usecols=usecols,
+                                  dtype=dtype,
+                                  sep=sep,
+                                  thousands=thousands,
+                                  decimal=decimal,
+                                  lineterminator=lineterminator,
+                                  quotechar=quotechar,
+                                  quoting=quoting,
+                                  escapechar=escapechar,
+                                  parse_dates=parse_dates,
+                                  infer_datetime_format=infer_datetime_format,
+                                  encoding=encoding,
+                                  converters=converters)

awswrangler/s3.py

@@ -231,13 +231,16 @@ def list_objects(self, path: str) -> List[str]:
         bucket: str
         bucket, path = self.parse_path(path=path)
         args: Dict[str, Any] = {"Bucket": bucket, "MaxKeys": 1000, "Prefix": path}
-        next_continuation_token: str = ""
+        next_continuation_token: Optional[str] = ""
         keys: List[str] = []
         while next_continuation_token is not None:
             res: Dict[str, Any] = self._client_s3.list_objects_v2(**args)
             if res.get("Contents") is None:
                 break
-            keys += [f"s3://{bucket}/{x.get('Key')}" for x in res.get("Contents")]
+            keys += [
+                f"s3://{bucket}/{x['Key']}" for x in res.get("Contents")  # type: ignore
+                if (x is not None) and ("Key" in x)
+            ]
             next_continuation_token = res.get("NextContinuationToken")
             if next_continuation_token:
                 args["ContinuationToken"] = next_continuation_token

testing/test_awswrangler/test_pandas.py

@@ -1929,22 +1929,20 @@ def test_aurora_postgres_load_special(bucket, postgres_parameters):
     })
 
     path = f"s3://{bucket}/test_aurora_postgres_slash"
-    wr.pandas.to_aurora(
-        dataframe=df,
-        connection="aws-data-wrangler-postgres",
-        schema="public",
-        table="test_aurora_postgres_special",
-        mode="overwrite",
-        temp_s3_path=path,
-        engine="postgres",
-        procs_cpu_bound=4
-    )
+    wr.pandas.to_aurora(dataframe=df,
+                        connection="aws-data-wrangler-postgres",
+                        schema="public",
+                        table="test_aurora_postgres_special",
+                        mode="overwrite",
+                        temp_s3_path=path,
+                        engine="postgres",
+                        procs_cpu_bound=4)
     conn = Aurora.generate_connection(database="postgres",
-                                     host=postgres_parameters["PostgresAddress"],
-                                     port=3306,
-                                     user="test",
-                                     password=postgres_parameters["Password"],
-                                     engine="postgres")
+                                      host=postgres_parameters["PostgresAddress"],
+                                      port=3306,
+                                      user="test",
+                                      password=postgres_parameters["Password"],
+                                      engine="postgres")
     with conn.cursor() as cursor:
         cursor.execute("SELECT * FROM public.test_aurora_postgres_special")
         rows = cursor.fetchall()
@@ -1971,22 +1969,20 @@ def test_aurora_mysql_load_special(bucket, mysql_parameters):
     })
 
     path = f"s3://{bucket}/test_aurora_mysql_special"
-    wr.pandas.to_aurora(
-        dataframe=df,
-        connection="aws-data-wrangler-mysql",
-        schema="test",
-        table="test_aurora_mysql_special",
-        mode="overwrite",
-        temp_s3_path=path,
-        engine="mysql",
-        procs_cpu_bound=1
-    )
+    wr.pandas.to_aurora(dataframe=df,
+                        connection="aws-data-wrangler-mysql",
+                        schema="test",
+                        table="test_aurora_mysql_special",
+                        mode="overwrite",
+                        temp_s3_path=path,
+                        engine="mysql",
+                        procs_cpu_bound=1)
     conn = Aurora.generate_connection(database="mysql",
-                                     host=mysql_parameters["MysqlAddress"],
-                                     port=3306,
-                                     user="test",
-                                     password=mysql_parameters["Password"],
-                                     engine="mysql")
+                                      host=mysql_parameters["MysqlAddress"],
+                                      port=3306,
+                                      user="test",
+                                      password=mysql_parameters["Password"],
+                                      engine="mysql")
     with conn.cursor() as cursor:
         cursor.execute("SELECT * FROM test.test_aurora_mysql_special")
         rows = cursor.fetchall()