fix: coerce_timestamps - allow None (#2556)

* fix: coerce_timestamps - allow None

Signed-off-by: Anton Kukushkin <[email protected]>

* temp skip glue dq multi-ruleset test due to the new service error

Signed-off-by: Anton Kukushkin <[email protected]>

* fix another occurrence

Signed-off-by: Anton Kukushkin <[email protected]>

* fix other params

Signed-off-by: Anton Kukushkin <[email protected]>

---------

Signed-off-by: Anton Kukushkin <[email protected]>

Author: kukushking

awswrangler/s3/_write_parquet.py

@@ -53,18 +53,18 @@ def _new_writer(
     writer: Optional[pyarrow.parquet.ParquetWriter] = None
     if not pyarrow_additional_kwargs:
         pyarrow_additional_kwargs = {}
-    if not pyarrow_additional_kwargs.get("coerce_timestamps"):
+    if "coerce_timestamps" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["coerce_timestamps"] = "ms"
     if "flavor" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["flavor"] = "spark"
     if "version" not in pyarrow_additional_kwargs:
         # By default, use version 1.0 logical type set to maximize compatibility
         pyarrow_additional_kwargs["version"] = "1.0"
-    if not pyarrow_additional_kwargs.get("use_dictionary"):
+    if "use_dictionary" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["use_dictionary"] = True
-    if not pyarrow_additional_kwargs.get("write_statistics"):
+    if "write_statistics" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["write_statistics"] = True
-    if not pyarrow_additional_kwargs.get("schema"):
+    if "schema" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["schema"] = schema
 
     with open_s3_object(
@@ -708,7 +708,7 @@ def to_parquet(  # pylint: disable=too-many-arguments,too-many-locals,too-many-b
     # Pyarrow defaults
     if not pyarrow_additional_kwargs:
         pyarrow_additional_kwargs = {}
-    if not pyarrow_additional_kwargs.get("coerce_timestamps"):
+    if "coerce_timestamps" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["coerce_timestamps"] = "ms"
     if "flavor" not in pyarrow_additional_kwargs:
         pyarrow_additional_kwargs["flavor"] = "spark"

tests/unit/test_data_quality.py

@@ -1,5 +1,6 @@
 import logging
 
+import botocore
 import pytest
 
 import awswrangler as wr
@@ -258,6 +259,7 @@ def test_upsert_ruleset(df: pd.DataFrame, glue_database: str, glue_table: str, g
     assert row_count.iloc[0]["expression"] == "between 2 and 8"
 
 
+@pytest.mark.xfail(raises=botocore.exceptions.ClientError, reason="Service error when evaluating multiple rulesets.")
 def test_two_evaluations_at_once(
     df: pd.DataFrame, glue_database: str, glue_table: str, glue_ruleset: str, glue_data_quality_role: str
 ) -> None: