Skip to content

Commit 4046387

Browse files
authored
Minor - Enable user to specify pyarrow flavor (#1057)
1 parent 42b46f6 commit 4046387

File tree

1 file changed

+2
-1
lines changed

1 file changed

+2
-1
lines changed

awswrangler/s3/_write_parquet.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ def _new_writer(
5050
pyarrow_additional_kwargs = {}
5151
if not pyarrow_additional_kwargs.get("coerce_timestamps"):
5252
pyarrow_additional_kwargs["coerce_timestamps"] = "ms"
53+
if "flavor" not in pyarrow_additional_kwargs:
54+
pyarrow_additional_kwargs["flavor"] = "spark"
5355

5456
with open_s3_object(
5557
path=file_path,
@@ -64,7 +66,6 @@ def _new_writer(
6466
write_statistics=True,
6567
use_dictionary=True,
6668
compression="NONE" if compression is None else compression,
67-
flavor="spark",
6869
schema=schema,
6970
**pyarrow_additional_kwargs,
7071
)

0 commit comments

Comments
 (0)