Fix \r\n line terminator for wr.s3.to_csv() #415

Author: igorborgest

awswrangler/s3/_write_text.py

@@ -37,15 +37,14 @@ def _to_text(
     else:
         raise RuntimeError("path and path_root received at the same time.")
     encoding: Optional[str] = pandas_kwargs.get("encoding", None)
-    newline: Optional[str] = pandas_kwargs.get("line_terminator", None)
     with open_s3_object(
         path=file_path,
         mode="w",
         use_threads=use_threads,
         s3_additional_kwargs=s3_additional_kwargs,
         boto3_session=boto3_session,
         encoding=encoding,
-        newline=newline,
+        newline=None,
     ) as f:
         _logger.debug("pandas_kwargs: %s", pandas_kwargs)
         if file_format == "csv":

tests/test_s3_text.py

@@ -263,3 +263,13 @@ def test_csv_additional_kwargs(path, kms_key_id, s3_additional_kwargs, use_threa
         assert desc.get("ServerSideEncryption") == "aws:kms"
     elif s3_additional_kwargs["ServerSideEncryption"] == "AES256":
         assert desc.get("ServerSideEncryption") == "AES256"
+
+
+@pytest.mark.parametrize("line_terminator", ["\n", "\r", "\n\r"])
+def test_csv_line_terminator(path, line_terminator):
+    file_path = f"{path}0.csv"
+    df = pd.DataFrame(data={"reading": ["col1", "col2"], "timestamp": [1601379427618, 1601379427625], "value": [1, 2]})
+    wr.s3.to_csv(df=df, path=file_path, index=False, line_terminator=line_terminator)
+    wr.s3.wait_objects_exist(paths=[file_path])
+    df2 = wr.s3.read_csv(file_path)
+    assert df.equals(df2)