Add thousands and decimal params to Pandas.read_csv()

Author: igorborgest

awswrangler/pandas.py

@@ -52,6 +52,8 @@ def read_csv(
             usecols=None,
             dtype=None,
             sep=",",
+            thousands=None,
+            decimal=".",
             lineterminator="\n",
             quotechar='"',
             quoting=csv.QUOTE_MINIMAL,
@@ -74,6 +76,8 @@ def read_csv(
         :param usecols: Same as pandas.read_csv()
         :param dtype: Same as pandas.read_csv()
         :param sep: Same as pandas.read_csv()
+        :param thousands: Same as pandas.read_csv()
+        :param decimal: Same as pandas.read_csv()
         :param lineterminator: Same as pandas.read_csv()
         :param quotechar: Same as pandas.read_csv()
         :param quoting: Same as pandas.read_csv()
@@ -98,6 +102,8 @@ def read_csv(
                                             usecols=usecols,
                                             dtype=dtype,
                                             sep=sep,
+                                            thousands=thousands,
+                                            decimal=decimal,
                                             lineterminator=lineterminator,
                                             quotechar=quotechar,
                                             quoting=quoting,
@@ -115,6 +121,8 @@ def read_csv(
                                         usecols=usecols,
                                         dtype=dtype,
                                         sep=sep,
+                                        thousands=thousands,
+                                        decimal=decimal,
                                         lineterminator=lineterminator,
                                         quotechar=quotechar,
                                         quoting=quoting,
@@ -136,6 +144,8 @@ def _read_csv_iterator(
             usecols=None,
             dtype=None,
             sep=",",
+            thousands=None,
+            decimal=".",
             lineterminator="\n",
             quotechar='"',
             quoting=csv.QUOTE_MINIMAL,
@@ -159,6 +169,8 @@ def _read_csv_iterator(
         :param usecols: Same as pandas.read_csv()
         :param dtype: Same as pandas.read_csv()
         :param sep: Same as pandas.read_csv()
+        :param thousands: Same as pandas.read_csv()
+        :param decimal: Same as pandas.read_csv()
         :param lineterminator: Same as pandas.read_csv()
         :param quotechar: Same as pandas.read_csv()
         :param quoting: Same as pandas.read_csv()
@@ -184,6 +196,8 @@ def _read_csv_iterator(
                                         usecols=usecols,
                                         dtype=dtype,
                                         sep=sep,
+                                        thousands=thousands,
+                                        decimal=decimal,
                                         lineterminator=lineterminator,
                                         quotechar=quotechar,
                                         quoting=quoting,
@@ -231,6 +245,8 @@ def _read_csv_iterator(
                                  names=names,
                                  usecols=usecols,
                                  sep=sep,
+                                 thousands=thousands,
+                                 decimal=decimal,
                                  quotechar=quotechar,
                                  quoting=quoting,
                                  escapechar=escapechar,
@@ -340,6 +356,8 @@ def _read_csv_once(
             usecols=None,
             dtype=None,
             sep=",",
+            thousands=None,
+            decimal=".",
             lineterminator="\n",
             quotechar='"',
             quoting=0,
@@ -362,6 +380,8 @@ def _read_csv_once(
         :param usecols: Same as pandas.read_csv()
         :param dtype: Same as pandas.read_csv()
         :param sep: Same as pandas.read_csv()
+        :param thousands: Same as pandas.read_csv()
+        :param decimal: Same as pandas.read_csv()
         :param lineterminator: Same as pandas.read_csv()
         :param quotechar: Same as pandas.read_csv()
         :param quoting: Same as pandas.read_csv()
@@ -381,6 +401,8 @@ def _read_csv_once(
             names=names,
             usecols=usecols,
             sep=sep,
+            thousands=thousands,
+            decimal=decimal,
             quotechar=quotechar,
             quoting=quoting,
             escapechar=escapechar,

requirements-dev.txt

@@ -1,10 +1,10 @@
 yapf~=0.28.0
-mypy~=0.730
+mypy~=0.740
 flake8~=3.7.8
 pytest-cov~=2.8.1
-cfn-lint~=0.23.3
-twine~=1.13.0
+cfn-lint~=0.24.4
+twine~=2.0.0
 wheel~=0.33.6
-sphinx~=2.1.2
+sphinx~=2.2.0
 pyspark~=2.4.4
 pyspark-stubs~=2.4.0

requirements.txt

@@ -1,8 +1,8 @@
-numpy~=1.17.2
-pandas~=0.25.1
+numpy~=1.17.3
+pandas~=0.25.2
 pyarrow~=0.14.0
-botocore~=1.12.239
-boto3~=1.9.239
-s3fs~=0.3.4
+botocore~=1.12.253
+boto3~=1.9.253
+s3fs~=0.3.5
 tenacity~=5.1.1
 pg8000~=1.13.2

setup.py

@@ -21,12 +21,12 @@
     packages=find_packages(include=["awswrangler", "awswrangler.*"], exclude=["tests"]),
     python_requires=">=3.6",
     install_requires=[
-        "numpy~=1.17.2",
-        "pandas~=0.25.1",
+        "numpy~=1.17.3",
+        "pandas~=0.25.2",
         "pyarrow~=0.14.0",
-        "botocore~=1.12.239",
-        "boto3~=1.9.239",
-        "s3fs~=0.3.4",
+        "botocore~=1.12.253",
+        "boto3~=1.9.253",
+        "s3fs~=0.3.5",
         "tenacity~=5.1.1",
         "pg8000~=1.13.2",
     ],

testing/test_awswrangler/test_pandas.py

@@ -130,6 +130,20 @@ def test_read_csv_iterator_usecols(session, bucket, sample, row_num):
     assert total_count == row_num
 
 
+def test_read_csv_thousands_and_decimal(session, bucket):
+    text = "col1;col2\n1.000.000,00;2.000.000,00\n3.000.000,00;4.000.000,00"
+    filename = "test_read_csv_thousands_and_decimal/sample.txt"
+    boto3.resource("s3").Object(bucket, filename).put(Body=text)
+    path = f"s3://{bucket}/{filename}"
+    df = session.pandas.read_csv(path=path, sep=";", thousands=".", decimal=",")
+    assert len(df.index) == 2
+    assert len(df.columns) == 2
+    assert df.iloc[0].col1 == 1_000_000
+    assert df.iloc[0].col2 == 2_000_000
+    assert df.iloc[1].col1 == 3_000_000
+    assert df.iloc[1].col2 == 4_000_000
+
+
 @pytest.mark.parametrize(
     "mode, file_format, preserve_index, partition_cols, procs_cpu_bound, factor",
     [
@@ -437,7 +451,7 @@ def test_to_parquet_with_empty_dataframe(session, bucket, database):
                                          procs_cpu_bound=1)
 
 
-def test_read_log_query(session, loggroup, logstream):
+def test_read_log_query(session, loggroup):
     dataframe = session.pandas.read_log_query(
         log_group_names=[loggroup],
         query="fields @timestamp, @message | sort @timestamp desc | limit 5",