Bumping version to 0.2.6

Author: igorborgest

README.md

@@ -2,7 +2,7 @@
 
 > DataFrames on AWS
 
-[![Release](https://img.shields.io/badge/release-0.2.5-brightgreen.svg)](https://pypi.org/project/awswrangler/)
+[![Release](https://img.shields.io/badge/release-0.2.6-brightgreen.svg)](https://pypi.org/project/awswrangler/)
 [![Downloads](https://img.shields.io/pypi/dm/awswrangler.svg)](https://pypi.org/project/awswrangler/)
 [![Python Version](https://img.shields.io/badge/python-3.6%20%7C%203.7-brightgreen.svg)](https://pypi.org/project/awswrangler/)
 [![Documentation Status](https://readthedocs.org/projects/aws-data-wrangler/badge/?version=latest)](https://aws-data-wrangler.readthedocs.io/en/latest/?badge=latest)

awswrangler/__version__.py

@@ -1,4 +1,4 @@
 __title__ = "awswrangler"
 __description__ = "DataFrames on AWS."
-__version__ = "0.2.5"
+__version__ = "0.2.6"
 __license__ = "Apache License 2.0"

awswrangler/data_types.py

@@ -1,4 +1,4 @@
-from typing import List, Tuple, Dict, Callable, Optional
+from typing import List, Tuple, Dict, Callable, Optional, Any
 from logging import getLogger, Logger
 from datetime import datetime, date
 from decimal import Decimal
@@ -361,7 +361,7 @@ def convert_schema(func: Callable, schema: List[Tuple[str, str]]) -> Dict[str, s
 
 def extract_pyarrow_schema_from_pandas(dataframe: pd.DataFrame,
                                        preserve_index: bool,
-                                       indexes_position: str = "right") -> List[Tuple[str, str]]:
+                                       indexes_position: str = "right") -> List[Tuple[str, Any]]:
     """
     Extract the related Pyarrow schema from any Pandas DataFrame
 
@@ -387,8 +387,7 @@ def extract_pyarrow_schema_from_pandas(dataframe: pd.DataFrame,
     indexes: List[str] = []
     for field in pa.Schema.from_pandas(df=dataframe[cols], preserve_index=preserve_index):
         name = str(field.name)
-        dtype = str(field.type)
-        cols_dtypes[name] = dtype
+        cols_dtypes[name] = field.type
         if name not in dataframe.columns:
             indexes.append(name)
 

awswrangler/glue.py

@@ -241,7 +241,7 @@ def _build_schema(
         if partition_cols is None:
             partition_cols = []
 
-        pyarrow_schema: List[Tuple[str, str]] = data_types.extract_pyarrow_schema_from_pandas(
+        pyarrow_schema: List[Tuple[str, Any]] = data_types.extract_pyarrow_schema_from_pandas(
             dataframe=dataframe, preserve_index=preserve_index, indexes_position=indexes_position)
 
         schema_built: List[Tuple[str, str]] = []

awswrangler/pandas.py

@@ -1950,6 +1950,7 @@ def read_csv_list(
                     receive_pipes[i].close()
                 logger.debug(f"Concatenating all {len(paths)} DataFrames...")
                 df = pd.concat(objs=dfs, ignore_index=True, sort=False)
+                logger.debug("Concatenation done!")
             return df
 
     def _read_csv_list_iterator(

building/Dockerfile

@@ -19,4 +19,7 @@ ADD requirements-dev.txt /root/
 RUN pip3 install --upgrade -r /root/requirements-dev.txt
 RUN rm -rf /root/requirements-dev.txt
 
+RUN pip3 install boto3==1.10.34
+RUN pip3 install botocore==1.13.34
+
 ENTRYPOINT ["/bin/sh"]

building/build-lambda-layer.sh

@@ -18,6 +18,7 @@ rm -f "dist/awswrangler-layer.zip"
 # Building
 pip install . -t ./python
 rm -rf python/pyarrow*
+rm -rf python/boto*
 cp -r /aws-data-wrangler/dist/pyarrow_files/pyarrow* python/
 find python -wholename "*/tests/*" -type f -delete
 zip -r9 "awswrangler-layer.zip" ./python
@@ -26,4 +27,4 @@ mv "awswrangler-layer.zip" dist/
 # Cleaning up the directory again
 rm -rf python
 
-cd building
+cd building

requirements-dev.txt

@@ -3,11 +3,11 @@ mypy~=0.761
 flake8~=3.7.9
 pytest-cov~=2.8.1
 scikit-learn~=0.22.1
-cfn-lint~=0.27.1
+cfn-lint~=0.27.2
 twine~=3.1.1
 wheel~=0.33.6
 sphinx~=2.3.1
 pyspark~=2.4.4
 pyspark-stubs~=2.4.0.post7
 jupyter~=1.0.0
-jupyterlab~=1.2.5
+jupyterlab~=1.2.6

requirements.txt

@@ -1,8 +1,8 @@
 numpy~=1.18.1
 pandas~=0.25.3
 pyarrow~=0.15.1
-botocore~=1.14.7
-boto3~=1.11.7
+botocore>=1.13.34
+boto3>=1.10.34
 s3fs~=0.4.0
 tenacity~=6.0.0
 pg8000~=1.13.2

testing/Dockerfile

@@ -29,4 +29,7 @@ ADD requirements-dev.txt /root/
 RUN $PIP install --upgrade -r /root/requirements-dev.txt
 RUN rm -rf /root/requirements-dev.txt
 
+RUN $PIP install boto3==1.10.34
+RUN $PIP install botocore==1.13.34
+
 ENTRYPOINT ["/bin/sh"]