Testing PyArrow 5 Distribution Build (#861)

Author: jaidisido

.github/dependabot.yml

@@ -10,4 +10,3 @@ updates:
       prefix: "[skip ci] pip"
       prefix-development: "[skip ci] pip"
       include: "scope"
-    open-pull-requests-limit: 5

.github/workflows/static-checking.yml

@@ -29,7 +29,7 @@ jobs:
           python -m pip install --upgrade pip
           pip install -U -r requirements-dev.txt
       - name: mypy check
-        run: yes y | mypy --install-types awswrangler
+        run: mypy --install-types --non-interactive awswrangler
       - name: Flake8 Lint
         run: flake8 .
       - name: Pylint Lint

awswrangler/_utils.py

@@ -227,7 +227,7 @@ def chunkify(lst: List[Any], num_chunks: int = 1, max_length: Optional[int] = No
     if not lst:
         return []
     n: int = num_chunks if max_length is None else int(math.ceil((float(len(lst)) / float(max_length))))
-    np_chunks = np.array_split(lst, n)
+    np_chunks = np.array_split(lst, n)  # type: ignore
     return [arr.tolist() for arr in np_chunks if len(arr) > 0]
 
 

awswrangler/timestream.py

@@ -200,15 +200,15 @@ def write(
 
 
 def query(
-    sql: str, pagination_config: Dict[str, Any] = None, boto3_session: Optional[boto3.Session] = None
+    sql: str, pagination_config: Optional[Dict[str, Any]] = None, boto3_session: Optional[boto3.Session] = None
 ) -> pd.DataFrame:
     """Run a query and retrieve the result as a Pandas DataFrame.
 
     Parameters
     ----------
     sql: str
         SQL query.
-    pagination_config: Dict[str, Any]
+    pagination_config: Dict[str, Any], optional
         Pagination configuration dictionary of a form {'MaxItems': 10, 'PageSize': 10, 'StartingToken': '...'}
     boto3_session : boto3.Session(), optional
         Boto3 Session. The default boto3 Session will be used if boto3_session receive None.

building/lambda/build-lambda-layer.sh

@@ -14,7 +14,7 @@ export ARROW_HOME=$(pwd)/dist
 export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
 
 git clone \
-  --branch apache-arrow-4.0.0 \
+  --branch apache-arrow-5.0.0 \
   --single-branch \
   https://github.com/apache/arrow.git
 

docs/environment.yml

@@ -8,7 +8,7 @@ dependencies:
   - pip:
     - nbsphinx
     - nbsphinx-link
-    - sphinx==4.0.3
+    - sphinx
     - sphinx_bootstrap_theme
     - IPython
     - -e ..

docs/source/api.rst

@@ -8,6 +8,8 @@ API Reference
 * `PostgreSQL`_
 * `MySQL`_
 * `Microsoft SQL Server`_
+* `Data API Redshift`_
+* `Data API RDS`_
 * `DynamoDB`_
 * `Amazon Timestream`_
 * `Amazon EMR`_
@@ -173,6 +175,28 @@ ____________________
     read_sql_table
     to_sql
 
+Data API Redshift
+-----------------
+
+.. currentmodule:: awswrangler.data_api.redshift
+
+.. autosummary::
+    :toctree: stubs
+
+    connect
+    read_sql_query
+
+Data API RDS
+------------
+
+.. currentmodule:: awswrangler.data_api.rds
+
+.. autosummary::
+    :toctree: stubs
+
+    connect
+    read_sql_query
+
 DynamoDB
 --------
 

requirements-dev.txt

@@ -1,9 +1,9 @@
 wheel==0.36.2
 isort==5.9.3
 black==21.7b0
-pylint==2.9.5
+pylint==2.9.6
 flake8==3.9.2
-mypy==0.902
+mypy==0.910
 pydocstyle==6.1.1
 doc8==0.9.0
 tox==3.24.1
@@ -14,13 +14,13 @@ pytest-xdist==2.3.0
 pytest-timeout==1.4.2
 pydot==1.4.2
 twine==3.4.2
-sphinx==4.0.3
+sphinx==4.1.2
 sphinx_bootstrap_theme==0.7.1
 nbsphinx==0.8.6
 nbsphinx-link==1.3.0
 IPython==7.16.0
-moto==2.2.0
-jupyterlab==3.1.1
+moto==2.2.1
+jupyterlab==3.1.2
 s3fs==2021.7.0
 python-Levenshtein==0.12.2
 bump2version==1.0.1

requirements.txt

@@ -3,7 +3,7 @@ botocore>=1.19.8,<2.0.0
 numpy>=1.18.0,<2.0.0
 pandas>=1.1.0,<2.0.0
 pyarrow>=2.0.0,<5.1.0
-redshift-connector~=2.0.883
+redshift-connector~=2.0.884
 pymysql>=0.9.0,<1.1.0
 pg8000>=1.16.0,<1.22.0
 openpyxl~=3.0.0

validate.sh

@@ -3,7 +3,7 @@ set -ex
 
 isort --check .
 black --check .
-yes y | mypy --install-types awswrangler
+mypy --install-types --non-interactive awswrangler
 flake8 .
 pylint -j 0 awswrangler
 pydocstyle awswrangler/ --convention=numpy