chore: update pyarrow to 18.1.0 in lambda layers (#3065)

* fix: update pyarrow to 18 in lambda layers
* unpin cython
* fix numpy for 3.10
* type stubs changed
* fix unrecognized pyarrow version via setuptools_scm
* arrow <19

Author: kukushking

awswrangler/data_api/rds.py

@@ -18,7 +18,7 @@
 from awswrangler.data_api import _connector
 
 if TYPE_CHECKING:
-    from mypy_boto3_rds_data.client import BotocoreClientError
+    from mypy_boto3_rds_data.client import BotocoreClientError  # type: ignore[attr-defined]
     from mypy_boto3_rds_data.type_defs import BatchExecuteStatementResponseTypeDef, ExecuteStatementResponseTypeDef
 
 

awswrangler/timestream/_write.py

@@ -179,7 +179,7 @@ def _write_batch(
                 Records=records,
             )
     except client_timestream.exceptions.RejectedRecordsException as ex:
-        return cast(List[Dict[str, str]], ex.response["RejectedRecords"])
+        return cast(List[Dict[str, str]], ex.response["RejectedRecords"])  # type: ignore[typeddict-item]
     return []
 
 

building/lambda/Dockerfile

@@ -23,9 +23,9 @@ WORKDIR /root
 FROM ${python_version}
 COPY pyproject.toml poetry.lock ./
 
-RUN pip3 install --upgrade pip wheel
+RUN pip3 install --upgrade pip wheel setuptools>=64 setuptools_scm>=8
 RUN pip3 install --upgrade urllib3==1.26.16  # temporary to avoid https://github.com/urllib3/urllib3/issues/2168 (TODO remove when the AL2 image updates to support OpenSSL 1.1.1+)
-RUN pip3 install --upgrade six cython==3.0.8 cmake hypothesis poetry
+RUN pip3 install --upgrade six cython cmake hypothesis poetry
 RUN poetry config virtualenvs.create false --local && poetry install --no-root --only main
 
 RUN rm -f pyproject.toml poetry.lock

building/lambda/Dockerfile.al2023

@@ -23,8 +23,8 @@ WORKDIR /root
 FROM ${python_version}
 COPY pyproject.toml poetry.lock ./
 
-RUN pip3 install --upgrade pip wheel
-RUN pip3 install --upgrade six cython==3.0.8 cmake hypothesis poetry
+RUN pip3 install --upgrade pip wheel setuptools>=64 setuptools_scm>=8
+RUN pip3 install --upgrade six cython cmake hypothesis poetry
 RUN poetry config virtualenvs.create false --local && poetry install --no-root --only main
 
 RUN rm -f pyproject.toml poetry.lock

building/lambda/build-lambda-layer.sh

@@ -11,12 +11,14 @@ popd
 rm -rf dist arrow
 
 export ARROW_HOME=$(pwd)/dist
+export ARROW_VERSION=18.1.0
 export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
 export CMAKE_PREFIX_PATH=$ARROW_HOME:$CMAKE_PREFIX_PATH
+export SETUPTOOLS_SCM_PRETEND_VERSION=$ARROW_VERSION
 
 git clone \
   --depth 1 \
-  --branch apache-arrow-16.1.0 \
+  --branch "apache-arrow-${ARROW_VERSION}" \
   --single-branch \
   https://github.com/apache/arrow.git
 

poetry.lock

@@ -32,10 +32,13 @@ python = ">=3.9, <4.0"
 boto3 = "^1.20.32"
 botocore = "^1.23.32"
 pandas = { version = ">=1.2.0,<3.0.0" }
-numpy = {  version = ">=1.26,<3.0" }
+numpy = [
+    { version = ">=1.26,<2.1.0", markers = "python_version < \"3.10\"" },
+    { version = ">=1.26,<3.0", markers = "python_version >= \"3.10\"" }
+]
 pyarrow = [
-    { version = ">=8.0.0,<18.0.0", markers = "python_version < \"3.13\"" },
-    { version = ">=18.0.0", markers = "python_version >= \"3.13\"" }
+    { version = ">=8.0.0,<19.0.0", markers = "python_version < \"3.13\"" },
+    { version = ">=18.0.0,<19.0.0", markers = "python_version >= \"3.13\"" }
 ]
 typing-extensions = "^4.4.0"
 packaging = ">=21.1,<25.0"