(enhancement) update lambda layers with pyarrow 10 (#1758)

* Update lambda layers with pyarrow 10

* Use C++17 standard

* [skip ci] Retire Python3.6 from Docker image

* [skip ci] Remove c++17 std cmake flags

* [skip ci] Add CMAKE_PREFIX_PATH

* Add separate dockerfile for AL1-based py37

* use single dockerfile for all

* [skip ci] add missing build arg for py37

* [skip ci] correct syntax

* [skip ci] use al2 as base image for py37 too

* [skip ci] comment

Co-authored-by: Lucas Hanson <[email protected]>
Co-authored-by: Cyril Fait <[email protected]>

Author: 3 people

building/lambda/Dockerfile

@@ -16,18 +16,27 @@ RUN yum install -y \
     git \
     ninja-build
 
-RUN pip3 install --upgrade pip wheel && pip3 install --upgrade six cython cmake hypothesis poetry
-
 WORKDIR /root
 
-FROM base AS python36
-# only for python3.6
-RUN yum install -y python36-devel && \
-    mkdir -p /var/lang/include/ && \
-    ln -s /usr/include/python3.6m /var/lang/include/python3.6m
+FROM base AS python37
+# Install C devtools & Python37
+ARG PYTHON_VERSION="3.7.15"
+RUN yum -y group install "C Development Tools and Libraries" "Development Tools" \
+    && yum -y install openssl openssl-devel libffi-devel bzip2-devel \
+    && curl -fsSL https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz | tar xzf - \
+    && cd Python-${PYTHON_VERSION}/ \
+    && ./configure --enable-optimizations --prefix=/var/lang \
+    && make altinstall \
+    && ln -fs /var/lang/bin/python3.7 /var/lang/bin/python \
+    && ln -fs /var/lang/bin/python3.7 /var/lang/bin/python3 \
+    && ln -fs /var/lang/bin/pip3.7 /var/lang/bin/pip \
+    && ln -fs /var/lang/bin/pip3.7 /var/lang/bin/pip3 \
+    && rm -Rf ./Python-${PYTHON_VERSION}
 
 FROM ${python_version}
 COPY pyproject.toml poetry.lock ./
+
+RUN pip3 install --upgrade pip wheel && pip3 install --upgrade six cython cmake hypothesis poetry
 RUN poetry config virtualenvs.create false --local && poetry install --no-root
 
 RUN rm -f pyproject.toml poetry.lock

building/lambda/build-docker-images.sh

@@ -9,11 +9,12 @@ export DOCKER_BUILDKIT=1
 ARCH=$(arch)
 
 if [ "${ARCH}" != "aarch64" ]; then
-  # Python 3.7
+  # Python 3.7 (using public.ecr.aws/lambda/python:3.8 is intentional)
   docker build \
     --pull \
     --tag awswrangler-build-py37 \
-    --build-arg base_image=public.ecr.aws/lambda/python:3.7 \
+    --build-arg base_image=public.ecr.aws/lambda/python:3.8 \
+    --build-arg python_version=python37 \
     .
 fi
 

building/lambda/build-lambda-layer.sh

@@ -15,7 +15,7 @@ export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
 
 git clone \
   --depth 1 \
-  --branch apache-arrow-8.0.0 \
+  --branch apache-arrow-10.0.0 \
   --single-branch \
   https://github.com/apache/arrow.git
 
@@ -50,6 +50,7 @@ popd
 
 pushd arrow/python
 
+export CMAKE_PREFIX_PATH=${ARROW_HOME}${CMAKE_PREFIX_PATH:+:${CMAKE_PREFIX_PATH}}
 export ARROW_PRE_0_15_IPC_FORMAT=0
 export PYARROW_WITH_HDFS=0
 export PYARROW_WITH_FLIGHT=0