Revisiting the lambda layer building

Author: igorborgest

building/build-lambda-layers.sh

@@ -2,34 +2,35 @@
 set -ex
 
 VERSION=$(python -c "import awswrangler as wr; print(wr.__version__)")
-echo "Building Lambda Layers for AWS Data Wrangler ${VERSION}"
 DIR_NAME=$(dirname "$PWD")
 
+echo "Building Lambda Layers for AWS Data Wrangler ${VERSION}"
+
 pushd lambda
 
 # Building all related docker images
 ./build-docker-images.sh
 
-# Building Apache Arrow binary artifacts
+# Python 3.6
 docker run \
-  --volume "$DIR_NAME":/aws-data-wrangler/ \
-  --workdir /aws-data-wrangler/building/lambda \
-  -it \
-  awswrangler-build-py36 \
-  build-apache-arrow.sh
+ --volume "$DIR_NAME":/aws-data-wrangler/ \
+ --workdir /aws-data-wrangler/building/lambda \
+ -it \
+ awswrangler-build-py36 \
+ build-lambda-layer.sh "${VERSION}-py3.6" "ninja"
 
-# Generating PyArrow Files for Python 3.6
-#docker run \
-#  --volume "$DIR_NAME":/aws-data-wrangler/ \
-#  --workdir /aws-data-wrangler/building/lambda \
-#  -it \
-#  awswrangler-build-py36 \
-#  build-pyarrow.sh
+# Python 3.7
+docker run \
+ --volume "$DIR_NAME":/aws-data-wrangler/ \
+ --workdir /aws-data-wrangler/building/lambda \
+ -it \
+ awswrangler-build-py37 \
+ build-lambda-layer.sh "${VERSION}-py3.7" "ninja"
 
-# Building the AWS Lambda Layer for Python 3.6
-#docker run \
-#  --volume "$DIR_NAME":/aws-data-wrangler/ \
-#  --workdir /aws-data-wrangler/building/lambda \
-#  -it \
-#  awswrangler-build-py36 \
-#  build-layer.sh "${VERSION}-py3.6"
+# Python 3.8
+docker run \
+ --volume "$DIR_NAME":/aws-data-wrangler/ \
+ --workdir /aws-data-wrangler/building/lambda \
+ -it \
+ awswrangler-build-py38 \
+ build-lambda-layer.sh "${VERSION}-py3.8" "ninja-build"

building/lambda/Dockerfile

@@ -9,9 +9,10 @@ RUN yum install -y \
     bison \
     flex \
     autoconf \
+    ninja-build \
     ${py_dev}
 
-RUN pip3 install --upgrade pip six cython cmake
+RUN pip3 install --upgrade pip six cython cmake hypothesis
 
 ADD requirements.txt /root/
 RUN pip3 install -r /root/requirements.txt

building/lambda/build-apache-arrow.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+set -ex
+
+FILENAME="awswrangler-layer-${1}.zip"
+NINJA=${2}
+
+pushd /aws-data-wrangler
+rm -rf python dist/pyarrow_files "dist/${FILENAME}" "${FILENAME}"
+popd
+
+rm -rf dist arrow
+
+export ARROW_HOME=$(pwd)/dist
+export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
+
+git clone \
+  --branch apache-arrow-0.16.0 \
+  --single-branch \
+  https://github.com/apache/arrow.git
+
+mkdir dist
+mkdir arrow/cpp/build
+pushd arrow/cpp/build
+
+cmake \
+    -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+    -DCMAKE_INSTALL_LIBDIR=lib \
+    -DARROW_FLIGHT=OFF \
+    -DARROW_GANDIVA=OFF \
+    -DARROW_ORC=OFF \
+    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_WITH_ZLIB=ON \
+    -DARROW_PARQUET=ON \
+    -DARROW_CSV=OFF \
+    -DARROW_PYTHON=ON \
+    -DARROW_PLASMA=OFF \
+    -DARROW_BUILD_TESTS=OFF \
+    -GNinja \
+    ..
+
+eval $NINJA
+eval "${NINJA} install"
+
+popd
+
+pushd arrow/python
+
+export ARROW_PRE_0_15_IPC_FORMAT=0
+export PYARROW_WITH_HDFS=0
+export PYARROW_WITH_FLIGHT=0
+export PYARROW_WITH_GANDIVA=0
+export PYARROW_WITH_ORC=0
+export PYARROW_WITH_CUDA=0
+export PYARROW_WITH_PLASMA=0
+export PYARROW_WITH_PARQUET=1
+
+python setup.py build_ext \
+  --build-type=release \
+  --bundle-arrow-cpp \
+  bdist_wheel
+
+pip install dist/pyarrow-*.whl -t /aws-data-wrangler/dist/pyarrow_files
+
+popd
+
+pushd /aws-data-wrangler
+
+pip install . -t ./python
+
+rm -rf python/pyarrow*
+rm -rf python/boto*
+
+rm -f /aws-data-wrangler/dist/pyarrow_files/pyarrow/libarrow.so
+rm -f /aws-data-wrangler/dist/pyarrow_files/pyarrow/libparquet.so
+rm -f /aws-data-wrangler/dist/pyarrow_files/pyarrow/libarrow_python.so
+
+cp -r /aws-data-wrangler/dist/pyarrow_files/pyarrow* python/
+
+find python -wholename "*/tests/*" -type f -delete
+
+zip -r9 "${FILENAME}" ./python
+mv "${FILENAME}" dist/
+
+rm -rf python dist/pyarrow_files "${FILENAME}"
+
+popd
+
+rm -rf dist arrow

building/lambda/build-docker-images.sh

@@ -4,3 +4,4 @@ set -ex
 pip install --upgrade pip
 pip install -r requirements-dev.txt
 pip install -r requirements.txt
+pip install -e .