Merge pull request #166 from awslabs/lambda-layers

Updating the lambda layers building processes

Author: igorborgest

CONTRIBUTING.md

@@ -62,9 +62,9 @@ We may ask you to sign a [Contributor License Agreement (CLA)](http://en.wikiped
 
 * AWS Data Wrangler practically only makes integrations. So we prefer to dedicate our energy / time writing integration tests instead of unit tests. We really like an end-to-end approach for all features.
 
-* All integration tests are between the local enviroment and a remote and real AWS service.
+* All integration tests are between the local environment and a remote and real AWS services.
 
-* We have a Cloudformation to set up the AWS end (testing/template.yaml).
+* We have a Cloudformation to set up the AWS end (testing/cloudformation.yaml).
 
 ## Step-by-step
 

building/build-image.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -ex
+
+VERSION=$(python -c "import awswrangler as wr; print(wr.__version__)")
+echo "Building Lambda Layers for AWS Data Wrangler ${VERSION}"
+DIR_NAME=$(dirname "$PWD")
+
+pushd lambda
+
+# Building all related docker images
+./build-docker-images.sh
+
+# Building Apache Arrow binary artifacts
+docker run \
+  --volume "$DIR_NAME":/aws-data-wrangler/ \
+  --workdir /aws-data-wrangler/building/lambda \
+  -it \
+  awswrangler-build-py36 \
+  build-apache-arrow.sh
+
+# Generating PyArrow Files for Python 3.6
+#docker run \
+#  --volume "$DIR_NAME":/aws-data-wrangler/ \
+#  --workdir /aws-data-wrangler/building/lambda \
+#  -it \
+#  awswrangler-build-py36 \
+#  build-pyarrow.sh
+
+# Building the AWS Lambda Layer for Python 3.6
+#docker run \
+#  --volume "$DIR_NAME":/aws-data-wrangler/ \
+#  --workdir /aws-data-wrangler/building/lambda \
+#  -it \
+#  awswrangler-build-py36 \
+#  build-layer.sh "${VERSION}-py3.6"

building/build-lambda-layers.sh

@@ -1,22 +1,24 @@
-FROM lambci/lambda:build-python3.6
+ARG base_image
+ARG py_dev
+
+FROM ${base_image}
 
 RUN yum install -y \
     boost-devel \
     jemalloc-devel \
     bison \
     flex \
     autoconf \
-    python36-devel
+    ${py_dev}
 
-RUN pip3 install --upgrade pip six cython pytest cmake wheel
+RUN pip3 install --upgrade pip six cython cmake
 
 ADD requirements.txt /root/
 RUN pip3 install -r /root/requirements.txt
-RUN rm -rf /root/requirements.txt
+
 ADD requirements-dev.txt /root/
 RUN pip3 install -r /root/requirements-dev.txt
-RUN rm -rf /root/requirements-dev.txt
 
-RUN pip3 install boto3==1.12.22 botocore==1.15.22 awscli==1.18.22
+RUN rm -rf /root/requirements*
 
 ENTRYPOINT ["/bin/sh"]

building/build-pyarrow.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -ex
+
+rm -rf arrow dist
+
+git clone \
+  --branch apache-arrow-0.16.0 \
+  --single-branch \
+  https://github.com/apache/arrow.git
+
+export ARROW_HOME=$(pwd)/dist
+export LD_LIBRARY_PATH=$(pwd)/dist/lib:$LD_LIBRARY_PATH
+
+mkdir dist
+mkdir arrow/cpp/build
+pushd arrow/cpp/build
+
+cmake \
+    -DCMAKE_INSTALL_PREFIX=$ARROW_HOME \
+    -DCMAKE_INSTALL_LIBDIR=lib \
+    -DARROW_FLIGHT=OFF \
+    -DARROW_GANDIVA=OFF \
+    -DARROW_ORC=OFF \
+    -DARROW_WITH_SNAPPY=ON \
+    -DARROW_WITH_ZLIB=ON \
+    -DARROW_PARQUET=ON \
+    -DARROW_CSV=OFF \
+    -DARROW_PYTHON=ON \
+    -DARROW_PLASMA=OFF \
+    -DARROW_BUILD_TESTS=OFF \
+    ..
+make -j
+make install
+
+popd
+
+rm -rf arrow

building/deploy-source.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -ex
+
+cp ../../requirements.txt .
+cp ../../requirements-dev.txt .
+
+# Python 3.6
+docker build \
+  --tag awswrangler-build-py36 \
+  --build-arg base_image=lambci/lambda:build-python3.6 \
+  --build-arg py_dev=python36-devel \
+  .
+
+# Python 3.7
+docker build \
+  --tag awswrangler-build-py37 \
+  --build-arg base_image=lambci/lambda:build-python3.7 \
+  --build-arg py_dev=python37-devel \
+  .
+
+# Python 3.8
+docker build \
+  --tag awswrangler-build-py38 \
+  --build-arg base_image=lambci/lambda:build-python3.8 \
+  --build-arg py_dev=python38-devel \
+  .
+
+rm -rf requirements*

building/Dockerfile renamed to building/lambda/Dockerfile

@@ -1,21 +1,13 @@
 #!/usr/bin/env bash
 set -ex
 
-# Go back to AWSWRANGLER directory
-pushd /aws-data-wrangler
+FILENAME="awswrangler-layer-${1}.zip"
 
-# Build PyArrow files
-pushd building
-./build-pyarrow.sh
-popd
+pushd /aws-data-wrangler
 
-# Preparing directories
 mkdir -p dist
-rm -rf python
-rm -f "awswrangler-layer.zip"
-rm -f "dist/awswrangler-layer.zip"
+rm -rf python "${FILENAME}" "dist/${FILENAME}"
 
-# Building
 pip install . -t ./python
 rm -rf python/pyarrow*
 rm -rf python/boto*
@@ -24,8 +16,7 @@ rm -f /aws-data-wrangler/dist/pyarrow_files/pyarrow/libparquet.so
 rm -f /aws-data-wrangler/dist/pyarrow_files/pyarrow/libarrow_python.so
 cp -r /aws-data-wrangler/dist/pyarrow_files/pyarrow* python/
 find python -wholename "*/tests/*" -type f -delete
-zip -r9 "awswrangler-layer.zip" ./python
-mv "awswrangler-layer.zip" dist/
+zip -r9 "${FILENAME}" ./python
+mv "${FILENAME}" dist/
 
-# Cleaning up the directory again
 rm -rf python

building/lambda/build-apache-arrow.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -ex
+
+rm -rf  /aws-data-wrangler/dist/pyarrow_files
+
+pushd arrow/python
+
+export ARROW_PRE_0_15_IPC_FORMAT=0
+export PYARROW_WITH_HDFS=0
+export PYARROW_WITH_FLIGHT=0
+export PYARROW_WITH_GANDIVA=0
+export PYARROW_WITH_ORC=0
+export PYARROW_WITH_CUDA=0
+export PYARROW_WITH_PARQUET=1
+
+python setup.py build_ext \
+  --build-type=release \
+  --bundle-arrow-cpp \
+  bdist_wheel
+
+pip install dist/pyarrow-*.whl -t /aws-data-wrangler/dist/pyarrow_files
+
+rm -rf dist