PDAL AWS Lambda layer creation

Author: hobu

.gitignore

@@ -0,0 +1,2 @@
+lambda-deploy.zip
+

Dockerfile

@@ -0,0 +1,111 @@
+FROM lambci/lambda:build-python3.6 as builder
+
+ARG http_proxy
+ARG CURL_VERSION=7.63.0
+ARG GDAL_VERSION=2.4.0
+ARG GEOS_VERSION=3.7.1
+ARG PROJ_VERSION=5.1.0
+ARG LASZIP_VERSION=3.2.9
+ARG GEOTIFF_VERSION=1.4.3
+ARG PDAL_VERSION=1.8.0
+ARG DESTDIR="/build"
+ARG PREFIX="/usr"
+
+RUN \
+  rpm --rebuilddb && \
+  yum makecache fast && \
+  yum install -y \
+    automake16 \
+    libpng-devel \
+    nasm wget tar gcc zlib-devel gcc-c++ curl-devel zip libjpeg-devel rsync git ssh bzip2 automake \
+        glib2-devel libtiff-devel pkg-config libcurl-devel;   # required for pkg-config
+
+RUN \
+    wget https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2.tar.gz; \
+    tar -zxvf cmake-3.13.2.tar.gz; \
+    cd cmake-3.13.2; \
+    ./bootstrap --prefix=/usr ;\
+    make ;\
+    make install DESTDIR=/
+
+
+RUN \
+    wget https://github.com/LASzip/LASzip/releases/download/$LASZIP_VERSION/laszip-src-$LASZIP_VERSION.tar.gz; \
+    tar -xzvf laszip-src-$LASZIP_VERSION.tar.gz; \
+    cd laszip-src-$LASZIP_VERSION;\
+    cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DBUILD_SHARED_LIBS=ON \
+        -DBUILD_STATIC_LIBS=OFF \
+        -DCMAKE_INSTALL_LIBDIR=lib \
+    ; \
+    make; make install; make install DESTDIR= ; cd ..; \
+    rm -rf laszip-src-${LASZIP_VERSION} laszip-src-$LASZIP_VERSION.tar.gz;
+
+RUN \
+    wget http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2; \
+    tar xjf geos*bz2; \
+    cd geos*; \
+    ./configure --prefix=$PREFIX CFLAGS="-O2 -Os"; \
+    make; make install; make install DESTDIR= ;\
+    cd ..; \
+    rm -rf geos*;
+
+RUN \
+    wget http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz; \
+    tar -zvxf proj-$PROJ_VERSION.tar.gz; \
+    cd proj-$PROJ_VERSION; \
+    ./configure --prefix=$PREFIX; \
+    make; make install; make install DESTDIR=; cd ..; \
+    rm -rf proj-$PROJ_VERSION proj-$PROJ_VERSION.tar.gz
+
+RUN \
+    wget https://download.osgeo.org/geotiff/libgeotiff/libgeotiff-$GEOTIFF_VERSION.tar.gz; \
+    tar -xzvf libgeotiff-$GEOTIFF_VERSION.tar.gz; \
+    cd libgeotiff-$GEOTIFF_VERSION; \
+    ./configure \
+        --prefix=$PREFIX --with-proj=/build/usr ;\
+    make; make install; make install DESTDIR=; cd ..; \
+    rm -rf libgeotiff-$GEOTIFF_VERSION.tar.gz libgeotiff-$GEOTIFF_VERSION;
+
+# GDAL
+RUN \
+    wget http://download.osgeo.org/gdal/$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz; \
+    tar -xzvf gdal-$GDAL_VERSION.tar.gz; \
+    cd gdal-$GDAL_VERSION; \
+    ./configure \
+        --prefix=$PREFIX \
+        --with-geotiff=$DESTDIR/usr \
+        --with-tiff=/usr \
+        --with-curl=yes \
+        --without-python \
+        --with-geos=$DESTDIR/usr/bin/geos-config \
+        --with-hide-internal-symbols=yes \
+        CFLAGS="-O2 -Os" CXXFLAGS="-O2 -Os"; \
+    make ; make install; make install DESTDIR= ; \
+    cd $BUILD; rm -rf gdal-$GDAL_VERSION*
+
+RUN \
+    git clone https://github.com/PDAL/PDAL.git; \
+    cd PDAL; \
+    mkdir -p _build; \
+    cd _build; \
+    cmake .. \
+        -G "Unix Makefiles" \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_CXX_FLAGS="-std=c++11" \
+        -DCMAKE_MAKE_PROGRAM=make \
+        -DBUILD_PLUGIN_I3S=ON \
+        -DWITH_LASZIP=ON \
+        -DCMAKE_LIBRARY_PATH:FILEPATH="$DESTDIR/usr/lib" \
+        -DCMAKE_INCLUDE_PATH:FILEPATH="$DESTDIR/usr/include" \
+        -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DWITH_TESTS=OFF \
+        -DCMAKE_INSTALL_LIBDIR=lib \
+    ; \
+    make ; make install; make install DESTDIR= ;
+
+RUN rm /build/usr/lib/*.la ; rm /build/usr/lib/*.a
+RUN ldconfig
+ADD package-pdal.sh /
+

README.rst

@@ -0,0 +1,54 @@
+================================================================================
+PDAL Lambda Layer for AWS
+================================================================================
+
+In December 2018, AWS released `Lambda Layers`_, which allow you to stack together
+software runtime layers with AWS Lambda functions. Layers relax the requirements that
+single Lambda functions must be less than 50mb, and now allow you to stack up to
+five layers at 50mb each, and these layers can be composed with versioning and
+other AWS management facilities that you would expect to use.
+
+This repository borrows concepts from Development Seed's `GeoLambda`_ project, but
+it simplifies the effort to a few things:
+
+1. Build a Docker container based on https://github.com/lambci/lambci for `PDAL`_
+   and its dependencies
+
+2. Construct a ``.zip`` file package of the binaries and ``.so`` libraries needed
+   to run PDAL and GDAL as a Lambda Layer
+
+3. Provide a script to create a public PDAL Lambda Layer using the package
+
+Instructions
+--------------------------------------------------------------------------------
+
+1. Build the Package. It's going to run docker and do a bunch of stuff. At the
+   end when it is successful, you should have a ``lambda-deploy.zip`` file
+   in your directory.
+
+   ::
+
+      $ ./build.sh
+
+2. Set your ``AWS_PROFILE`` and ``AWS_REGION`` variables:
+
+
+   ::
+
+      $ export AWS_PROFILE=hobu
+      $ export AWS_REGION=us-east-1
+
+3. Execute the ``create-lambda-layer.sh`` bash script. It requires the `jq`_
+   command in your path.
+
+   ::
+
+      $ ./create-lambda-layer.sh
+      Published version 6 for Lambda layer pdal
+      Setting execution access to public for version 6 for Lambda layer pdal
+      Layer pdal is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal'
+
+_`Lambda Layers`: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
+_`GeoLambda`: https://github.com/developmentseed/geolambda
+_`jq`: https://stedolan.github.io/jq/
+_`PDAL`: https://pdal.io

build.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+CONTAINER="pdal-lambda"
+docker build -t $CONTAINER -f Dockerfile .
+rm -rf lambda
+mkdir -p lambda
+
+docker run -v `pwd`:/output $CONTAINER /package-pdal.sh

create-lambda-layer.sh

@@ -0,0 +1,37 @@
+
+LAYERNAME="pdal"
+LAYER=$(aws lambda publish-layer-version \
+    --layer-name $LAYERNAME \
+    --description "PDAL 1.8.0 softare" \
+    --zip-file fileb://./lambda-deploy.zip \
+    --compatible-runtimes "provided" \
+    --license-info BSD \
+    --region $AWS_REGION \
+    --profile $AWS_PROFILE)
+
+
+VERSION=$(aws lambda list-layers --region $AWS_REGION |jq '.Layers[]| select(.LayerName=="'$LAYERNAME'").LatestMatchingVersion.Version' -r)
+
+echo "Published version $VERSION for Lambda layer $LAYERNAME"
+
+LAYER=$(aws lambda get-layer-version \
+    --layer-name pdal \
+    --version-number $VERSION \
+    --region $AWS_REGION \
+    --profile $AWS_PROFILE)
+
+
+echo "Setting execution access to public for version $VERSION for Lambda layer $LAYERNAME"
+PERMISSION=$(aws lambda add-layer-version-permission \
+    --layer-name pdal \
+    --version-number $VERSION \
+    --statement-id "run-pdal-publicly" \
+    --principal '*' \
+    --action lambda:GetLayerVersion \
+    --region $AWS_REGION \
+    --profile $AWS_PROFILE )
+
+LAYERARN=$(echo $LAYER | jq -r .LayerArn)
+
+echo "Layer $LAYERNAME is available at '$LAYERARN'"
+

package-pdal.sh

@@ -0,0 +1,28 @@
+#!/bin/sh
+
+export DEPLOY_DIR=/output/lambda
+
+PACKAGE_NAME="lambda-deploy.zip"
+
+# make deployment directory and add lambda handler
+cp -r /build/usr/lib $DEPLOY_DIR/lib
+cp -r /build/usr/bin $DEPLOY_DIR/bin
+cp /usr/lib64/libjpeg.so.62.0.0 $DEPLOY_DIR/lib/
+cp /usr/lib64/libxml2.so.2.9.1 $DEPLOY_DIR/lib/
+cp /usr/lib64/liblzma.so.5.0.99 $DEPLOY_DIR/lib/
+cp /usr/lib64/libtiff.so.5.2.0 $DEPLOY_DIR/lib/
+cp /usr/lib64/libpng.so.3.49.0 $DEPLOY_DIR/lib/
+cp /usr/lib64/libsqlite3.so.0.8.6 $DEPLOY_DIR/lib/
+rm -rf $DEPLOY_DIR/lib/*.a
+rm -rf $DEPLOY_DIR/lib/libpdal_plugin*
+rm -rf $DEPLOY_DIR/lib/python3.6
+
+rsync -ax /build/usr/share/gdal $DEPLOY_DIR/share/
+rsync -ax /build/usr/share/proj $DEPLOY_DIR/share/
+
+cd $DEPLOY_DIR
+zip -ruq ../$PACKAGE_NAME ./
+rm -rf $DEPLOY_DIR
+
+
+