pdal and pdal-python layers

Author: hobu

Dockerfile

@@ -173,9 +173,6 @@ RUN \
     && cd /var/task \
     && rm -rf xerces*
 
-ADD https://api.github.com/repos/PDAL/PDAL/commits?sha=${PDAL_VERSION} \
-    /tmp/bust-cache
-
 
 RUN \
     git clone https://github.com/PDAL/PDAL.git --branch ${PDAL_VERSION} \
@@ -194,15 +191,15 @@ RUN \
         -DWITH_ZSTD=ON \
         -DCMAKE_LIBRARY_PATH:FILEPATH="$DESTDIR/usr/lib" \
         -DCMAKE_INCLUDE_PATH:FILEPATH="$DESTDIR/usr/include" \
-        -DCMAKE_INSTALL_PREFIX=$PREFIX \
+        -DCMAKE_INSTALL_PREFIX=/usr \
         -DWITH_TESTS=OFF \
         -DCMAKE_INSTALL_LIBDIR=lib \
     && make  -j ${PARALLEL} \
     && make  install \
     && make install DESTDIR=/ \
     && DESTDIR=/ make install  \
     && cd /var/task \
-    && rm -rf pdal*
+    && rm -rf PDAL*
 
 #RUN \
 #    git clone https://github.com/PDAL/python.git pdal-python \
@@ -227,6 +224,12 @@ RUN \
     && cd /var/task \
     && rm -rf entwine*
 
+
+RUN DESTDIR= python -m pip install PDAL --prefix /build/python \
+    && python -m pip install scipy scikit-learn --prefix /build/python \
+    && cd /var/task \
+    && rm -rf pdal-python
+
 RUN rm /build/usr/lib/*.la ; rm /build/usr/lib/*.a
 RUN rm /build/usr/lib64/*.a
 RUN ldconfig

README.rst

@@ -44,9 +44,12 @@ Instructions
    ::
 
       $ ./create-lambda-layer.sh
-      Published version 32 for Lambda layer pdal
-      Setting execution access to public for version 32 for Lambda layer pdal
+      Published version 38 for Lambda layer pdal
+      Setting execution access to public for version 38 for Lambda layer pdal
       Layer pdal is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal'
+      Published version 2 for Lambda layer pdal-python
+      Setting execution access to public for version 2 for Lambda layer pdal-python
+      Layer pdal-python is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal-python'
 
 .. _`Lambda Layers`: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
 .. _`GeoLambda`: https://github.com/developmentseed/geolambda

create-lambda-layer.sh

@@ -1,37 +1,48 @@
+#!/bin/bash
 
-LAYERNAME="pdal"
-LAYER=$(aws lambda publish-layer-version \
-    --layer-name $LAYERNAME \
-    --description "PDAL 2.0.1 softare" \
-    --zip-file fileb://./lambda-deploy.zip \
-    --compatible-runtimes "provided" \
-    --license-info BSD \
-    --region $AWS_REGION \
-    --profile $AWS_PROFILE)
+#LAYERNAME="pdal"
 
+create_layer()
+{
+    LAYERNAME="$1"
+    ZIPFILE="$2"
+    DESCRIPTION="$3"
 
-VERSION=$(aws lambda list-layers --region $AWS_REGION |jq '.Layers[]| select(.LayerName=="'$LAYERNAME'").LatestMatchingVersion.Version' -r)
+    LAYER=$(aws lambda publish-layer-version \
+        --layer-name $LAYERNAME \
+        --description "$DESCRIPTION" \
+        --zip-file fileb://./$ZIPFILE\
+        --compatible-runtimes "provided" \
+        --license-info BSD \
+        --region $AWS_REGION \
+        --profile $AWS_PROFILE)
 
-echo "Published version $VERSION for Lambda layer $LAYERNAME"
 
-LAYER=$(aws lambda get-layer-version \
-    --layer-name pdal \
-    --version-number $VERSION \
-    --region $AWS_REGION \
-    --profile $AWS_PROFILE)
+    VERSION=$(aws lambda list-layers --region $AWS_REGION |jq '.Layers[]| select(.LayerName=="'$LAYERNAME'").LatestMatchingVersion.Version' -r)
 
+    echo "Published version $VERSION for Lambda layer $LAYERNAME"
 
-echo "Setting execution access to public for version $VERSION for Lambda layer $LAYERNAME"
-PERMISSION=$(aws lambda add-layer-version-permission \
-    --layer-name pdal \
-    --version-number $VERSION \
-    --statement-id "run-pdal-publicly" \
-    --principal '*' \
-    --action lambda:GetLayerVersion \
-    --region $AWS_REGION \
-    --profile $AWS_PROFILE )
+    LAYER=$(aws lambda get-layer-version \
+        --layer-name $LAYERNAME \
+        --version-number $VERSION \
+        --region $AWS_REGION \
+        --profile $AWS_PROFILE)
 
-LAYERARN=$(echo $LAYER | jq -r .LayerArn)
 
-echo "Layer $LAYERNAME is available at '$LAYERARN'"
+    echo "Setting execution access to public for version $VERSION for Lambda layer $LAYERNAME"
+    PERMISSION=$(aws lambda add-layer-version-permission \
+        --layer-name $LAYERNAME \
+        --version-number $VERSION \
+        --statement-id "run-pdal-publicly" \
+        --principal '*' \
+        --action lambda:GetLayerVersion \
+        --region $AWS_REGION \
+        --profile $AWS_PROFILE )
 
+    LAYERARN=$(echo $LAYER | jq -r .LayerArn)
+
+    echo "Layer $LAYERNAME is available at '$LAYERARN'"
+
+}
+create_layer "pdal" "pdal-lambda-deploy.zip" "PDAL 2.2.0 software"
+create_layer "pdal-python" "sk-lambda-deploy.zip" "PDAL Python 2.3.5 software"

package-pdal.sh

@@ -1,42 +1,90 @@
 #!/bin/sh
 
-export DEPLOY_DIR=/output/lambda
-
-PACKAGE_NAME="lambda-deploy.zip"
-mkdir -p $DEPLOY_DIR/lib
-mkdir -p $DEPLOY_DIR/share
-
-cp -r /build/usr/lib/* $DEPLOY_DIR/lib
-cp -r /build/usr/lib64/* $DEPLOY_DIR/lib
-cp -r /build/usr/bin $DEPLOY_DIR/bin
-cp /usr/lib64/libjpeg.so.62.0.0 $DEPLOY_DIR/lib/
-cp /usr/lib64/libxml2.so.2.9.1 $DEPLOY_DIR/lib/
-cp /usr/lib64/liblzma.so.5.2.2 $DEPLOY_DIR/lib/
-cp /usr/lib64/libtiff.so.5.2.0 $DEPLOY_DIR/lib/
-cp /usr/lib64/libpng.so.3.49.0 $DEPLOY_DIR/lib/
-cp /usr/lib/libsqlite3.so.0.8.6 $DEPLOY_DIR/lib/
-rm -rf $DEPLOY_DIR/lib/*.a
-rm -rf $DEPLOY_DIR/bin/projinfo
-rm -rf $DEPLOY_DIR/bin/gie
-rm -rf $DEPLOY_DIR/lib/libpdal_plugin*
-rm -rf $DEPLOY_DIR/lib/python3.6
-cp -r /build/python/* $DEPLOY_DIR
-
-FILES=$DEPLOY_DIR/lib/*.so*
+export PDAL_DEPLOY_DIR=/output/pdal
+export SK_DEPLOY_DIR=/output/sk
+
+PDAL_PACKAGE_NAME="pdal-lambda-deploy.zip"
+SK_PACKAGE_NAME="pdal-python-lambda-deploy.zip"
+
+make_dirs()
+{
+    OUTDIR=$1
+
+    mkdir -p $OUTDIR/lib
+    mkdir -p $OUTDIR/bin
+    mkdir -p $OUTDIR/share
+}
+
+make_dirs $PDAL_DEPLOY_DIR
+make_dirs $SK_DEPLOY_DIR
+
+cd /build
+FILES=$(find . -name '*.so*')
+
 for f in $FILES
 do
 echo "Stripping unneeded symbols from $f"
 	strip --strip-unneeded $f
 done;
 
-rsync -ax /build/usr/share/gdal $DEPLOY_DIR/share/
-rsync -ax /build/usr/share/proj $DEPLOY_DIR/share/
+rm -rf /build/usr/lib/*.a
+
+cd /build
+find . -name '*.pyc' -delete
+find . -name '*cmake*' -exec rm -rf {} +;
+find . -name '*tests*' -exec rm -rf {} +;
+find . -name '*test*' -exec rm -rf {} +;
+find . -name '*__pycache__*' -exec rm -rf {} +;
+find . -name '*datasets*' -exec rm -rf {} +;
+
+rm -rf ./python/lib/python3.7/site-packages/joblib/test
+cd python
+find . -type d -name "datasets" -exec rm -rf \;
+cp lib/libpdal_* $SK_DEPLOY_DIR/lib
+cd ..
+
+cp -r /build/usr/lib/* $PDAL_DEPLOY_DIR/lib
+cp -r /build/usr/lib/libpdal_* $PDAL_DEPLOY_DIR/lib
+cp -r /build/usr/lib64/* $PDAL_DEPLOY_DIR/lib
+cp -r /build/usr/bin/gdal* $PDAL_DEPLOY_DIR/bin
+cp -r /build/usr/bin/entwine $PDAL_DEPLOY_DIR/bin
+cp -r /build/usr/bin/pdal $PDAL_DEPLOY_DIR/bin
+cp -r /build/usr/bin/cs2cs $PDAL_DEPLOY_DIR/bin
+cp -r /build/usr/bin/ogr* $PDAL_DEPLOY_DIR/bin
+
+rm -rf /build/lib/cmake
+
+copy_support()
+{
+    OUTDIR=$1
+    cp /usr/lib64/libjpeg.so.62.0.0 $OUTDIR/lib/
+    cp /usr/lib64/libxml2.so.2.9.1 $OUTDIR/lib/
+    cp /usr/lib64/liblzma.so.5.2.2 $OUTDIR/lib/
+    cp /usr/lib64/libtiff.so.5.2.0 $OUTDIR/lib/
+    cp /usr/lib64/libpng.so.3.49.0 $OUTDIR/lib/
+    cp /usr/lib/libsqlite3.so.0.8.6 $OUTDIR/lib/
+}
+
+copy_support $PDAL_DEPLOY_DIR
+copy_support $SK_DEPLOY_DIR
+
+cp -r /build/python/lib/python3.7/ $SK_DEPLOY_DIR/lib/python3.7/
+
+
+
+
+#rsync -ax /build/usr/share/gdal $PDAL_DEPLOY_DIR/share/
+rsync -ax /build/usr/share/proj/proj.db $PDAL_DEPLOY_DIR/share/proj/
 
-cd $DEPLOY_DIR
-zip --symlinks -9 -ruq ../$PACKAGE_NAME ./
-rm -rf $DEPLOY_DIR
-echo "writing deploy dir $DEPLOY_DIR"
+cd $PDAL_DEPLOY_DIR
+zip --symlinks -9 -ruq ../$PDAL_PACKAGE_NAME ./
+rm -rf $PDAL_DEPLOY_DIR
+echo "writing deploy dir $PDAL_DEPLOY_DIR"
 
+cd $SK_DEPLOY_DIR
+zip --symlinks -9 -ruq ../$SK_PACKAGE_NAME ./
+rm -rf $SK_DEPLOY_DIR
+echo "writing deploy dir $SK_DEPLOY_DIR"