pdal-python layer creation

Author: hobu

Dockerfile

@@ -27,13 +27,15 @@ RUN \
         glib2-devel libtiff-devel pkg-config libcurl-devel;   # required for pkg-config
 
 
+#    curl -O https://mirror.centos.org/centos/6/extras/x86_64/Packages/centos-release-scl-rh-2-3.el6.centos.noarch.rpm && \
+#    curl -O https://mirror.centos.org/centos/6/extras/x86_64/Packages/centos-release-scl-7-3.el6.centos.noarch.rpm && \
 
 RUN \
     yum install -y iso-codes && \
-    curl -O http://vault.centos.org/6.5/SCL/x86_64/scl-utils/scl-utils-20120927-11.el6.centos.alt.x86_64.rpm && \
-    curl -O http://vault.centos.org/6.5/SCL/x86_64/scl-utils/scl-utils-build-20120927-11.el6.centos.alt.x86_64.rpm && \
-    curl -O http://mirror.centos.org/centos/6/extras/x86_64/Packages/centos-release-scl-rh-2-3.el6.centos.noarch.rpm && \
-    curl -O http://mirror.centos.org/centos/6/extras/x86_64/Packages/centos-release-scl-7-3.el6.centos.noarch.rpm && \
+    curl -O https://vault.centos.org/6.5/SCL/x86_64/scl-utils/scl-utils-20120927-11.el6.centos.alt.x86_64.rpm && \
+    curl -O https://vault.centos.org/6.5/SCL/x86_64/scl-utils/scl-utils-build-20120927-11.el6.centos.alt.x86_64.rpm && \
+    curl -O https://mirror.facebook.net/centos/6/extras/x86_64/Packages/centos-release-scl-rh-2-3.el6.centos.noarch.rpm && \
+    curl -O https://mirror.facebook.net/centos/6/extras/x86_64/Packages/centos-release-scl-7-3.el6.centos.noarch.rpm && \
     rpm -Uvh *.rpm  && \
     rm *.rpm &&  \
     yum install -y devtoolset-7-gcc-c++ devtoolset-7-make devtoolset-7-build ;
@@ -43,13 +45,15 @@ SHELL [ "/usr/bin/scl", "enable", "devtoolset-7"]
 RUN gcc --version
 
 
-RUN \
-    wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
-    && chmod +x cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
-    && ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh  --skip-license --prefix=/usr \
-    && cd /var/task \
-    && rm -rf cmake*
+#RUN \
+#    wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
+#    && chmod +x cmake-${CMAKE_VERSION}-Linux-x86_64.sh \
+#    && ./cmake-${CMAKE_VERSION}-Linux-x86_64.sh  --skip-license --prefix=/usr \
+#    && cd /var/task \
+#    && rm -rf cmake*
 
+RUN \
+    pip install cmake ninja
 
 RUN git clone https://github.com/LASzip/LASzip.git laszip \
     && cd laszip \
@@ -74,6 +78,7 @@ RUN \
         -G Ninja \
         -DCMAKE_INSTALL_PREFIX=/usr/ \
         -DCMAKE_BUILD_TYPE="Release" \
+        -DBUILD_TESTING=OFF \
      .  \
     && ninja -j ${PARALLEL} \
     && ninja install \
@@ -123,6 +128,7 @@ RUN \
         -G Ninja \
         -DCMAKE_INSTALL_PREFIX=/usr/ \
         -DCMAKE_BUILD_TYPE="Release" \
+        -DBUILD_TESTING=OFF \
      ..  \
     && ninja -j ${PARALLEL} \
     && ninja install \
@@ -167,6 +173,7 @@ RUN \
         -G "Ninja" \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_INSTALL_PREFIX=/usr \
+        -DBUILD_TESTING=OFF \
     && ninja -j ${PARALLEL} \
     && ninja install \
     && DESTDIR= ninja install \
@@ -201,15 +208,6 @@ RUN \
     && cd /var/task \
     && rm -rf PDAL*
 
-#RUN \
-#    git clone https://github.com/PDAL/python.git pdal-python \
-#    && cd pdal-python \
-#    && python -m pip install --upgrade pip \
-#    && pip install numpy scikit-build packaging ninja cmake cython \
-#    && bash -c 'python setup.py build' \
-#    && bash -c 'python setup.py bdist_wheel' \
-#    && ls dist
-
 RUN \
     git clone https://github.com/connormanning/entwine.git --branch ${ENTWINE_VERSION} \
     && cd entwine \
@@ -224,14 +222,40 @@ RUN \
     && cd /var/task \
     && rm -rf entwine*
 
+RUN wget https://github.com/Reference-LAPACK/lapack/archive/v3.9.0.tar.gz \
+    && tar zxvf v3.9.0.tar.gz \
+    && cd lapack-3.9.0/ \
+    && mkdir -p _build \
+    && cd _build \
+    && cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -G Ninja -DBUILD_SHARED_LIBS=ON \
+    && ninja -j ${PARALLEL} \
+    && ninja install \
+    && DESTDIR=/ ninja install \
+    && cd /var/task \
+    && rm -rf lapack*
+
+RUN wget https://github.com/xianyi/OpenBLAS/archive/v0.3.10.tar.gz \
+    && tar zxvf v0.3.10.tar.gz \
+    && cd OpenBLAS-0.3.10/ \
+    && mkdir -p _build \
+    && cd _build \
+    && cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release -DBUILD_TESTING=OFF -G Ninja -DBUILD_SHARED_LIBS=ON \
+    && ninja -j ${PARALLEL} \
+    && ninja install \
+    && DESTDIR=/ ninja install \
+    && cd /var/task \
+    && rm -rf OpenBLAS*
 
+# scikit-build will respect our DESTDIR and put things in the wrong directory
 RUN DESTDIR= python -m pip install PDAL --prefix /build/python \
-    && python -m pip install scipy scikit-learn --prefix /build/python \
+    && python -m pip install pandas scipy scikit-learn --no-binary :all: --verbose --prefix /build/python \
     && cd /var/task \
     && rm -rf pdal-python
 
+RUN DESTDIR= python -m pip install pytz  --target /build/python/lib/python3.7/site-packages/
+
 RUN rm /build/usr/lib/*.la ; rm /build/usr/lib/*.a
-RUN rm /build/usr/lib64/*.a
+RUN rm  /build/usr/lib64/*.a
 RUN ldconfig
 ADD package-pdal.sh /
 

README.rst

@@ -44,12 +44,12 @@ Instructions
    ::
 
       $ ./create-lambda-layer.sh
-      Published version 38 for Lambda layer pdal
-      Setting execution access to public for version 38 for Lambda layer pdal
-      Layer pdal is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal'
-      Published version 2 for Lambda layer pdal-python
-      Setting execution access to public for version 2 for Lambda layer pdal-python
-      Layer pdal-python is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal-python'
+      Published version 40 for Lambda layer pdal
+      Setting execution access to public for version 40 for Lambda layer pdal
+      Layer pdal is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal:40'
+      Published version 17 for Lambda layer pdal-python
+      Setting execution access to public for version 17 for Lambda layer pdal-python
+      Layer pdal-python is available at 'arn:aws:lambda:us-east-1:163178234892:layer:pdal-python:17'
 
 .. _`Lambda Layers`: https://docs.aws.amazon.com/lambda/latest/dg/configuration-layers.html
 .. _`GeoLambda`: https://github.com/developmentseed/geolambda

build.sh

@@ -3,7 +3,6 @@
 CONTAINER="pdal-lambda"
 docker build -t $CONTAINER -f Dockerfile .
 rm -rf lambda
-rm lambda-deploy.zip
 mkdir -p lambda
 
 docker run -v `pwd`:/output $CONTAINER /package-pdal.sh

create-lambda-layer.sh

@@ -7,12 +7,13 @@ create_layer()
     LAYERNAME="$1"
     ZIPFILE="$2"
     DESCRIPTION="$3"
+    RUNTIMES="$4"
 
     LAYER=$(aws lambda publish-layer-version \
         --layer-name $LAYERNAME \
         --description "$DESCRIPTION" \
         --zip-file fileb://./$ZIPFILE\
-        --compatible-runtimes "provided" \
+        --compatible-runtimes "$RUNTIMES" \
         --license-info BSD \
         --region $AWS_REGION \
         --profile $AWS_PROFILE)
@@ -44,5 +45,6 @@ create_layer()
     echo "Layer $LAYERNAME is available at '$LAYERARN'"
 
 }
-create_layer "pdal" "pdal-lambda-deploy.zip" "PDAL 2.2.0 software"
-create_layer "pdal-python" "sk-lambda-deploy.zip" "PDAL Python 2.3.5 software"
+
+create_layer "pdal" "pdal-lambda-deploy.zip" "PDAL 2.2.0 software" "provided"
+create_layer "pdal-python" "pdal-python-lambda-deploy.zip" "PDAL Python 2.3.5 software" "python3.7"

package-pdal.sh

@@ -1,46 +1,63 @@
 #!/bin/sh
 
-export PDAL_DEPLOY_DIR=/output/pdal
-export SK_DEPLOY_DIR=/output/sk
+export PDAL_DEPLOY_DIR=/pdal-package
+export SK_DEPLOY_DIR=/sk-package
 
 PDAL_PACKAGE_NAME="pdal-lambda-deploy.zip"
 SK_PACKAGE_NAME="pdal-python-lambda-deploy.zip"
 
-make_dirs()
-{
-    OUTDIR=$1
+mkdir -p $PDAL_DEPLOY_DIR/lib
+mkdir -p $PDAL_DEPLOY_DIR/bin
+mkdir -p $PDAL_DEPLOY_DIR/share
 
-    mkdir -p $OUTDIR/lib
-    mkdir -p $OUTDIR/bin
-    mkdir -p $OUTDIR/share
-}
+PYTHON_DEPLOY_DIR=$SK_DEPLOY_DIR/python/lib/python3.7/site-packages
+mkdir -p $PYTHON_DEPLOY_DIR
+mkdir -p $SK_DEPLOY_DIR/lib
 
-make_dirs $PDAL_DEPLOY_DIR
-make_dirs $SK_DEPLOY_DIR
 
-cd /build
-FILES=$(find . -name '*.so*')
+cp /build/python/lib/libpdal_plugin_filter_python.so $SK_DEPLOY_DIR/lib
+cp /build/python/lib/libpdal_plugin_reader_numpy.so $SK_DEPLOY_DIR/lib
+#cp -r /usr/lib64/atlas/lib*.so* $SK_DEPLOY_DIR/lib
+cp -r /usr/lib64/libgfortran.so* $SK_DEPLOY_DIR/lib
+cp -r /usr/lib64/libquadmath.so* $SK_DEPLOY_DIR/lib
+cp -r /usr/lib64/libblas.so* $SK_DEPLOY_DIR/lib
+cp -r /usr/lib64/libopenblas.so* $SK_DEPLOY_DIR/lib
+cp -r /usr/lib64/liblapack.so* $SK_DEPLOY_DIR/lib
 
-for f in $FILES
-do
-echo "Stripping unneeded symbols from $f"
-	strip --strip-unneeded $f
-done;
+
+strip_symbols()
+{
+    DIR=$1
+    pushd $1
+
+    FILES=$(find . -name '*.so*')
+
+    for f in $FILES
+    do
+    echo "Stripping unneeded symbols from $f"
+        strip --strip-unneeded $f
+    done;
+    popd
+}
+
+strip_symbols /build
+strip_symbols $SK_DEPLOY_DIR
 
 rm -rf /build/usr/lib/*.a
 
 cd /build
 find . -name '*.pyc' -delete
 find . -name '*cmake*' -exec rm -rf {} +;
-find . -name '*tests*' -exec rm -rf {} +;
-find . -name '*test*' -exec rm -rf {} +;
+#find . -name '*tests*' -exec rm -rf {} +;
+#find . -name '*test*' -exec rm -rf {} +;
 find . -name '*__pycache__*' -exec rm -rf {} +;
 find . -name '*datasets*' -exec rm -rf {} +;
 
 rm -rf ./python/lib/python3.7/site-packages/joblib/test
 cd python
 find . -type d -name "datasets" -exec rm -rf \;
 cp lib/libpdal_* $SK_DEPLOY_DIR/lib
+
 cd ..
 
 cp -r /build/usr/lib/* $PDAL_DEPLOY_DIR/lib
@@ -57,6 +74,7 @@ rm -rf /build/lib/cmake
 copy_support()
 {
     OUTDIR=$1
+
     cp /usr/lib64/libjpeg.so.62.0.0 $OUTDIR/lib/
     cp /usr/lib64/libxml2.so.2.9.1 $OUTDIR/lib/
     cp /usr/lib64/liblzma.so.5.2.2 $OUTDIR/lib/
@@ -66,9 +84,15 @@ copy_support()
 }
 
 copy_support $PDAL_DEPLOY_DIR
-copy_support $SK_DEPLOY_DIR
 
-cp -r /build/python/lib/python3.7/ $SK_DEPLOY_DIR/lib/python3.7/
+
+#cp -r /usr/lib64/atlas/libptf77blas.so* $SK_DEPLOY_DIR/lib
+#cp -r /usr/lib64/atlas/libgfortran.so* $SK_DEPLOY_DIR/lib
+
+rm /build/python/lib/libpdal_plugin_filter_python.so
+rm /build/python/lib/libpdal_plugin_reader_numpy.so
+
+cp -r /build/python/lib/python3.7/site-packages/* $PYTHON_DEPLOY_DIR
 
 
 
@@ -86,5 +110,7 @@ zip --symlinks -9 -ruq ../$SK_PACKAGE_NAME ./
 rm -rf $SK_DEPLOY_DIR
 echo "writing deploy dir $SK_DEPLOY_DIR"
 
+cp /$PDAL_PACKAGE_NAME /output
+cp /$SK_PACKAGE_NAME /output