Skip to content

Commit 6eddf86

Browse files
hobuhobu
committed
parallelize builds and strip for smaller packaging
1 parent 7f521bd commit 6eddf86

File tree

2 files changed

+186
-89
lines changed

2 files changed

+186
-89
lines changed

‎Dockerfile‎

Lines changed: 174 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@ ARG PDAL_VERSION=master
1111
ARG ENTWINE_VERSION=2.1.0
1212
ARG DESTDIR="/build"
1313
ARG PREFIX="/usr"
14+
ARG PARALLEL=2
15+
1416

1517
RUN \
1618
rpm --rebuilddb && \
@@ -19,7 +21,7 @@ RUN \
1921
automake16 \
2022
libpng-devel \
2123
nasm wget tar zlib-devel curl-devel zip libjpeg-devel rsync git ssh bzip2 automake \
22-
jq-libs jq-devel jq xz-devel openssl-devel \
24+
jq-libs jq-devel jq xz-devel openssl-devel ninja-build wget \
2325
glib2-devel libtiff-devel pkg-config libcurl-devel; # required for pkg-config
2426

2527

@@ -40,83 +42,109 @@ RUN gcc --version
4042

4143

4244
RUN \
43-
wget https://github.com/Kitware/CMake/releases/download/v3.15.1/cmake-3.15.1.tar.gz; \
44-
tar -zxvf cmake-3.15.1.tar.gz; \
45-
cd cmake-3.15.1; \
46-
./bootstrap --prefix=/usr ;\
47-
make ;\
48-
make install DESTDIR=/
45+
wget https://github.com/Kitware/CMake/releases/download/v3.15.1/cmake-3.15.1.tar.gz \
46+
&& tar -zxvf cmake-3.15.1.tar.gz \
47+
&& cd cmake-3.15.1 \
48+
&& ./bootstrap --parallel=${PARALLEL} --prefix=/usr \
49+
&& make -j ${PARALLEL} \
50+
&& make install DESTDIR=/ \
51+
&& cd / \
52+
&& rm -rf cmake*
4953

5054

51-
RUN \
52-
wget https://github.com/LASzip/LASzip/releases/download/$LASZIP_VERSION/laszip-src-$LASZIP_VERSION.tar.gz; \
53-
tar -xzvf laszip-src-$LASZIP_VERSION.tar.gz; \
54-
cd laszip-src-$LASZIP_VERSION;\
55-
cmake -DCMAKE_BUILD_TYPE=Release \
56-
-DCMAKE_INSTALL_PREFIX=$PREFIX \
57-
-DBUILD_SHARED_LIBS=ON \
58-
-DBUILD_STATIC_LIBS=OFF \
59-
-DCMAKE_INSTALL_LIBDIR=lib \
60-
; \
61-
make; make install; make install DESTDIR= ; cd ..; \
62-
rm -rf laszip-src-${LASZIP_VERSION} laszip-src-$LASZIP_VERSION.tar.gz;
55+
RUN git clone https://github.com/LASzip/LASzip.git laszip \
56+
&& cd laszip \
57+
&& git checkout ${LASZIP_VERSION} \
58+
&& cmake \
59+
-G Ninja \
60+
-DCMAKE_INSTALL_PREFIX=/usr/ \
61+
-DCMAKE_BUILD_TYPE="Release" \
62+
. \
63+
&& ninja -j ${PARALLEL} \
64+
&& ninja install \
65+
&& DESTDIR=/ ninja install \
66+
&& cd / \
67+
&& rm -rf laszip*
6368

64-
RUN \
65-
wget http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2; \
66-
tar xjf geos*bz2; \
67-
cd geos*; \
68-
./configure --prefix=$PREFIX CFLAGS="-O2 -Os"; \
69-
make; make install; make install DESTDIR= ;\
70-
cd ..; \
71-
rm -rf geos*;
7269

73-
RUN \
74-
wget http://download.osgeo.org/proj/proj-$PROJ_VERSION.tar.gz; \
75-
tar -zvxf proj-$PROJ_VERSION.tar.gz; \
76-
cd proj-$PROJ_VERSION; \
77-
./configure --prefix=$PREFIX; \
78-
make; make install; make install DESTDIR=; cd ..; \
79-
rm -rf proj-$PROJ_VERSION proj-$PROJ_VERSION.tar.gz
8070

8171
RUN \
82-
wget https://github.com/OSGeo/libgeotiff/releases/download/$GEOTIFF_VERSION/libgeotiff-$GEOTIFF_VERSION.tar.gz; \
83-
tar -xzvf libgeotiff-$GEOTIFF_VERSION.tar.gz; \
84-
cd libgeotiff-$GEOTIFF_VERSION; \
85-
./configure \
86-
--prefix=$PREFIX --with-proj=/build/usr ;\
87-
make; make install; make install DESTDIR=; cd ..; \
88-
rm -rf libgeotiff-$GEOTIFF_VERSION.tar.gz libgeotiff-$GEOTIFF_VERSION;
89-
90-
# GDAL
91-
RUN \
92-
wget http://download.osgeo.org/gdal/$GDAL_VERSION/gdal-$GDAL_VERSION.tar.gz; \
93-
tar -xzvf gdal-$GDAL_VERSION.tar.gz; \
94-
cd gdal-$GDAL_VERSION; \
95-
./configure \
96-
--prefix=$PREFIX \
97-
--with-geotiff=$DESTDIR/usr \
98-
--with-tiff=/usr \
99-
--with-curl=yes \
100-
--without-python \
101-
--with-geos=$DESTDIR/usr/bin/geos-config \
102-
--with-hide-internal-symbols=yes \
103-
CFLAGS="-O2 -Os" CXXFLAGS="-O2 -Os"; \
104-
make ; make install; make install DESTDIR= ; \
105-
cd $BUILD; rm -rf gdal-$GDAL_VERSION*
72+
wget http://download.osgeo.org/geos/geos-$GEOS_VERSION.tar.bz2 && \
73+
tar xjf geos*bz2 && \
74+
cd geos* \
75+
&& cmake \
76+
-G Ninja \
77+
-DCMAKE_INSTALL_PREFIX=/usr/ \
78+
-DCMAKE_BUILD_TYPE="Release" \
79+
. \
80+
&& ninja -j ${PARALLEL} \
81+
&& ninja install \
82+
&& DESTDIR=/ ninja install \
83+
&& cd / \
84+
&& rm -rf geos*
85+
86+
RUN git clone https://github.com/OSGeo/PROJ.git --branch ${PROJ_VERSION} proj \
87+
&& cd proj \
88+
&& ./autogen.sh \
89+
&& ./configure --prefix=/usr \
90+
&& make -j ${PARALLEL} \
91+
&& make install \
92+
&& DESTDIR=/ make install \
93+
&& cd / \
94+
&& rm -rf /proj*
95+
96+
RUN git clone --branch master https://github.com/OSGeo/libgeotiff.git --branch ${GEOTIFF_VERSION} \
97+
&& cd libgeotiff/libgeotiff \
98+
&& ./autogen.sh \
99+
&& ./configure --prefix=/usr --with-proj=/usr \
100+
&& make -j ${PARALLEL} \
101+
&& make install \
102+
&& DESTDIR=/ make install \
103+
&& cd / \
104+
&& rm -rf /libgeotiff*
105+
106+
107+
RUN git clone --branch release/ https://github.com/OSGeo/gdal.git --branch v${GDAL_VERSION} \
108+
&& cd gdal/gdal \
109+
&& ./configure --prefix=/usr \
110+
--mandir=/usr/share/man \
111+
--includedir=/usr/include/gdal \
112+
--with-threads \
113+
--with-grass=no \
114+
--with-hide-internal-symbols=yes \
115+
--with-rename-internal-libtiff-symbols=yes \
116+
--with-rename-internal-libgeotiff-symbols=yes \
117+
--with-libtiff=/usr/ \
118+
--with-geos=/usr/bin/geos-config \
119+
--with-geotiff=/usr \
120+
--with-proj=/usr \
121+
--with-ogdi=no \
122+
--with-curl \
123+
--with-ecw=no \
124+
--with-mrsid=no \
125+
&& make -j ${PARALLEL} \
126+
&& make install \
127+
&& DESTDIR=/ make install \
128+
&& cd / \
129+
&& rm -rf /gdal*
130+
106131

107132
RUN \
108133
wget https://github.com/facebook/zstd/releases/download/v1.4.2/zstd-1.4.2.tar.gz \
109134
&& tar zxvf zstd-1.4.2.tar.gz \
110135
&& cd zstd-1.4.2/build/cmake \
111136
&& mkdir -p _build \
112137
&& cd _build \
113-
&& cmake .. \
114-
-G "Unix Makefiles" \
115-
-DCMAKE_BUILD_TYPE=Release \
116-
-DCMAKE_INSTALL_PREFIX=/usr \
117-
&& make \
118-
&& make install \
119-
&& make install DESTDIR=
138+
&& cmake \
139+
-G Ninja \
140+
-DCMAKE_INSTALL_PREFIX=/usr/ \
141+
-DCMAKE_BUILD_TYPE="Release" \
142+
.. \
143+
&& ninja -j ${PARALLEL} \
144+
&& ninja install \
145+
&& DESTDIR=/ ninja install \
146+
&& cd / \
147+
&& rm -rf zstd*
120148

121149
RUN \
122150
wget http://apache.mirrors.hoobly.com//xerces/c/3/sources/xerces-c-3.2.2.tar.gz \
@@ -125,20 +153,22 @@ RUN \
125153
&& mkdir -p _build \
126154
&& cd _build \
127155
&& cmake .. \
128-
-G "Unix Makefiles" \
156+
-G "Ninja" \
129157
-DCMAKE_BUILD_TYPE=Release \
130158
-DCMAKE_INSTALL_PREFIX=/usr \
131-
&& make \
132-
&& make install \
133-
&& make install DESTDIR=
159+
&& ninja -j ${PARALLEL} \
160+
&& ninja install \
161+
&& DESTDIR= ninja install \
162+
&& cd / \
163+
&& rm -rf xerces*
134164

135165
RUN \
136-
git clone https://github.com/PDAL/PDAL.git; \
137-
cd PDAL; \
138-
git checkout $PDAL_VERSION; \
139-
mkdir -p _build; \
140-
cd _build; \
141-
cmake .. \
166+
git clone https://github.com/PDAL/PDAL.git --branch ${PDAL_VERSION} \
167+
&& cd PDAL \
168+
&& git checkout $PDAL_VERSION \
169+
&& mkdir -p _build \
170+
&& cd _build \
171+
&& cmake .. \
142172
-G "Unix Makefiles" \
143173
-DCMAKE_BUILD_TYPE=Release \
144174
-DCMAKE_CXX_FLAGS="-std=c++11" \
@@ -152,23 +182,81 @@ RUN \
152182
-DCMAKE_INSTALL_PREFIX=$PREFIX \
153183
-DWITH_TESTS=OFF \
154184
-DCMAKE_INSTALL_LIBDIR=lib \
155-
; \
156-
make ; make install; make install DESTDIR= ;
185+
&& make -j ${PARALLEL} \
186+
&& make install \
187+
&& make install DESTDIR=/ \
188+
&& cd / \
189+
&& rm -rf pdal*
157190

158191
RUN \
159-
git clone https://github.com/connormanning/entwine.git; \
160-
cd entwine; \
161-
git checkout $ENTWINE_VERSION; \
162-
mkdir -p _build; \
163-
cd _build; \
164-
cmake -G "Unix Makefiles" \
192+
git clone https://github.com/connormanning/entwine.git --branch ${ENTWINE_VERSION} \
193+
&& cd entwine \
194+
&& mkdir -p _build \
195+
&& cd _build \
196+
&& cmake -G "Ninja" \
165197
-DCMAKE_INSTALL_PREFIX=/usr \
166-
-DCMAKE_BUILD_TYPE=Release .. && \
167-
make -j4 && \
168-
make install DESTDIR= ;
198+
-DCMAKE_BUILD_TYPE=Release .. \
199+
&& ninja -j ${PARALLEL} \
200+
&& ninja install \
201+
&& DESTDIR=/ ninja install \
202+
&& cd / \
203+
&& rm -rf entwine*
169204

170205
RUN rm /build/usr/lib/*.la ; rm /build/usr/lib/*.a
171206
RUN rm /build/usr/lib64/*.a
172207
RUN ldconfig
173208
ADD package-pdal.sh /
174209

210+
211+
# --disable-driver-airsar \
212+
# --disable-driver-arg \
213+
# --disable-driver-blx \
214+
# --disable-driver-bsb \
215+
# --disable-driver-cals \
216+
# --disable-driver-ceos \
217+
# --disable-driver-ceos2 \
218+
# --disable-driver-coasp \
219+
# --disable-driver-cosar \
220+
# --disable-driver-ctg \
221+
# --disable-driver-dimap \
222+
# --disable-driver-elas \
223+
# --disable-driver-ingr \
224+
# --disable-driver-jdem \
225+
# --disable-driver-r \
226+
# --disable-driver-pds \
227+
# --disable-driver-prf \
228+
# --disable-driver-rmf \
229+
# --disable-driver-safe \
230+
# --disable-driver-saga \
231+
# --disable-driver-sigdem \
232+
# --disable-driver-sgi \
233+
# --disable-driver-zmap \
234+
# --disable-driver-cad \
235+
# --disable-driver-dgn \
236+
# --disable-driver-edigeo \
237+
# --disable-driver-geoconcept \
238+
# --disable-driver-georss \
239+
# --disable-driver-gtm \
240+
# --disable-driver-htf \
241+
# --disable-driver-jml \
242+
# --disable-driver-openair \
243+
# --disable-driver-rec \
244+
# --disable-driver-segukooa \
245+
# --disable-driver-segy \
246+
# --disable-driver-selafin \
247+
# --disable-driver-xplane \
248+
# --disable-driver-eeda \
249+
# --disable-driver-plmosaic \
250+
# --disable-driver-rda \
251+
# --disable-driver-vdv \
252+
# --disable-driver-sxf \
253+
# --disable-driver-sua \
254+
# --disable-driver-amigocloud \
255+
# --disable-driver-daas \
256+
# --disable-driver-elastic \
257+
# --disable-driver-gft \
258+
# --disable-driver-ngw \
259+
# --disable-driver-plscenes \
260+
# --disable-driver-rasterlite \
261+
# --disable-driver-vfk \
262+

‎package-pdal.sh‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,11 @@
33
export DEPLOY_DIR=/output/lambda
44

55
PACKAGE_NAME="lambda-deploy.zip"
6+
mkdir -p $DEPLOY_DIR/lib
7+
mkdir -p $DEPLOY_DIR/share
68

7-
# make deployment directory and add lambda handler
8-
cp -r /build/usr/lib $DEPLOY_DIR/lib
9-
cp -r /build/usr/lib64/*.so* $DEPLOY_DIR/lib
9+
cp -r /build/usr/lib/* $DEPLOY_DIR/lib
10+
cp -r /build/usr/lib64/* $DEPLOY_DIR/lib
1011
cp -r /build/usr/bin $DEPLOY_DIR/bin
1112
cp /usr/lib64/libjpeg.so.62.0.0 $DEPLOY_DIR/lib/
1213
cp /usr/lib64/libxml2.so.2.9.1 $DEPLOY_DIR/lib/
@@ -19,13 +20,21 @@ rm -rf $DEPLOY_DIR/bin/projinfo
1920
rm -rf $DEPLOY_DIR/bin/gie
2021
rm -rf $DEPLOY_DIR/lib/libpdal_plugin*
2122
rm -rf $DEPLOY_DIR/lib/python3.6
23+
FILES=$DEPLOY_DIR/lib/*.so*
24+
for f in $FILES
25+
do
26+
echo "Stripping unneeded symbols from $f"
27+
strip --strip-unneeded $f
28+
done;
2229

2330
rsync -ax /build/usr/share/gdal $DEPLOY_DIR/share/
2431
rsync -ax /build/usr/share/proj $DEPLOY_DIR/share/
2532

2633
cd $DEPLOY_DIR
2734
zip --symlinks -9 -ruq ../$PACKAGE_NAME ./
2835
rm -rf $DEPLOY_DIR
36+
echo "writing deploy dir $DEPLOY_DIR"
37+
2938

3039

3140

0 commit comments

Comments
 (0)