Skip to content

Commit 3c31538

Browse files
author
Ruben Lopez M
authored
Merge pull request #201 from developmentseed/full_history_updates
Update full-history container generator
2 parents 47dfb65 + b8ef1d2 commit 3c31538

File tree

6 files changed

+73
-35
lines changed

6 files changed

+73
-35
lines changed

‎chartpress.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ charts:
1414
valuesPath: dbBackupRestore.image
1515
planet-dump:
1616
valuesPath: planetDump.image
17+
full-history:
18+
valuesPath: fullHistory.image
1719
replication-job:
1820
valuesPath: replicationJob.image
1921
populate-apidb:

‎images/full-history/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:latest
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-n487.hbce9469
22
WORKDIR /mnt/data
33
COPY ./start.sh .
4-
CMD ./start.sh
4+
CMD ./start.sh

‎images/full-history/start.sh

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ else
88
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
99
fi
1010

11-
# Fixing name for file
11+
# Fixing name for historical file
1212
date=$(date '+%y%m%d_%H%M')
13-
fullHistoryFile=history-${date}.osm.bz2
13+
fullHistoryFile=history-${date}.osh.pbf
1414
# In case overwrite the file
1515
if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
16-
fullHistoryFile=history-latest.osm.bz2
16+
fullHistoryFile=history-latest.osh.pbf
1717
fi
18-
# State file
18+
19+
# State file nname
1920
stateFile="state.txt"
21+
osm_tmp_file="osm_tmp.osm"
2022

2123
# Creating full history
2224
osmosis --read-apidb-change \
@@ -26,21 +28,25 @@ osmosis --read-apidb-change \
2628
password=$POSTGRES_PASSWORD \
2729
validateSchemaVersion=no \
2830
readFullHistory=yes \
29-
--write-xml-change \
30-
compressionMethod=bzip2 \
31-
$fullHistoryFile
31+
--write-xml-change \
32+
compressionMethod=auto \
33+
$osm_tmp_file
34+
35+
# Convert file to PBF file
36+
osmium cat $osm_tmp_file -o $fullHistoryFile
37+
osmium fileinfo $fullHistoryFile
3238

3339
# AWS
3440
if [ $CLOUDPROVIDER == "aws" ]; then
35-
echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" > $stateFile
41+
echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
3642
# Upload to S3
37-
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
38-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
43+
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
44+
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
3945
fi
4046

4147
# Google Storage
4248
if [ $CLOUDPROVIDER == "gcp" ]; then
43-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" > $stateFile
49+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile
4450
# Upload to GS
4551
gsutil cp -a public-read $fullHistoryFile $GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile
4652
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/$stateFile

‎images/osm-processor/Dockerfile

Lines changed: 25 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
FROM ubuntu:16.04
1+
FROM ubuntu:20.04
22
ENV workdir /mnt/data
3+
34
RUN apt-get -y update
4-
RUN apt-get -y install \
5+
RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \
56
build-essential \
67
libboost-program-options-dev \
78
libbz2-dev \
@@ -10,14 +11,13 @@ RUN apt-get -y install \
1011
cmake \
1112
pandoc \
1213
git \
13-
python-pip \
14+
python3 \
15+
python3-pip \
1416
curl \
1517
unzip \
1618
wget \
1719
software-properties-common \
18-
python-software-properties \
1920
libz-dev \
20-
zlib1g-dev \
2121
gdal-bin \
2222
tar \
2323
bzip2 \
@@ -27,7 +27,7 @@ RUN apt-get -y install \
2727
gradle \
2828
apt-utils
2929

30-
# Install osmosis
30+
# Install osmosis 0.47
3131
RUN git clone https://github.com/openstreetmap/osmosis.git
3232
WORKDIR osmosis
3333
RUN git checkout 9cfb8a06d9bcc948f34a6c8df31d878903d529fc
@@ -38,22 +38,29 @@ RUN ln -s "$PWD"/dist/bin/osmosis /usr/bin/osmosis
3838
RUN osmosis --version 2>&1 | grep "Osmosis Version"
3939

4040
# Install osmium-tool
41-
RUN git clone https://github.com/mapbox/protozero
42-
RUN cd protozero && git checkout 23d48fd2a441c6e3b2852ff84a0ba398e48f74be && mkdir build && cd build && cmake .. && make && make install
43-
RUN git clone https://github.com/osmcode/libosmium
44-
RUN cd libosmium && git checkout a1f88fe44b01863a1ac84efccff54b98bb2dc886 && mkdir build && cd build && cmake .. && make && make install
45-
RUN git clone https://github.com/osmcode/osmium-tool
46-
RUN cd osmium-tool && git checkout ddbcb44f3ec0c1a8d729e69e3cee40d25f5a00b4 && mkdir build && cd build && cmake .. && make && make install
41+
RUN apt-get -y install \
42+
libbz2-dev \
43+
libgd-dev \
44+
libosmium2-dev \
45+
libprotozero-dev \
46+
libsqlite3-dev \
47+
make \
48+
jq \
49+
ca-certificates
50+
51+
# Other useful packages
52+
RUN apt-get install -y \
53+
osmium-tool \
54+
pyosmium \
55+
rsync \
56+
tmux \
57+
zsh
58+
59+
RUN pip install osmium
4760

4861
# Install AWS and GCP cli
4962
RUN pip install awscli
5063
RUN curl -sSL https://sdk.cloud.google.com | bash
5164
RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
5265

53-
# Install postgresql-client
54-
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
55-
RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list'
56-
RUN apt-get update
57-
RUN apt-get install -y postgresql-client
58-
5966
WORKDIR $workdir

‎images/tiler-server/Dockerfile

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,13 @@ RUN apk add --update \
2828
python-dev \
2929
linux-headers \
3030
musl-dev \
31-
openssl-dev
31+
openssl-dev \
32+
curl
3233

3334
# Install aws-cli and gsutil
34-
RUN pip install awscli \
35-
&& pip install gsutil
35+
RUN pip install awscli
36+
RUN curl -sSL https://sdk.cloud.google.com | bash
37+
RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
3638

3739
RUN pip install mercantile \
3840
&& apk del build-deps \

‎osm-seed/values.yaml

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,29 @@ memcached:
141141
idEditor:
142142
enabled: false
143143

144+
# ====================================================================================================
145+
# Variables for full-history container
146+
# ====================================================================================================
144147
fullHistory:
145-
enabled: false
148+
enabled: true
149+
image:
150+
name: ''
151+
tag: ''
152+
nodeSelector:
153+
enabled: true
154+
label_key: nodegroup-type
155+
label_value: ohm-non-tiler
156+
schedule: '* */24 * * *'
157+
env:
158+
OVERWRITE_FHISTORY_FILE: false
159+
resources:
160+
enabled: false
161+
requests:
162+
memory: "14Gi"
163+
cpu: "4"
164+
limits:
165+
memory: "16Gi"
166+
cpu: "4"
146167

147168
# ====================================================================================================
148169
# Variables for osm-seed database backup and restore

0 commit comments

Comments
 (0)