Skip to content

Commit 5ba0a8d

Browse files
author
Rub21
committed
Merge branch 'develop' into web_env_config
2 parents d4e904b + fe9b8b7 commit 5ba0a8d

File tree

14 files changed

+118
-60
lines changed

14 files changed

+118
-60
lines changed

‎chartpress.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ charts:
1414
valuesPath: dbBackupRestore.image
1515
planet-dump:
1616
valuesPath: planetDump.image
17+
full-history:
18+
valuesPath: fullHistory.image
1719
replication-job:
1820
valuesPath: replicationJob.image
1921
populate-apidb:

‎docker-compose.yml

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ services:
8181
echo Sleep the replication-job for a while!;
8282
sleep 40;
8383
echo Creating the replication-job!;
84-
./start.sh"
84+
/start.sh"
8585
env_file:
8686
- ./.env
8787
######################################################
@@ -168,6 +168,8 @@ services:
168168
build:
169169
context: ./images/tiler-server
170170
dockerfile: Dockerfile
171+
env_file:
172+
- ./.env-tiler
171173
volumes:
172174
- ./tiler-server-data:/mnt/data
173175
depends_on:
@@ -220,4 +222,19 @@ services:
220222
volumes:
221223
- ./overpass-api-db:/db
222224
env_file:
223-
- ./.env-overpass
225+
- ./.env-overpass
226+
227+
######################################################
228+
### Tasking Manager section
229+
######################################################
230+
tasking-manager-api:
231+
image: osmseed-tasking-manager-api:v1
232+
build:
233+
context: ./images/tasking-manager-api
234+
dockerfile: Dockerfile
235+
ports:
236+
- '5000:5000'
237+
volumes:
238+
- ./tasking-manager-api-db:/db
239+
env_file:
240+
- ./.env-tasking-manager

‎images/full-history/Dockerfile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:latest
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-n487.hbce9469
22
WORKDIR /mnt/data
33
COPY ./start.sh .
4-
CMD ./start.sh
4+
CMD ./start.sh

‎images/full-history/start.sh

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ else
88
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
99
fi
1010

11-
# Fixing name for file
11+
# Fixing name for historical file
1212
date=$(date '+%y%m%d_%H%M')
13-
fullHistoryFile=history-${date}.osm.bz2
13+
fullHistoryFile=history-${date}.osh.pbf
1414
# In case overwrite the file
1515
if [ "$OVERWRITE_FHISTORY_FILE" == "true" ]; then
16-
fullHistoryFile=history-latest.osm.bz2
16+
fullHistoryFile=history-latest.osh.pbf
1717
fi
18-
# State file
18+
19+
# State file nname
1920
stateFile="state.txt"
21+
osm_tmp_file="osm_tmp.osm"
2022

2123
# Creating full history
2224
osmosis --read-apidb-change \
@@ -26,21 +28,25 @@ osmosis --read-apidb-change \
2628
password=$POSTGRES_PASSWORD \
2729
validateSchemaVersion=no \
2830
readFullHistory=yes \
29-
--write-xml-change \
30-
compressionMethod=bzip2 \
31-
$fullHistoryFile
31+
--write-xml-change \
32+
compressionMethod=auto \
33+
$osm_tmp_file
34+
35+
# Convert file to PBF file
36+
osmium cat $osm_tmp_file -o $fullHistoryFile
37+
osmium fileinfo $fullHistoryFile
3238

3339
# AWS
3440
if [ $CLOUDPROVIDER == "aws" ]; then
35-
echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" > $stateFile
41+
echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
3642
# Upload to S3
37-
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
38-
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
43+
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
44+
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read
3945
fi
4046

4147
# Google Storage
4248
if [ $CLOUDPROVIDER == "gcp" ]; then
43-
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" > $stateFile
49+
echo "https://storage.cloud.google.com/$GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile" >$stateFile
4450
# Upload to GS
4551
gsutil cp -a public-read $fullHistoryFile $GCP_STORAGE_BUCKET/planet/full-history/$fullHistoryFile
4652
gsutil cp -a public-read $stateFile $GCP_STORAGE_BUCKET/planet/full-history/$stateFile

‎images/osm-processor/Dockerfile

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
1-
FROM ubuntu:16.04
1+
FROM ubuntu:20.04
22
ENV workdir /mnt/data
3+
34
RUN apt-get -y update
4-
RUN apt-get -y install \
5+
RUN DEBIAN_FRONTEND=noninteractive apt-get -y install \
56
build-essential \
67
libboost-program-options-dev \
78
libbz2-dev \
@@ -10,24 +11,24 @@ RUN apt-get -y install \
1011
cmake \
1112
pandoc \
1213
git \
13-
python-pip \
14+
python3 \
15+
python3-pip \
1416
curl \
1517
unzip \
1618
wget \
1719
software-properties-common \
18-
python-software-properties \
1920
libz-dev \
20-
zlib1g-dev \
2121
gdal-bin \
2222
tar \
2323
bzip2 \
2424
clang \
2525
default-jre \
2626
default-jdk \
2727
gradle \
28-
apt-utils
28+
apt-utils \
29+
postgresql-client
2930

30-
# Install osmosis
31+
# Install osmosis 0.47
3132
RUN git clone https://github.com/openstreetmap/osmosis.git
3233
WORKDIR osmosis
3334
RUN git checkout 9cfb8a06d9bcc948f34a6c8df31d878903d529fc
@@ -38,22 +39,29 @@ RUN ln -s "$PWD"/dist/bin/osmosis /usr/bin/osmosis
3839
RUN osmosis --version 2>&1 | grep "Osmosis Version"
3940

4041
# Install osmium-tool
41-
RUN git clone https://github.com/mapbox/protozero
42-
RUN cd protozero && git checkout 23d48fd2a441c6e3b2852ff84a0ba398e48f74be && mkdir build && cd build && cmake .. && make && make install
43-
RUN git clone https://github.com/osmcode/libosmium
44-
RUN cd libosmium && git checkout a1f88fe44b01863a1ac84efccff54b98bb2dc886 && mkdir build && cd build && cmake .. && make && make install
45-
RUN git clone https://github.com/osmcode/osmium-tool
46-
RUN cd osmium-tool && git checkout ddbcb44f3ec0c1a8d729e69e3cee40d25f5a00b4 && mkdir build && cd build && cmake .. && make && make install
42+
RUN apt-get -y install \
43+
libbz2-dev \
44+
libgd-dev \
45+
libosmium2-dev \
46+
libprotozero-dev \
47+
libsqlite3-dev \
48+
make \
49+
jq \
50+
ca-certificates
51+
52+
# Other useful packages
53+
RUN apt-get install -y \
54+
osmium-tool \
55+
pyosmium \
56+
rsync \
57+
tmux \
58+
zsh
59+
60+
RUN pip install osmium
4761

4862
# Install AWS and GCP cli
4963
RUN pip install awscli
5064
RUN curl -sSL https://sdk.cloud.google.com | bash
5165
RUN ln -f -s /root/google-cloud-sdk/bin/gsutil /usr/bin/gsutil
5266

53-
# Install postgresql-client
54-
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
55-
RUN sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list'
56-
RUN apt-get update
57-
RUN apt-get install -y postgresql-client
58-
5967
WORKDIR $workdir

‎images/populate-apidb/Dockerfile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
FROM developmentseed/osmseed-osm-processor:latest
1+
FROM developmentseed/osmseed-osm-processor:0.1.0-n497.h0368039
22
WORKDIR /mnt/data
33
COPY ./start.sh .
44
CMD ./start.sh

‎images/replication-job/Dockerfile

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
FROM developmentseed/osmseed-osm-processor:ohm-7dd7b8a
2-
2+
COPY ./start.sh /
33
WORKDIR /mnt/data
4-
# VOLUME ["/mnt/data"]
5-
COPY ./start.sh .
6-
CMD ./start.sh
4+
CMD /start.sh

‎images/replication-job/start.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ else
88
echo JAVACMD_OPTIONS=\"-server -Xmx$memory\" >~/.osmosis
99
fi
1010

11-
workingDirectory="data"
11+
workingDirectory="/mnt/data/data"
1212

1313
# Check if state.txt exist in the workingDirectory,
1414
# in case the file does not exist locally and does not exist in the cloud the replication will start from 0

‎images/tasking-manager-api/Dockerfile

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ ENV workdir /usr/src/app
77

88
RUN git clone https://github.com/hotosm/tasking-manager.git $workdir
99
# Commits on Oct 23, 2020
10-
RUN cd $workdir && git checkout 951039f86938b9cc9d8b6986c4e5f3423a78e4d3
10+
RUN cd $workdir && git checkout -f 951039f86938b9cc9d8b6986c4e5f3423a78e4d3
1111
WORKDIR $workdir
1212

1313
# Setup backend dependencies
@@ -25,6 +25,7 @@ RUN apk update && \
2525
proj-dev
2626

2727
RUN pip install -r requirements.txt
28+
RUN pip install apscheduler==3.7.0
2829

2930
## INITIALIZATION
3031
EXPOSE 5000

‎images/tiler-imposm/start.sh

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ mkdir -p $imposm3_expire_dir
1818
settingDir=/osm
1919
# Folder to store the imposm expider files in s3 or gs
2020
BUCKET_IMPOSM_FOLDER=imposm
21-
21+
INIT_FILE=/mnt/data/init_done
2222
# Create config file to set variable for imposm
2323
echo "{" > $workDir/config.json
2424
echo "\"cachedir\": \"$cachedir\"," >> $workDir/config.json
@@ -148,23 +148,20 @@ function importData () {
148148
# These index will help speed up tegola tile generation
149149
psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_index.sql
150150

151-
touch /mnt/data/init_done
151+
touch $INIT_FILE
152152
# Update the DB
153153
updateData
154154
}
155155

156156

157-
echo "Connecting to DB..."
157+
echo "Connecting to $POSTGRES_HOST DB"
158158
flag=true
159159
while "$flag" = true; do
160160
pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
161161
# Change flag to false to stop ping the DB
162162
flag=false
163-
hasData=$(psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" \
164-
-c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'" | sed -n 3p | sed 's/ //g')
165-
# After import there are more than 70 tables
166-
echo "$hasData tables in the DB"
167-
if ([ $hasData \> 70 ] && [[ -f /mnt/data/init_done ]]); then
163+
echo "Check if $INIT_FILE exists"
164+
if ([[ -f $INIT_FILE ]]); then
168165
echo "Update the DB with osm data"
169166
updateData
170167
else

0 commit comments

Comments
 (0)