Skip to content

Commit 001076a

Browse files
author
Ruben L. Mendoza
authored
Merge pull request #200 from developmentseed/taginfo
Taginfo container
2 parents 450121d + 4b2bdd9 commit 001076a

File tree

14 files changed

+438
-4
lines changed

14 files changed

+438
-4
lines changed

‎.dockerignore

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,18 @@ package.json
2020
db-backup-restore/
2121
nominatim-data/
2222
postgresdata/
23+
taginfo-data/
24+
.env
25+
.env-nominatim
26+
.env-nominatim.example
27+
.env-overpass
28+
.env-overpass.example
29+
.env-taginfo
30+
.env-taginfo.example
31+
.env-tasking-manager.example
32+
.env-tiler
33+
.env-tiler.example
34+
.env.example
2335
# ignore all *.class files in all folders, including build root
2436
**/*.json
2537

‎.env-taginfo.example

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
URL_PLANET_FILE_STATE=https://planet.osm.org/pbf/state.txt
2+
URL_HISTORY_PLANET_FILE_STATE=https://planet.osm.org/pbf/full-history/state.txt
3+
URL_PLANET_FILE=https://planet.osm.org/pbf/planet-latest.osm.pbf
4+
URL_HISTORY_PLANET_FILE=https://planet.osm.org/pbf/full-history/history-latest.osm.pbf
5+
TIME_UPDATE_INTERVAL=5d
6+
INSTANCE_URL=http://localhost:4567
7+
INSTANCE_NAME=OHM Taginfo
8+
INSTANCE_DESCRIPTION="This is a <b>taginfo test instance</b>. Change this text in your <tt>taginfo-config.json</tt>."
9+
INSTANCE_ICON=/img/logo/world.png
10+
INSTANCE_CONTACT=Anonymous
11+
TAGINFO_PROJECT_REPO=https://github.com/taginfo/taginfo-projects.git
12+
DOWNLOAD_DB='languages wiki'
13+
CREATE_DB='db projects chronology'

‎.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
.env-nominatim
44
.env-tasking-manager
55
.env-overpass
6+
.env-taginfo
67
.DS_Store
78

89
usa.values.yaml
@@ -19,6 +20,7 @@ postgres-gis-data/
1920
tiler-imposm-data/
2021
tiler-server-data/
2122
nominatim-pgdata/
23+
taginfo-data/
2224
data/
2325
*.osm
2426
*.pbf

‎chartpress.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,6 @@ charts:
3535
nominatim:
3636
valuesPath: nominatim.image
3737
overpass-api:
38-
valuesPath: overpassApi.image
38+
valuesPath: overpassApi.image
39+
taginfo:
40+
valuesPath: taginfo.image

‎docker-compose.yml

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -237,4 +237,21 @@ services:
237237
volumes:
238238
- ./tasking-manager-api-db:/db
239239
env_file:
240-
- ./.env-tasking-manager
240+
- ./.env-tasking-manager
241+
# #####################################################
242+
# ## Taginfo section
243+
# #####################################################
244+
taginfo:
245+
image: osmseed-taginfo:v1
246+
build:
247+
context: ./images/taginfo
248+
dockerfile: Dockerfile
249+
ports:
250+
- '4567:80'
251+
volumes:
252+
- ./taginfo-data:/apps/data/
253+
env_file:
254+
- ./.env-taginfo
255+
command: >
256+
/bin/bash -c "
257+
./start.sh"

‎images/full-history/start.sh

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,8 @@ osmium fileinfo $fullHistoryFile
3838

3939
# AWS
4040
if [ $CLOUDPROVIDER == "aws" ]; then
41-
echo "https://$AWS_S3_BUCKET.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
41+
AWS_URL=${AWS_S3_BUCKET/s3:\/\//http:\/\/}
42+
echo "$AWS_URL.s3.amazonaws.com/planet/full-history/$fullHistoryFile" >$stateFile
4243
# Upload to S3
4344
aws s3 cp $fullHistoryFile $AWS_S3_BUCKET/planet/full-history/$fullHistoryFile --acl public-read
4445
aws s3 cp $stateFile $AWS_S3_BUCKET/planet/full-history/$stateFile --acl public-read

‎images/taginfo/Dockerfile

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
FROM ruby:2.7
2+
ENV workdir /apps
3+
4+
# Install Taginfo site
5+
RUN apt-get update && apt-get -y install \
6+
curl \
7+
sqlite3 \
8+
sqlite3-pcre \
9+
ruby-passenger \
10+
libapache2-mod-passenger \
11+
git
12+
13+
# Commit ae5a950f7aa4c0de4e706839619a1dc05fc4450a, at 2021-10-18
14+
RUN git clone https://github.com/taginfo/taginfo.git $workdir/taginfo
15+
WORKDIR $workdir/taginfo
16+
RUN echo "gem 'thin' " >>Gemfile
17+
RUN gem install bundler
18+
RUN bundle install
19+
20+
# Install Taginfo tools
21+
RUN apt-get -y install \
22+
cmake \
23+
libbz2-dev \
24+
libexpat1-dev \
25+
libgd-dev \
26+
libicu-dev \
27+
libosmium2-dev \
28+
libprotozero-dev \
29+
libsqlite3-dev \
30+
make \
31+
zlib1g-dev \
32+
jq \
33+
ca-certificates
34+
35+
# Other useful packages
36+
RUN apt-get install -y \
37+
git \
38+
osmium-tool \
39+
pyosmium \
40+
rsync \
41+
tmux \
42+
zsh
43+
44+
RUN git clone https://github.com/taginfo/taginfo-tools.git $workdir/taginfo-tools
45+
WORKDIR $workdir/taginfo-tools
46+
RUN git submodule update --init
47+
RUN mkdir build && cd build && cmake .. && make
48+
49+
RUN apt-get install -y nano vim
50+
COPY overwrite_config.py $workdir/
51+
COPY start.sh $workdir/
52+
53+
WORKDIR $workdir/
54+
CMD $workdir/start.sh

‎images/taginfo/README.md

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# OSM-Seed taginfo
2+
3+
We build a docker container for taginfo software, the container will start the web service and also process required files to create databases.
4+
5+
## Environment Variables
6+
7+
All environment variables are located at [`.env-taginfo.example`](https://github.com/developmentseed/osm-seed/blob/develop/.env-taginfo.example), make a copy and name it as `.env-tagninfo` to use in osm-seed.
8+
9+
- `URL_PLANET_FILE_STATE`: Url to the state file, that contains the URL for the latest planet PBF file. e.g [`state.txt`](https://planet.openhistoricalmap.org.s3.amazonaws.com/planet/state.txt), This is no required in case you set the `URL_PLANET_FILE` env var
10+
11+
- `URL_HISTORY_PLANET_FILE_STATE`: Url to the full history state file, that contains the URL for the latest full history planet PBF file. e.g [`state.txt`](https://planet.openhistoricalmap.org.s3.amazonaws.com/planet/full-history/state.txt), This is no required in case you set the `URL_HISTORY_PLANET_FILE` env var
12+
13+
- `URL_PLANET_FILE`: URL for the latest planet PBF file.
14+
- `URL_HISTORY_PLANET_FILE`: URL for the latest full history planet PBF file.
15+
- `TIME_UPDATE_INTERVAL` Interval time to update the databases, e.g: `50m` = every 50 minutes, `20h` = every 20 hours , `5d` = every 5 days
16+
17+
The following env vars are required in the instance to update the values at: https://github.com/taginfo/taginfo/blob/master/taginfo-config-example.json
18+
19+
- `OVERWRITE_CONFIG_URL`: config file with the values to update
20+
21+
- `DOWNLOAD_DB`: Taginfo instances need 7 Sqlite databases to start up the web service, all of them can be downloaded from https://taginfo.openstreetmap.org/download. Or if you can download only some of them you can pass herec. e.g DOWNLOAD_DB=`languages wiki`, or DOWNLOAD_DB=`languages wiki projects chronology`.
22+
23+
- `CREATE_DB`: If you want process you of data using the PBF files, you can pass the values. eg. CREATE_DB=`db projects` or CREATE_DB=`db projects chronology`.
24+
Note:
25+
- Value `db` require to pass `URL_PLANET_FILE` or `URL_PLANET_FILE_STATE`
26+
- Value `projects` require to pass `TAGINFO_PROJECT_REPO`
27+
- Value `chronology` require to pass `URL_PLANET_FILE` or `URL_HISTORY_PLANET_FILE`

‎images/taginfo/overwrite_config.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
""" Script to overwrite values in set config
2+
python3 overwrite_config.py \
3+
-u https://gist.githubusercontent.com/Rub21/1a82fb3e4c0efd15524709a5e2d8ab89/raw/23c399802ba2a01cc30379875ac02a7b1b5ac8e1/taginfo.json\
4+
-f taginfo-config.json
5+
"""
6+
7+
import argparse
8+
import urllib.request
9+
import json
10+
11+
def main(config_file, overwrite_config_url):
12+
with urllib.request.urlopen(overwrite_config_url) as url:
13+
overwrite_values = json.loads(url.read())
14+
with open(config_file) as f:
15+
current_values = json.loads(f.read())
16+
# Hardcode for certain values
17+
if 'instance' in overwrite_values.keys():
18+
current_values['instance'] = overwrite_values['instance']
19+
if 'turbo' in overwrite_values.keys():
20+
current_values['turbo'] = overwrite_values['turbo']
21+
if 'sources' in overwrite_values.keys() and 'master' in overwrite_values['sources'].keys():
22+
current_values['sources']['master'] = overwrite_values['sources']['master']
23+
# Overwrite file
24+
with open(config_file, 'w') as f:
25+
f.write(json.dumps(current_values))
26+
27+
28+
parser = argparse.ArgumentParser(description='Set config values')
29+
parser.add_argument(
30+
'-u',
31+
type=str,
32+
help='URL of the config to overwrite',
33+
dest='overwrite_config_url')
34+
35+
parser.add_argument(
36+
'-f'
37+
'--config_file',
38+
type=str,
39+
help='Path of the config file',
40+
dest='config_file')
41+
42+
args = parser.parse_args()
43+
44+
if args.config_file and args.overwrite_config_url:
45+
main(args.config_file, args.overwrite_config_url)

‎images/taginfo/start.sh

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
#!/usr/bin/env bash
2+
3+
WORKDIR=/apps
4+
DATA_DIR=$WORKDIR/data
5+
UPDATE_DIR=$DATA_DIR/update
6+
DOWNLOAD_DIR=$DATA_DIR/download
7+
8+
set_taginfo_config() {
9+
echo "Setting up...$WORKDIR/taginfo-config.json"
10+
# Update dir values in taginfo-config.json
11+
grep -v '^ *//' $WORKDIR/taginfo/taginfo-config-example.json |
12+
jq '.logging.directory = "'$UPDATE_DIR'/log"' |
13+
jq '.paths.download_dir = "'$UPDATE_DIR'/download"' |
14+
jq '.paths.bin_dir = "'$WORKDIR'/taginfo-tools/build/src"' |
15+
jq '.sources.db.planetfile = "'$UPDATE_DIR'/planet/planet.osm.pbf"' |
16+
jq '.sources.chronology.osm_history_file = "'$UPDATE_DIR'/planet/history-planet.osh.pbf"' |
17+
jq '.sources.db.bindir = "'$UPDATE_DIR'/build/src"' |
18+
jq '.paths.data_dir = "'$DATA_DIR'"' \
19+
>$WORKDIR/taginfo-config.json
20+
21+
# languages wiki databases will be downloaded from OSM
22+
[[ ! -z $DOWNLOAD_DB+z} ]] && jq --arg a "${DOWNLOAD_DB}" '.sources.download = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
23+
24+
# Update instance values in taginfo-config.json
25+
python3 overwrite_config.py -u $OVERWRITE_CONFIG_URL -f $WORKDIR/taginfo-config.json
26+
27+
}
28+
29+
updates_create_db() {
30+
local CREATE_DB="$1"
31+
[[ ! -z $CREATE_DB+z} ]] && jq --arg a "${CREATE_DB}" '.sources.create = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
32+
}
33+
34+
updates_source_code() {
35+
echo "Update...Procesor source code"
36+
# Function to replace the projects repo to get the projects information
37+
TAGINFO_PROJECT_REPO=${TAGINFO_PROJECT_REPO//\//\\/}
38+
sed -i -e 's/https:\/\/github.com\/taginfo\/taginfo-projects.git/'$TAGINFO_PROJECT_REPO'/g' $WORKDIR/taginfo/sources/projects/update.sh
39+
# The follow line is requiered to avoid sqlite3 issues
40+
sed -i -e 's/run_ruby "$SRCDIR\/update_characters.rb"/ruby "$SRCDIR\/update_characters.rb"/g' $WORKDIR/taginfo/sources/db/update.sh
41+
sed -i -e 's/run_ruby "$SRCDIR\/import.rb"/ruby "$SRCDIR\/import.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
42+
sed -i -e 's/run_ruby "$SRCDIR\/parse.rb"/ruby "$SRCDIR\/parse.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
43+
sed -i -e 's/run_ruby "$SRCDIR\/get_icons.rb"/ruby "$SRCDIR\/get_icons.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
44+
}
45+
46+
download_planet_files() {
47+
mkdir -p $UPDATE_DIR/planet/
48+
# Check if URL_PLANET_FILE_STATE exist and set URL_PLANET_FILE
49+
if [[ ${URL_PLANET_FILE_STATE} && ${URL_PLANET_FILE_STATE-x} ]]; then
50+
wget -q -O state.planet.txt --no-check-certificate - $URL_PLANET_FILE_STATE
51+
URL_PLANET_FILE=$(cat state.planet.txt)
52+
fi
53+
# Check if URL_HISTORY_PLANET_FILE_STATE exist and set URL_HISTORY_PLANET_FILE
54+
if [[ ${URL_HISTORY_PLANET_FILE_STATE} && ${URL_HISTORY_PLANET_FILE_STATE-x} ]]; then
55+
wget -q -O state.history.txt --no-check-certificate - $URL_HISTORY_PLANET_FILE_STATE
56+
URL_HISTORY_PLANET_FILE=$(cat state.history.txt)
57+
fi
58+
# Download pbf files
59+
echo "Downloading...$URL_PLANET_FILE"
60+
wget -q -O $UPDATE_DIR/planet/planet.osm.pbf --no-check-certificate - $URL_PLANET_FILE
61+
echo "Downloading...$URL_HISTORY_PLANET_FILE"
62+
wget -q -O $UPDATE_DIR/planet/history-planet.osh.pbf --no-check-certificate - $URL_HISTORY_PLANET_FILE
63+
rm state.planet.txt
64+
rm state.history.txt
65+
}
66+
67+
update() {
68+
echo "Update...sqlite databases at $(date +%Y-%m-%d:%H-%M)"
69+
# Download OSM planet replication and full-history files
70+
download_planet_files
71+
# In order to make it work we need to pass first one by one the creation and then all of them "db projects chronology"
72+
for db in $CREATE_DB; do
73+
echo "Update...taginfo-$db.db"
74+
updates_create_db $db
75+
$WORKDIR/taginfo/sources/update_all.sh $UPDATE_DIR
76+
done
77+
echo "Update...$CREATE_DB"
78+
updates_create_db $CREATE_DB
79+
$WORKDIR/taginfo/sources/update_all.sh $UPDATE_DIR
80+
# Copy db files into data folder
81+
cp $UPDATE_DIR/*/taginfo-*.db $DATA_DIR/
82+
cp $UPDATE_DIR/taginfo-*.db $DATA_DIR/
83+
# Link to download db zip files
84+
chmod a=r $UPDATE_DIR/download
85+
ln -sf $UPDATE_DIR/download $WORKDIR/taginfo/web/public/download
86+
}
87+
88+
start_web() {
89+
echo "Start...Taginfo web service"
90+
cd $WORKDIR/taginfo/web && bundle exec rackup --host 0.0.0.0 -p 80
91+
}
92+
93+
continuous_update() {
94+
while true; do
95+
update
96+
sleep $TIME_UPDATE_INTERVAL
97+
done
98+
}
99+
100+
main() {
101+
set_taginfo_config
102+
updates_source_code
103+
# Check if db files are store in the $DATA_DIR in order to start the service or start procesing the file
104+
NUM_DB_FILES=$(ls $DATA_DIR/*.db | wc -l)
105+
if [ $NUM_DB_FILES -lt 7 ]; then
106+
update
107+
fi
108+
start_web &
109+
continuous_update
110+
}
111+
main

0 commit comments

Comments
 (0)