Skip to content

Commit 8cf0764

Browse files
author
Rub21
committed
Update taginfo databases processing
1 parent b185bea commit 8cf0764

File tree

1 file changed

+59
-52
lines changed

1 file changed

+59
-52
lines changed

‎images/taginfo/start.sh

Lines changed: 59 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -5,64 +5,70 @@ DATA_DIR=$WORKDIR/data
55
UPDATE_DIR=$DATA_DIR/update
66
DOWNLOAD_DIR=$DATA_DIR/download
77

8-
# Update dir values in taginfo-config.json
9-
grep -v '^ *//' $WORKDIR/taginfo/taginfo-config-example.json |
10-
jq '.logging.directory = "'$UPDATE_DIR'/log"' |
11-
jq '.paths.download_dir = "'$UPDATE_DIR'/download"' |
12-
jq '.paths.bin_dir = "'$WORKDIR'/taginfo-tools/build/src"' |
13-
jq '.sources.db.planetfile = "'$UPDATE_DIR'/planet/planet.osm.pbf"' |
14-
jq '.sources.chronology.osm_history_file = "'$UPDATE_DIR'/planet/history-planet.osh.pbf"' |
15-
jq '.sources.db.bindir = "'$UPDATE_DIR'/build/src"' |
16-
jq '.paths.data_dir = "'$DATA_DIR'"' \
17-
>$WORKDIR/taginfo-config.json
18-
19-
# Update instance values in taginfo-config.json
20-
[[ ! -z ${INSTANCE_URL+z} ]] && jq --arg a "${INSTANCE_URL}" '.instance.url = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
21-
[[ ! -z $INSTANCE_NAME+z} ]] && jq --arg a "${INSTANCE_NAME}" '.instance.name = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
22-
[[ ! -z $INSTANCE_DESCRIPTION+z} ]] && jq --arg a "${INSTANCE_DESCRIPTION}" '.instance.description = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
23-
[[ ! -z $INSTANCE_ICON+z} ]] && jq --arg a "${INSTANCE_ICON}" '.instance.icon = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
24-
[[ ! -z $INSTANCE_CONTACT+z} ]] && jq --arg a "${INSTANCE_CONTACT}" '.instance.contact = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
25-
26-
27-
# Update DBs to create
28-
[[ ! -z $DOWNLOAD_DB+z} ]] && jq --arg a "${DOWNLOAD_DB}" '.sources.download = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
29-
[[ ! -z $CREATE_DB+z} ]] && jq --arg a "${CREATE_DB}" '.sources.create = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
30-
31-
# Function to replace the repo to get the projects information
32-
TAGINFO_PROJECT_REPO=${TAGINFO_PROJECT_REPO//\//\\/}
33-
sed -i -e 's/https:\/\/github.com\/taginfo\/taginfo-projects.git/'$TAGINFO_PROJECT_REPO'/g' $WORKDIR/taginfo/sources/projects/update.sh
34-
35-
# The follow line is requiered to avoid an issue -> require cannot load such file -- sqlite3
36-
sed -i -e 's/run_ruby "$SRCDIR\/update_characters.rb"/ruby "$SRCDIR\/update_characters.rb"/g' $WORKDIR/taginfo/sources/db/update.sh
37-
sed -i -e 's/run_ruby "$SRCDIR\/import.rb"/ruby "$SRCDIR\/import.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
38-
sed -i -e 's/run_ruby "$SRCDIR\/parse.rb"/ruby "$SRCDIR\/parse.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
39-
sed -i -e 's/run_ruby "$SRCDIR\/get_icons.rb"/ruby "$SRCDIR\/get_icons.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
8+
set_taginfo_config() {
9+
echo "Setting up $WORKDIR/taginfo-config.json file ..."
10+
11+
# Update dir values in taginfo-config.json
12+
grep -v '^ *//' $WORKDIR/taginfo/taginfo-config-example.json |
13+
jq '.logging.directory = "'$UPDATE_DIR'/log"' |
14+
jq '.paths.download_dir = "'$UPDATE_DIR'/download"' |
15+
jq '.paths.bin_dir = "'$WORKDIR'/taginfo-tools/build/src"' |
16+
jq '.sources.db.planetfile = "'$UPDATE_DIR'/planet/planet.osm.pbf"' |
17+
jq '.sources.chronology.osm_history_file = "'$UPDATE_DIR'/planet/history-planet.osh.pbf"' |
18+
jq '.sources.db.bindir = "'$UPDATE_DIR'/build/src"' |
19+
jq '.paths.data_dir = "'$DATA_DIR'"' \
20+
>$WORKDIR/taginfo-config.json
21+
22+
# Update instance values in taginfo-config.json
23+
[[ ! -z ${INSTANCE_URL+z} ]] && jq --arg a "${INSTANCE_URL}" '.instance.url = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
24+
[[ ! -z $INSTANCE_NAME+z} ]] && jq --arg a "${INSTANCE_NAME}" '.instance.name = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
25+
[[ ! -z $INSTANCE_DESCRIPTION+z} ]] && jq --arg a "${INSTANCE_DESCRIPTION}" '.instance.description = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
26+
[[ ! -z $INSTANCE_ICON+z} ]] && jq --arg a "${INSTANCE_ICON}" '.instance.icon = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
27+
[[ ! -z $INSTANCE_CONTACT+z} ]] && jq --arg a "${INSTANCE_CONTACT}" '.instance.contact = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
28+
29+
# languages wiki databases will be downloaded from OSM
30+
jq --arg a "languages wiki" '.sources.download = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
31+
}
32+
33+
updates_create_db() {
34+
local CREATE_DB="$1"
35+
[[ ! -z $CREATE_DB+z} ]] && jq --arg a "${CREATE_DB}" '.sources.create = $a' $WORKDIR/taginfo-config.json >tmp.json && mv tmp.json $WORKDIR/taginfo-config.json
36+
}
37+
38+
updates_source_code() {
39+
echo "Updating source code, in order to make the code running..."
40+
41+
# Function to replace the projects repo to get the projects information
42+
TAGINFO_PROJECT_REPO=${TAGINFO_PROJECT_REPO//\//\\/}
43+
sed -i -e 's/https:\/\/github.com\/taginfo\/taginfo-projects.git/'$TAGINFO_PROJECT_REPO'/g' $WORKDIR/taginfo/sources/projects/update.sh
44+
45+
# The follow line is requiered to avoid sqlite3 issues
46+
sed -i -e 's/run_ruby "$SRCDIR\/update_characters.rb"/ruby "$SRCDIR\/update_characters.rb"/g' $WORKDIR/taginfo/sources/db/update.sh
47+
sed -i -e 's/run_ruby "$SRCDIR\/import.rb"/ruby "$SRCDIR\/import.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
48+
sed -i -e 's/run_ruby "$SRCDIR\/parse.rb"/ruby "$SRCDIR\/parse.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
49+
sed -i -e 's/run_ruby "$SRCDIR\/get_icons.rb"/ruby "$SRCDIR\/get_icons.rb"/g' $WORKDIR/taginfo/sources/projects/update.sh
50+
}
4051

4152
update() {
42-
echo "Download and update pbf files at $(date +%Y-%m-%d:%H-%M)"
53+
echo "Download and update pbf files at $(date +%Y-%m-%d:%H-%M)..."
4354

44-
# Download OSM planet replication and full-history files
55+
# Download OSM planet replication and full-history files,
56+
# TODO: Download latest files,comparing with something like md5 decryption
4557
mkdir -p $UPDATE_DIR/planet/
4658
[ ! -f $UPDATE_DIR/planet/planet.osm.pbf ] && wget --no-check-certificate -O $UPDATE_DIR/planet/planet.osm.pbf $URL_PLANET_FILE
4759
[ ! -f $UPDATE_DIR/planet/history-planet.osh.pbf ] && wget --no-check-certificate -O $UPDATE_DIR/planet/history-planet.osh.pbf $URL_HISTORY_PLANET_FILE
4860

49-
# Update pbf files ussing replication files, TODO, fix the certification issue
50-
# pyosmium-up-to-date \
51-
# -v \
52-
# --size 5000 \
53-
# --server $REPLICATION_SERVER \
54-
# $UPDATE_DIR/planet/history-planet.osh.pbf
55-
56-
# pyosmium-up-to-date \
57-
# -v \
58-
# --size 5000 \
59-
# --server $REPLICATION_SERVER \
60-
# $UPDATE_DIR/planet/planet.osm.pbf
61+
# Update local DB,twice, with different parameters,
62+
# in order to make it work we need to pass first "db projects" and then "db projects chronology"
63+
updates_create_db "db projects"
64+
$WORKDIR/taginfo/sources/update_all.sh $UPDATE_DIR
6165

62-
# Update local DB
66+
updates_create_db "db projects chronology"
6367
$WORKDIR/taginfo/sources/update_all.sh $UPDATE_DIR
64-
mv $UPDATE_DIR/*/taginfo-*.db $DATA_DIR/
65-
mv $UPDATE_DIR/taginfo-*.db $DATA_DIR/
68+
69+
# Copy db files into data folder
70+
cp $UPDATE_DIR/*/taginfo-*.db $DATA_DIR/
71+
cp $UPDATE_DIR/taginfo-*.db $DATA_DIR/
6672

6773
# Link to download db zip files
6874
chmod a=r $UPDATE_DIR/download
@@ -82,11 +88,12 @@ continuous_update() {
8288
}
8389

8490
main() {
85-
# Check if db files are store in the $DATA_DIR
91+
set_taginfo_config
92+
updates_source_code
93+
94+
# Check if db files are store in the $DATA_DIR in order to start the service or start procesing the file
8695
NUM_DB_FILES=$(ls $DATA_DIR/*.db | wc -l)
8796
if [ $NUM_DB_FILES -lt 7 ]; then
88-
# We are changing the default values, we need to run update twice at the beginning
89-
update
9097
update
9198
fi
9299
start_web &

0 commit comments

Comments
 (0)