Skip to content

Commit f1355c0

Browse files
Rub21Rub21
committed
Merge from develop
2 parents c024542 + 438aa37 commit f1355c0

29 files changed

+943
-122
lines changed

‎.dockerignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ id-editor/
1818
tiler-visor/
1919
package.json
2020
db-backup-restore/
21-
21+
nominatim-data/
22+
postgresdata/
2223
# ignore all *.class files in all folders, including build root
2324
**/*.json
2425

‎.env-nominatim.example

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
PG_HOST=nominatim-db
2+
PG_PORT=5432
3+
PG_USER=nominatim
4+
PG_PASSWORD=password1234
5+
PG_DATABASE=nominatim
6+
OSM_URL_FILE=http://download.geofabrik.de/europe/monaco-latest.osm.bz2
7+
THREADS=4
8+
REPLICATION_URL=http://download.geofabrik.de/europe/monaco-updates
9+
REPLICATION_MAXINTERVAL=86500
10+
REPLICATION_UPDATE_INTERVAL=86500
11+
REPLICATION_RECHECK_INTERVAL=900

‎.env-tiler.example

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ POSTGRES_PORT=5432
2424
# tiler-imposm evn variables
2525
# ====================================================================================================
2626
TILER_IMPORT_FROM=osm
27-
TILER_IMPORT_PBF_URL=https://osmseed.s3.amazonaws.com/planet/planet-latest.osm.pbf
27+
TILER_IMPORT_PBF_URL=http://glunimore.geofabrik.de/europe/monaco-latest.osm.pbf
2828
#TILER_IMPORT_LIMIT='geojson url'
2929
SEQUENCE_NUMBER=0
30-
REPLICATION_URL=https://osmseed.s3.amazonaws.com/replication/minute/
30+
REPLICATION_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/
3131

3232
# ====================================================================================================
3333
# tiler-serve env variables,

‎.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
.env
22
.env-tiler
3+
.env-nominatim
4+
.env-tasking-manager
5+
.DS_Store
36
usa.values.yaml
47
prod.values.yaml
58
planet.values.yaml
@@ -13,6 +16,7 @@ osm-processor-data/
1316
postgres-gis-data/
1417
tiler-imposm-data/
1518
tiler-server-data/
19+
nominatim-pgdata/
1620
data/
1721
*.osm
1822
*.pbf

‎chartpress.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,4 +30,5 @@ charts:
3030
valuesPath: tmApi.image
3131
# tiler-visor:
3232
# valuesPath: tilerVisor.image
33-
33+
nominatim:
34+
valuesPath: nominatim.image

‎docker-compose.yml

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -113,30 +113,30 @@ services:
113113
./start.sh"
114114
env_file:
115115
- ./.env
116-
# # #=========================================================================
117-
# # OMS-SEED-TILER section
118-
# #=========================================================================
116+
#=========================================================================
117+
# OMS-SEED-TILER section
118+
#=========================================================================
119119
tiler-db:
120120
image: osmseed-tiler-db:v1
121-
build:
121+
build:
122122
context: ./images/tiler-db
123123
dockerfile: Dockerfile
124124
ports:
125-
- '5433:5432'
125+
- "5433:5432"
126126
volumes:
127127
- ./postgres-gis-data:/var/lib/postgresql/data
128128
env_file:
129129
- ./.env-tiler
130130
tiler-imposm:
131131
image: osmseed-tiler-imposm:v1
132-
build:
132+
build:
133133
context: ./images/tiler-imposm
134134
dockerfile: Dockerfile
135135
volumes:
136136
- ./tiler-imposm-data:/mnt/data
137137
depends_on:
138138
- tiler-db
139-
env_file:
139+
env_file:
140140
- ./.env-tiler
141141
command: >
142142
/bin/bash -c "
@@ -146,7 +146,7 @@ services:
146146
./start.sh"
147147
tiler-server:
148148
image: osmseed-tiler-server:v1
149-
build:
149+
build:
150150
context: ./images/tiler-server
151151
dockerfile: Dockerfile
152152
volumes:
@@ -155,12 +155,10 @@ services:
155155
- tiler-db
156156
- tiler-imposm
157157
ports:
158-
- '9090:9090'
158+
- "9090:9090"
159159
command: >
160160
/bin/bash -c "
161161
echo Sleep for a while!;
162162
sleep 100;
163163
echo Starting tiles server!;
164-
./start.sh"
165-
env_file:
166-
- ./.env-tiler
164+
./start.sh"

‎images/nominatim/Dockerfile

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
FROM ubuntu:focal AS build
2+
3+
ENV DEBIAN_FRONTEND noninteractive
4+
ENV LANG C.UTF-8
5+
6+
WORKDIR /app
7+
8+
RUN true \
9+
# Do not start daemons after installation.
10+
&& echo '#!/bin/sh\nexit 101' > /usr/sbin/policy-rc.d \
11+
&& chmod +x /usr/sbin/policy-rc.d \
12+
# Install all required packages.
13+
&& apt-get -y update -qq \
14+
&& apt-get -y install \
15+
locales \
16+
&& locale-gen en_US.UTF-8 \
17+
&& update-locale LANG=en_US.UTF-8 \
18+
&& apt-get -y install \
19+
-o APT::Install-Recommends="false" \
20+
-o APT::Install-Suggests="false" \
21+
# Build tools from sources.
22+
build-essential \
23+
g++ \
24+
cmake \
25+
libpq-dev \
26+
zlib1g-dev \
27+
libbz2-dev \
28+
libproj-dev \
29+
libexpat1-dev \
30+
libboost-dev \
31+
libboost-system-dev \
32+
libboost-filesystem-dev \
33+
# PostgreSQL.
34+
postgresql-contrib \
35+
postgresql-server-dev-12 \
36+
postgresql-12-postgis-3 \
37+
postgresql-12-postgis-3-scripts \
38+
# PHP and Apache 2.
39+
php \
40+
php-intl \
41+
php-pgsql \
42+
apache2 \
43+
libapache2-mod-php \
44+
# Python 3.
45+
python3-dev \
46+
python3-pip \
47+
python3-tidylib \
48+
python3-psycopg2 \
49+
python3-setuptools \
50+
# Misc.
51+
git \
52+
curl \
53+
wget \
54+
sudo
55+
56+
# Configure postgres.
57+
RUN true \
58+
&& echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/12/main/pg_hba.conf \
59+
&& echo "listen_addresses='*'" >> /etc/postgresql/12/main/postgresql.conf
60+
61+
# Osmium install to run continuous updates.
62+
RUN pip3 install osmium
63+
64+
# Nominatim install.
65+
ENV NOMINATIM_VERSION v3.6.0
66+
67+
RUN true \
68+
&& git clone \
69+
--config advice.detachedHead=false \
70+
--single-branch \
71+
--branch $NOMINATIM_VERSION \
72+
--depth 1 \
73+
--recursive \
74+
https://github.com/openstreetmap/Nominatim.git \
75+
src \
76+
&& cd src \
77+
&& mkdir build \
78+
&& cd build \
79+
&& cmake .. \
80+
&& make -j`nproc` \
81+
&& ./utils/setup.php --setup-website \
82+
&& chmod o=rwx .
83+
84+
# Apache configure.
85+
# COPY local.php /app/src/build/settings/local.php
86+
COPY nominatim.conf /etc/apache2/sites-enabled/000-default.conf
87+
88+
# Load initial data.
89+
ARG with_postcodes_gb
90+
ARG with_postcodes_us
91+
92+
RUN if [ "$with_postcodes_gb" = "" ]; then \
93+
echo "Skipping optional GB postcode file"; \
94+
else \
95+
echo "Downloading optional GB postcode file"; \
96+
curl http://www.nominatim.org/data/gb_postcode_data.sql.gz > /app/src/data/gb_postcode_data.sql.gz; \
97+
fi;
98+
99+
RUN if [ "$with_postcodes_us" = "" ]; then \
100+
echo "Skipping optional US postcode file"; \
101+
else \
102+
echo "Downloading optional US postcode file"; \
103+
curl http://www.nominatim.org/data/us_postcode_data.sql.gz > /app/src/data/us_postcode_data.sql.gz; \
104+
fi;
105+
106+
RUN curl http://www.nominatim.org/data/country_grid.sql.gz > /app/src/data/country_osm_grid.sql.gz
107+
108+
RUN true \
109+
# Remove development and unused packages.
110+
&& apt-get -y remove --purge \
111+
cpp-9 \
112+
gcc-9* \
113+
g++ \
114+
git \
115+
make \
116+
cmake* \
117+
llvm-10* \
118+
libc6-dev \
119+
linux-libc-dev \
120+
libclang-*-dev \
121+
build-essential \
122+
postgresql-server-dev-12 \
123+
&& apt-get clean \
124+
# Clear temporary files and directories.
125+
&& rm -rf \
126+
/tmp/* \
127+
/var/tmp/* \
128+
/root/.cache \
129+
/app/src/.git \
130+
/var/lib/apt/lists/* \
131+
/var/lib/postgresql/12/main/*
132+
133+
# Postgres PGDATA as volume
134+
ENV PATH $PATH:/usr/lib/postgresql/12/bin
135+
ENV PGDATA /var/lib/postgresql/data
136+
RUN mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA"
137+
VOLUME /var/lib/postgresql/data
138+
139+
COPY local_api.php /app/src/build/settings/local_api.php
140+
COPY init.sh /app/init.sh
141+
COPY startapache.sh /app/startapache.sh
142+
COPY startpostgres.sh /app/startpostgres.sh
143+
COPY update.sh /app/update.sh
144+
COPY start.sh /app/start.sh
145+
146+
# Collapse image to single layer.
147+
FROM scratch
148+
149+
COPY --from=build / /
150+
151+
WORKDIR /app
152+
153+
EXPOSE 5432
154+
EXPOSE 8080

‎images/nominatim/README.md

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# Nominatim Docker (Nominatim version 3.6)
2+
3+
0. Copy .env-nominatim.example to .env-nominatim
4+
5+
1. Build
6+
7+
```
8+
cd images/nominatim
9+
docker build --pull --rm -t nominatim .
10+
```
11+
See below for optional build arguments to include postcode data in your image.
12+
13+
2. Copy <your_country>.osm.pbf to a local directory (i.e. /home/me/nominatimdata)
14+
15+
3. Initialize Nominatim Database
16+
17+
```
18+
docker run -t -v /home/me/nominatimdata:/data nominatim sh /app/init.sh /data/<your_country>.osm.pbf postgresdata 4
19+
```
20+
Where 4 is the number of threads to use during import. In general the import of data in postgres is a very time consuming
21+
process that may take hours or days. If you run this process on a multiprocessor system make sure that it makes the best use
22+
of it. You can delete the /home/me/nominatimdata/<your_country>.osm.pbf once the import is finished.
23+
24+
5. Start database
25+
26+
```
27+
docker run --name nominatim-db --restart=always -p 6432:5432 -d -v /home/me/nominatimdata/postgresdata:/var/lib/postgresql/12/main nominatim sh /app/startpostgres.sh
28+
```
29+
30+
6. Start API server
31+
```
32+
docker run --name web --restart=always -p 7070:8080 -d -v /home/me/nominatimdata/:/data nominatim sh /app/startapache.sh
33+
```
34+
35+
7. Configure incremental update. By default CONST_Replication_Url configured for Monaco.
36+
If you want a different update source, you will need to declare `CONST_Replication_Url` in local.php. Documentation [here] (https://github.com/openstreetmap/Nominatim/blob/master/docs/admin/Import-and-Update.md#updates). For example, to use the daily country extracts diffs for Gemany from geofabrik add the following:
37+
```
38+
@define('CONST_Replication_Url', 'http://download.geofabrik.de/europe/germany-updates');
39+
```
40+
41+
Now you will have a fully functioning nominatim instance available at : [http://localhost:7070/](http://localhost:7070). Unlike the previous versions
42+
this one does not store data in the docker context and this results to a much slimmer docker image.
43+
44+
# Postcodes
45+
46+
Nominatim requires additional data files to accurately assign postcodes data in the US and Great Britain (Northern Ireland postcodes are not included in this file) as described in [these Nominatim docs](https://nominatim.org/release-docs/latest/admin/Import-and-Update/#downloading-additional-data). Without this data, you may get incorrect postcodes for some address lookups.
47+
48+
These data files aren't downloaded by default, but you can add them with additional arguments at the build stage. To include the US postcode data file, add "--build-arg with_postcodes_us=1" to the command line in stage 1, above. To include GB postcodes, run with "--build-arg with_postcodes_gb=1". You can run with both at once if desired, eg:
49+
```
50+
docker build --pull --rm -t nominatim --build-arg with_postcodes_us=1 --build-arg with_postcodes_gb=1 .
51+
```
52+
53+
# Update
54+
55+
Full documentation for Nominatim update available [here](https://github.com/osm-search/Nominatim/blob/master/docs/admin/Update.md). For a list of other methods see the output of:
56+
```
57+
docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --help
58+
```
59+
60+
To initialise the updates run
61+
```
62+
docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --init-updates
63+
```
64+
65+
The following command will keep your database constantly up to date:
66+
```
67+
docker exec -it nominatim sudo -u postgres ./src/build/utils/update.php --import-osmosis-all
68+
```
69+
If you have imported multiple country extracts and want to keep them
70+
up-to-date, have a look at the script in
71+
[issue #60](https://github.com/openstreetmap/Nominatim/issues/60).
72+
73+
# Upgrade Guide for 3.6.x
74+
75+
## Upgrade from 3.5.0 to 3.6.0
76+
77+
As referenced in the Nominatim release ([https://github.com/osm-search/Nominatim/releases/tag/v3.5.2](https://github.com/osm-search/Nominatim/releases/tag/v3.6.0)) the HTML frontend was removed from the project and moved to a separate project ([https://github.com/osm-search/nominatim-ui](https://github.com/osm-search/nominatim-ui)) if you need more than the API.
78+
79+
In addition there is an extensive migration path to upgrade from 3.5 to 3.6 (see: [https://nominatim.org/release-docs/latest/admin/Migration/#350-360](https://nominatim.org/release-docs/latest/admin/Migration/#350-360)), so you should consider a full reimport of your data.
80+
81+
# Docker image upgrade to 3.6 from <= 3.4
82+
83+
With 3.5 we have switched to Ubuntu 20.04 (LTS) which uses PostgreSQL 12. If you want to reuse your old data dictionary without importing the data again you have to make sure to migrate the data from PostgreSQL 11 to 12 with a command like ```pg_upgrade``` (see: [https://www.postgresql.org/docs/current/pgupgrade.html](https://www.postgresql.org/docs/current/pgupgrade.html)).
84+
85+
You can try a script like [https://github.com/tianon/docker-postgres-upgrade](https://github.com/tianon/docker-postgres-upgrade) with some modifications.

‎images/nominatim/init.sh

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
#!/bin/bash
2+
3+
FILENAME=$(basename -- "$OSM_URL_FILE")
4+
EXTENSION="${FILENAME##*.}"
5+
OSMFILE=osmfile.osm.$EXTENSION
6+
PGDIR=$PGDATA
7+
# sudo rm -rf $PGDATA/*
8+
mkdir -p $PGDIR
9+
10+
if [ -f "$PGDIR/postgresql.conf" ]; then
11+
echo "$PGDIR/postgresql.conf exist, Import won't happen, if you want start an empty dataset remove the directory $PGDIR"
12+
exit 0
13+
else
14+
chown postgres:postgres $PGDIR && \
15+
export PGDATA=$PGDIR && \
16+
sudo -u postgres /usr/lib/postgresql/12/bin/initdb -D $PGDIR && \
17+
sudo -u postgres /usr/lib/postgresql/12/bin/pg_ctl -D $PGDIR start && \
18+
sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='$PG_USER'" | grep -q 1 || sudo -u postgres createuser -s $PG_USER && \
19+
sudo -u postgres psql postgres -tAc "SELECT 1 FROM pg_roles WHERE rolname='www-data'" | grep -q 1 || sudo -u postgres createuser -SDR www-data && \
20+
sudo -u postgres psql postgres -c "DROP DATABASE IF EXISTS nominatim" && \
21+
useradd -m -p $PG_PASSWORD $PG_USER && \
22+
echo "Download OSM file ... $OSM_URL_FILE" && \
23+
chown -R $PG_USER:$PG_USER ./src && \
24+
wget $OSM_URL_FILE -O $OSMFILE --no-check-certificate && \
25+
sudo -u $PG_USER ./src/build/utils/setup.php --osm-file $OSMFILE --all --threads $THREADS && \
26+
sudo -u $PG_USER ./src/build/utils/check_import_finished.php && \
27+
sudo -u postgres /usr/lib/postgresql/12/bin/pg_ctl -D $PGDIR stop && \
28+
sudo chown -R postgres:postgres $PGDIR && \
29+
sudo chmod 0700 -R $PGDATA
30+
echo "host all all 0.0.0.0/0 trust" >> $PGDATA/pg_hba.conf
31+
echo "listen_addresses='*'" >> $PGDATA/postgresql.conf
32+
fi

0 commit comments

Comments
 (0)