diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..f489616d2 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +FROM nginx:stable + +RUN addgroup overpass && adduser --home /db --disabled-password --gecos overpass --ingroup overpass overpass + +RUN apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y \ + autoconf \ + automake \ + expat \ + libexpat1-dev \ + g++ \ + liblz4-1 \ + liblz4-dev \ + libtool \ + m4 \ + make \ + zlib1g \ + zlib1g-dev + +COPY . /app/ + +RUN cd /app/src \ + && autoscan \ + && aclocal \ + && autoheader \ + && libtoolize \ + && automake --add-missing \ + && autoconf \ + && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \ + && make -j $(grep -c ^processor /proc/cpuinfo) install clean \ + && apt-get remove -y \ + autoconf \ + automake \ + libexpat1-dev \ + g++ \ + libtool \ + liblz4-dev \ + m4 \ + make \ + zlib1g-dev \ + && apt-get autoremove -y + +RUN apt-get install --no-install-recommends --no-install-suggests -y \ + supervisor \ + bash \ + lftp \ + wget \ + fcgiwrap \ + bzip2 + + +RUN mkdir /nginx && chown nginx:nginx /nginx && mkdir -p /db/db /db/diffs && chown -R overpass:overpass /db +COPY src/rules /db/db/rules +COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf +VOLUME /db + +EXPOSE 80 +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] +CMD ["/app/docker-entrypoint.sh"] diff --git a/README_Docker.md b/README_Docker.md new file mode 100644 index 000000000..bdc6ae330 --- /dev/null +++ b/README_Docker.md @@ -0,0 +1,66 @@ +# How to use this image + +This image on startup initializes with data on first start. +To do it properly it needs a bit of guidance in form of enviroment variables. +* `OVERPASS_MODE` - takes the value of either `init` or `clone` +* `OVERPASS_META` - `yes`, `no` or `attic` - passed to Overpass as `--meta` or `--keep-attic` +* `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/) +* `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode +* `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied +* `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. +Ony useful in `init` mode. Defaults to `gz` +* `OVERPASS_RULES_LOAD` - desired load generated by areas generation. Controls how long the script will sleep before regenerating +areas. Set it to 1, and script will sleep 99x times longer than it works, set it to 50 - and script will sleep as long as last +execution, set it to 100, and script will sleep 3 seconds between each execution. + + +Image works in two modes `clone` or `init`. This affects how the instance gets initialized. If the mode is set to `clone` +then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated from diffs. This will result in Overpass instance +covering whole world. This mode works only with minute diffs. + +In `init` mode you need to point `OVERPASS_PLANET_URL` to address with planet (partial) dump. This file will be downloaded, +indexed by Overpass and later - updated using `OVERPASS_PLANET_SEQUENCE_ID` and `OVERPASS_DIFF_URL`. You need to check which +sequence number is for your planet file. Take it from desctiption or just take a sequence number a day before your planet +file is dated. + +Start initalization mode with `-i` and `-t` options to `docker run` so you will have a chance to monitor the progress of +file downloads etc. + +After initialization is finished Docker container will stop. Once you start it again (with `docker start` command) it will start +downloading minute diffs, applying them to database and serving API requests. + +Container exposes port 80. + +All data resides within /db directory in container. + +# Examples +## Overpass instance covering part of the world +In this example Overpass instance will be initialized with planet file for Poland downloaded from Geofabrik. Data will be stored in folder +`/big/docker/overpass_db/` on the host machine. Overpass will be available on port 12345 on host machine. +``` +docker run \ + -e OVERPASS_META=yes \ + -e OVERPASS_MODE=init \ + -e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/poland-latest.osm.bz2 \ + -e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/poland/minute/ \ + -e OVERPASS_PLANET_SEQUENCE_ID=2346486 \ + -v /big/docker/overpass_db/:/db \ + -p 12345:80 \ + -i -t \ + --name overpass_poland overpass +``` + +## Overpass clone covering whole world +In this example Overpass instance will be initialized with data from main Overpass instance and updated with master planet diffs. +Data will be stored in /big/docker/overpass_clone_db/ directory on the host machine and API will be exposed on port 12346 on host machine. +``` +docker run \ + -e OVERPASS_META=yes + -e OVERPASS_MODE=clone + -e OVERPASS_DIFF_URL=https://planet.openstreetmap.org/replication/minute/ + -v /big/docker/overpass_clone_db/:/db + -p 12346:80 + -i -t + --name overpass_world + overpass +``` diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 000000000..2165974aa --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -eo pipefail +shopt -s nullglob +OVERPASS_META=${OVERPASS_META:-no} +OVERPASS_MODE=${OVERPASS_MODE:-clone} +OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz} + +if [ ! -d /db/db ] ; then + if [ "$OVERPASS_MODE" = "clone" ]; then + if [[ "$OVERPASS_META" == "attic" ]] ; then + META="--keep-attic" + else + META="--meta=$OVERPASS_META" + fi + mkdir -p /db/db \ + && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META \ + && cp -r /app/src/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi + + if [ "$OVERPASS_MODE" = "init" ]; then + lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \ + && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" "--compression-method=$OVERPASS_COMPRESSION" \ + && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \ + && rm /db/planet \ + && cp -r /app/src/rules /db/db \ + && chown -R overpass:overpass /db \ + && echo "Overpass ready, you can start your container with docker start" + exit + fi +fi + +exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf diff --git a/etc/nginx-overpass.conf b/etc/nginx-overpass.conf new file mode 100644 index 000000000..c01961579 --- /dev/null +++ b/etc/nginx-overpass.conf @@ -0,0 +1,66 @@ +daemon off; + +user nginx; +worker_processes 4; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + server { + listen 80; + server_name localhost; + + #charset koi8-r; + #access_log /var/log/nginx/log/host.access.log main; + + location / { + root /usr/share/nginx/html; + index index.html index.htm; + } + + #error_page 404 /404.html; + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } + + location /cgi-bin/ { + include fastcgi_params; + fastcgi_param SCRIPT_FILENAME /app/$fastcgi_script_name; + fastcgi_param PATH_INFO $fastcgi_path_info; + fastcgi_pass unix:/nginx/fcgiwrap.socket; + } + + location /api/ { + rewrite ^/api/(.+)$ /cgi-bin/$1 last; + } + } + + +} diff --git a/etc/supervisord.conf b/etc/supervisord.conf new file mode 100644 index 000000000..1f0af3644 --- /dev/null +++ b/etc/supervisord.conf @@ -0,0 +1,42 @@ +[supervisord] +nodaemon=true + +[program:overpass_dispatch] +command=/bin/bash -c "find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base --meta --db-dir=/db/db" +user=overpass +redirect_stderr=true +priority=1 + +[program:nginx] +command=nginx +priority=2 + +[program:fcgiwrap] +command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket" +redirect_stderr=true +user=nginx +priority=3 + +[program:fetch_diff] +command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs +user=overpass +redirect_stderr=true +priority=5 + +[program:apply_diff] +command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s +redirect_stderr=true +user=overpass +priority=4 + +[program:dispatcher_areas] +command=nice /app/bin/dispatcher --areas --db-dir="/db/db" +user=overpass +redirect_stderr=true +priority=6 + +[program:areas_rules] +command=nice /app/bin/rules_loop.sh /db/db %(ENV_OVERPASS_RULES_LOAD)s +user=overpass +redirect_stderr=true +priority=7