Docker image for Overpass

Dockerfile

@@ -0,0 +1,60 @@
+FROM nginx:stable
+
+RUN addgroup overpass && adduser --home /db --disabled-password --ingroup overpass overpass
+
+RUN apt-get update \
+    && apt-get install --no-install-recommends --no-install-suggests -y \
+        autoconf \
+        automake \
+        expat \
+        libexpat1-dev \
+        g++ \
+        liblz4-1 \
+        liblz4-dev \
+        libtool \
+        m4 \
+        make \
+        zlib1g \
+        zlib1g-dev
+
+COPY . /app/
+
+RUN cd /app/src \
+    && autoscan \
+    && aclocal \
+    && autoheader \
+    && libtoolize \
+    && automake --add-missing  \
+    && autoconf \
+    && CXXFLAGS='-O2' CFLAGS='-O2' ./configure --prefix=/app --enable-lz4 \
+    && make -j $(grep -c ^processor /proc/cpuinfo) install clean \
+    && apt-get remove -y \
+        autoconf \
+        automake \
+        libexpat1-dev \
+        g++ \
+        libtool \
+        liblz4-dev \
+        m4 \
+        make \
+        zlib1g-dev \
+    && apt-get autoremove -y 
+
+RUN apt-get install --no-install-recommends --no-install-suggests -y \
+        supervisor \
+        bash \
+        lftp \
+        wget \
+        fcgiwrap \
+        bzip2
+
+
+RUN mkdir /nginx && chown nginx:nginx /nginx && mkdir -p /db/db /db/diffs && chown -R overpass:overpass /db
+COPY src/rules /db/db/rules
+COPY etc/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY etc/nginx-overpass.conf /etc/nginx/nginx.conf
+VOLUME /db
+
+EXPOSE 80
+# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
+CMD ["/app/docker-entrypoint.sh"]

README_Docker.md

@@ -0,0 +1,62 @@
+# How to use this image
+
+This image on startup initializes with data on first start.
+To do it properly it needs a bit of guidance in form of enviroment variables.
+* `OVERPASS_MODE` - takes the value of either `init` or `clone`
+* `OVERPASS_META` - `yes`, `no` or `attic` - passed to Overpass as `--meta` or `--keep-attic`
+* `OVERPASS_DIFF_URL` - url to diff's directory for updating the instance (eg. https://planet.openstreetmap.org/replication/minute/)
+* `OVERPASS_PLANET_URL` - url to "planet" file in `init` mode
+* `OVERPASS_PLANET_SEQUENCE_ID` - sequence identifier corresponding to planet file above. All files after this one will be applied
+* `OVERPASS_COMPRESSION` - takes values of `no`, `gz` or `lz4`. Specifies compression mode of the Overpass database. 
+Ony useful in `init` mode. Defaults to `gz`
+
+Image works in two modes `clone` or `init`. This affects how the instance gets initialized. If the mode is set to `clone`
+then data is copied from http://dev.overpass-api.de/api_drolbr/ and then updated from diffs. This will result in Overpass instance
+covering whole world. This mode works only with minute diffs.
+
+In `init` mode you need to point `OVERPASS_PLANET_URL` to address with planet (partial) dump. This file will be downloaded,
+indexed by Overpass and later - updated using `OVERPASS_PLANET_SEQUENCE_ID` and `OVERPASS_DIFF_URL`. You need to check which
+sequence number is for your planet file. Take it from desctiption or just take a sequence number a day before your planet
+file is dated.
+
+Start initalization mode with `-i` and `-t` options to `docker run` so you will have a chance to monitor the progress of
+file downloads etc.
+
+After initialization is finished Docker container will stop. Once you start it again (with `docker start` command) it will start
+downloading minute diffs, applying them to database and serving API requests.
+
+Container exposes port 80.
+
+All data resides within /db directory in container.
+
+# Examples
+## Overpass instance covering part of the world
+In this example Overpass instance will be initialized with planet file for Poland downloaded from Geofabrik. Data will be stored in folder
+`/big/docker/overpass_db/` on the host machine. Overpass will be available on port 12345 on host machine.
+```
+docker run \
+  -e OVERPASS_META=yes \
+  -e OVERPASS_MODE=init \
+  -e OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/poland-latest.osm.bz2 \
+  -e OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/poland/minute/ \
+  -e OVERPASS_PLANET_SEQUENCE_ID=2346486 \
+  -v /big/docker/overpass_db/:/db \
+  -p 12345:80 \
+  -i -t \
+  --name overpass_poland overpass
+```
+
+## Overpass clone covering whole world
+In this example Overpass instance will be initialized with data from main Overpass instance and updated with master planet diffs.
+Data will be stored in /big/docker/overpass_clone_db/ directory on the host machine and API will be exposed on port 12346 on host machine.
+```
+docker run \
+  -e OVERPASS_META=yes
+  -e OVERPASS_MODE=clone
+  -e OVERPASS_DIFF_URL=https://planet.openstreetmap.org/replication/minute/
+  -v /big/docker/overpass_clone_db/:/db
+  -p 12346:80
+  -i -t
+  --name overpass_world
+  overpass
+```

docker-entrypoint.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -eo pipefail
+shopt -s nullglob
+OVERPASS_META=${OVERPASS_META:-no}
+OVERPASS_MODE=${OVERPASS_MODE:-clone}
+OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
+
+if [ ! -d /db/db ] ; then
+    if [ "$OVERPASS_MODE" = "clone" ]; then
+        if [[ "$OVERPASS_META" == "attic" ]] ; then
+            META="--keep-attic"
+        else
+            META="--meta=$OVERPASS_META"
+        fi
+        mkdir -p /db/db \
+        && /app/bin/download_clone.sh --db-dir=/db/db --source=http://dev.overpass-api.de/api_drolbr/ $META "--compresion_method=$OVERPASS_COMPRESSION" \
+        && cp -r /app/src/rules /db/db \
+        && chown -R overpass:overpass /db \
+        && echo "Overpass ready, you can start your container with docker start"
+        exit
+    fi
+
+    if [ "$OVERPASS_MODE" = "init" ]; then
+        lftp -c "get -c \"$OVERPASS_PLANET_URL\" -o /db/planet; exit" \
+        && /app/bin/init_osm3s.sh /db/planet /db/db /app "--meta=$OVERPASS_META" \
+        && echo $OVERPASS_PLANET_SEQUENCE_ID > /db/db/replicate_id \
+        && rm /db/planet \
+        && cp -r /app/src/rules /db/db \
+        && chown -R overpass:overpass /db \
+        && echo "Overpass ready, you can start your container with docker start"
+        exit
+    fi
+fi
+
+exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf

etc/nginx-overpass.conf

@@ -0,0 +1,66 @@
+daemon off;
+
+user  nginx;
+worker_processes  4;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+
+events {
+    worker_connections  1024;
+}
+
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile        on;
+    #tcp_nopush     on;
+
+    keepalive_timeout  65;
+
+    #gzip  on;
+
+	server {
+		listen       80;
+		server_name  localhost;
+
+		#charset koi8-r;
+		#access_log  /var/log/nginx/log/host.access.log  main;
+
+		location / {
+			root   /usr/share/nginx/html;
+			index  index.html index.htm;
+		}
+
+		#error_page  404              /404.html;
+
+		# redirect server error pages to the static page /50x.html
+		#
+		error_page   500 502 503 504  /50x.html;
+		location = /50x.html {
+			root   /usr/share/nginx/html;
+		}
+
+		location /cgi-bin/ {
+			include        fastcgi_params;
+			fastcgi_param  SCRIPT_FILENAME  /app/$fastcgi_script_name;
+			fastcgi_param  PATH_INFO        $fastcgi_path_info;
+			fastcgi_pass   unix:/nginx/fcgiwrap.socket;
+		}
+
+		location /api/ {
+            rewrite ^/api/(.+)$ /cgi-bin/$1 last;
+		}
+	}
+
+
+}

etc/supervisord.conf

@@ -0,0 +1,42 @@
+[supervisord]
+nodaemon=true
+
+[program:overpass_dispatch]
+command=/bin/bash -c "find /db/db -type s -print0 | xargs -0 --no-run-if-empty rm && /app/bin/dispatcher --osm-base --meta --db-dir=/db/db"
+user=overpass
+redirect_stderr=true
+priority=1
+
+[program:nginx]
+command=nginx
+priority=2
+
+[program:fcgiwrap]
+command=/bin/bash -c "find /nginx -type s -print0 | xargs -0 --no-run-if-empty rm && fcgiwrap -s unix:/nginx/fcgiwrap.socket"
+redirect_stderr=true
+user=nginx
+priority=3
+
+[program:fetch_diff]
+command=/app/bin/fetch_osc.sh auto %(ENV_OVERPASS_DIFF_URL)s /db/diffs
+user=overpass
+redirect_stderr=true
+priority=5
+
+[program:apply_diff]
+command=/app/bin/apply_osc_to_db.sh /db/diffs/ auto --meta=%(ENV_OVERPASS_META)s
+redirect_stderr=true
+user=overpass
+priority=4
+
+[program:dispatcher_areas]
+command=nice /app/bin/dispatcher --areas --db-dir="/db/db"
+user=overpass
+redirect_stderr=true
+priority=6
+
+[program:areas_rules]
+command=nice /app/bin/rules_loop.sh /db/db
+user=overpass
+redirect_stderr=true
+priority=7