Skip to content

Commit cc49f05

Browse files
author
Ruben Lopez M
authored
Merge pull request #190 from developmentseed/overpass_api
Overpass container
2 parents 774207b + 7f4a98e commit cc49f05

File tree

11 files changed

+377
-4
lines changed

11 files changed

+377
-4
lines changed

‎.env-overpass.example

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
OVERPASS_META=yes
2+
OVERPASS_MODE=init
3+
OVERPASS_PLANET_URL=http://download.geofabrik.de/europe/monaco-latest.osm.bz2
4+
OVERPASS_DIFF_URL=http://download.openstreetmap.fr/replication/europe/monaco/minute/
5+
OVERPASS_RULES_LOAD=10

‎.gitignore

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22
.env-tiler
33
.env-nominatim
44
.env-tasking-manager
5+
.env-overpass
56
.DS_Store
7+
68
usa.values.yaml
79
prod.values.yaml
810
planet.values.yaml
@@ -26,4 +28,5 @@ tiler-imposm/config.json
2628
tiler-imposm/cachedir/
2729
tiler-imposm/diff/
2830
tiler-imposm/imposm3_expire_dir
29-
tiler-server/imposm/
31+
tiler-server/imposm/
32+
overpass-api-db/

‎chartpress.yaml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,6 @@ charts:
3131
# tiler-visor:
3232
# valuesPath: tilerVisor.image
3333
nominatim:
34-
valuesPath: nominatim.image
34+
valuesPath: nominatim.image
35+
overpass-api:
36+
valuesPath: overpassApi.image

‎docker-compose.yml

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
version: '3'
23
services:
34
######################################################
@@ -205,4 +206,18 @@ services:
205206
- '7070:8080'
206207
env_file:
207208
- ./.env-nominatim
208-
entrypoint: sh /app/startapache.sh
209+
entrypoint: sh /app/startapache.sh
210+
######################################################
211+
### Overpass API section
212+
######################################################
213+
overpass-api:
214+
image: osmseed-overpass-api:v1
215+
build:
216+
context: ./images/overpass-api
217+
dockerfile: Dockerfile
218+
ports:
219+
- '8081:80'
220+
volumes:
221+
- ./overpass-api-db:/db
222+
env_file:
223+
- ./.env-overpass

‎images/overpass-api/Dockerfile

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
FROM wiktorn/overpass-api:0.7.56.8
2+
COPY docker-entrypoint.sh /app/
3+
RUN chmod a+rx /app/docker-entrypoint.sh
4+
CMD ["/app/docker-entrypoint.sh"]

‎images/overpass-api/README.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Overpass api
2+
3+
This continaer is base on: https://github.com/wiktorn/Overpass-API

‎images/overpass-api/docker-entrypoint.sh

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/bin/bash
2+
3+
set -eo pipefail
4+
shopt -s nullglob
5+
OVERPASS_META=${OVERPASS_META:-no}
6+
OVERPASS_MODE=${OVERPASS_MODE:-clone}
7+
OVERPASS_COMPRESSION=${OVERPASS_COMPRESSION:-gz}
8+
OVERPASS_FLUSH_SIZE=${OVERPASS_FLUSH_SIZE:-16}
9+
OVERPASS_CLONE_SOURCE=${OVERPASS_CLONE_SOURCE:-http://dev.overpass-api.de/api_drolbr/}
10+
11+
# this is used by other processes, so needs to be exported
12+
export OVERPASS_MAX_TIMEOUT=${OVERPASS_MAX_TIMEOUT:-1000s}
13+
14+
if [[ "$OVERPASS_META" == "attic" ]] ; then
15+
META="--keep-attic"
16+
elif [[ "${OVERPASS_META}" == "yes" ]] ; then
17+
META="--meta"
18+
else
19+
META=""
20+
fi
21+
22+
for f in /docker-entrypoint-initdb.d/*; do
23+
case "$f" in
24+
*.sh)
25+
if [[ -x "$f" ]]; then
26+
echo "$0: running $f"
27+
"$f"
28+
else
29+
echo "$0: sourcing $f"
30+
. "$f"
31+
fi
32+
;;
33+
*) echo "$0: ignoring $f" ;;
34+
esac
35+
echo
36+
done
37+
38+
function startAPIServer(){
39+
envsubst '${OVERPASS_MAX_TIMEOUT}' < /etc/nginx/nginx.conf.template > /etc/nginx/nginx.conf
40+
echo "Starting supervisord process"
41+
exec /usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
42+
}
43+
44+
if [[ ! -f /db/init_done ]] ; then
45+
echo "No database directory. Initializing"
46+
if [[ "${USE_OAUTH_COOKIE_CLIENT}" = "yes" ]]; then
47+
/app/venv/bin/python /app/bin/oauth_cookie_client.py -o /db/cookie.jar -s /secrets/oauth-settings.json --format netscape
48+
# necessary to add newline at the end as oauth_cookie_client doesn't do that
49+
echo >> /db/cookie.jar
50+
else
51+
echo "# Netscape HTTP Cookie File" > /db/cookie.jar
52+
echo "${OVERPASS_COOKIE_JAR_CONTENTS}" >> /db/cookie.jar
53+
fi
54+
chown overpass /db/cookie.jar
55+
56+
if [[ "$OVERPASS_MODE" = "clone" ]]; then
57+
mkdir -p /db/db \
58+
&& /app/bin/download_clone.sh --db-dir=/db/db --source="${OVERPASS_CLONE_SOURCE}" --meta="${OVERPASS_META}" \
59+
&& cp /db/db/replicate_id /db/replicate_id \
60+
&& cp -r /app/etc/rules /db/db \
61+
&& chown -R overpass:overpass /db \
62+
&& touch /db/init_done \
63+
&& echo "Overpass ready, you can start your container with docker start"
64+
exit
65+
fi
66+
67+
if [[ "$OVERPASS_MODE" = "init" ]]; then
68+
while true ; do
69+
CURL_STATUS_CODE=$(curl -L -b /db/cookie.jar -o /db/planet.osm.bz2 -w "%{http_code}" "${OVERPASS_PLANET_URL}")
70+
case "${CURL_STATUS_CODE}" in
71+
429)
72+
echo "Server responded with 429 Too many requests. Trying again in 5 minutes..."
73+
sleep 300
74+
continue
75+
;;
76+
# for `file:///` scheme curl returns `000` HTTP status code
77+
200 | 000)
78+
(
79+
if [[ ! -z "${OVERPASS_PLANET_PREPROCESS+x}" ]]; then
80+
echo "Running preprocessing command: ${OVERPASS_PLANET_PREPROCESS}"
81+
eval "${OVERPASS_PLANET_PREPROCESS}"
82+
fi \
83+
&& /app/bin/init_osm3s.sh /db/planet.osm.bz2 /db/db /app "${META}" "--version=$(osmium fileinfo -e -g data.timestamp.last /db/planet.osm.bz2) --compression-method=${OVERPASS_COMPRESSION} --map-compression-method=${OVERPASS_COMPRESSION} --flush-size=${OVERPASS_FLUSH_SIZE}" \
84+
&& echo "Database created. Now updating it." \
85+
&& cp -r /app/etc/rules /db/db \
86+
&& chown -R overpass:overpass /db \
87+
&& echo "Updating" \
88+
&& /app/bin/update_overpass.sh "-O /db/planet.osm.bz2" \
89+
&& /app/bin/osm3s_query --progress --rules --db-dir=/db/db < /db/db/rules/areas.osm3s \
90+
&& touch /db/init_done \
91+
&& rm /db/planet.osm.bz2 \
92+
&& chown -R overpass:overpass /db \
93+
&& echo "Overpass ready, you can start your container with docker start" \
94+
&& startAPIServer
95+
) || (
96+
echo "Failed to process planet file"
97+
exit
98+
)
99+
;;
100+
403)
101+
echo "Access denied when downloading planet file. Check your OVERPASS_PLANET_URL and OVERPASS_COOKIE_JAR_CONTENTS or USE_OAUTH_COOKIE_CLIENT"
102+
cat /db/cookie.jar
103+
exit
104+
;;
105+
*)
106+
echo "Failed to download planet file. HTTP status code: ${CURL_STATUS_CODE}"
107+
cat /db/planet.osm.bz2
108+
exit
109+
;;
110+
esac
111+
exit
112+
done
113+
fi
114+
fi
115+
116+
# Start api server
117+
startAPIServer

‎osm-seed/templates/overpass-api-dp.yaml

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
{{- if .Values.overpassApi.enabled -}}
2+
{{- if .Values.overpassApi.persistenceDisk.enabled -}}
3+
apiVersion: v1
4+
kind: PersistentVolume
5+
metadata:
6+
name: {{ .Release.Name }}-overpass-api-pv
7+
labels:
8+
app: {{ template "osm-seed.name" . }}
9+
component: overpass-api-pd
10+
environment: {{ .Values.environment }}
11+
release: {{ .Release.Name }}
12+
spec:
13+
storageClassName: "" # It's necessary to specify "" as the storageClassName ,so that the default storage class won't be used, see : https://kubernetes.io/docs/concepts/storage/persistent-volumes/#class-1
14+
# AWS Provider
15+
{{- if eq .Values.cloudProvider "aws" }}
16+
capacity:
17+
storage: {{ .Values.overpassApi.persistenceDisk.AWS_ElasticBlockStore_size }}
18+
accessModes:
19+
- ReadWriteOnce
20+
awsElasticBlockStore:
21+
volumeID: {{ .Values.overpassApi.persistenceDisk.AWS_ElasticBlockStore_volumeID }}
22+
fsType: ext4
23+
{{- end }}
24+
# GCP Provider
25+
{{- if eq .Values.cloudProvider "gcp" }}
26+
capacity:
27+
storage: {{ .Values.overpassApi.persistenceDisk.GCP_gcePersistentDisk_size }}
28+
accessModes:
29+
- ReadWriteOnce
30+
gcePersistentDisk:
31+
pdName: {{ .Values.overpassApi.persistenceDisk.GCP_gcePersistentDisk_pdName}}
32+
fsType: ext4
33+
{{- end }}
34+
---
35+
apiVersion: v1
36+
kind: PersistentVolumeClaim
37+
metadata:
38+
name: {{ .Release.Name }}-overpass-api-pv-claim
39+
labels:
40+
app: {{ template "osm-seed.name" . }}
41+
component: db-pd
42+
environment: {{ .Values.environment }}
43+
release: {{ .Release.Name }}
44+
spec:
45+
storageClassName: ""
46+
volumeName: {{ .Release.Name }}-overpass-api-pv
47+
accessModes:
48+
- ReadWriteOnce
49+
resources:
50+
requests:
51+
{{- if eq .Values.cloudProvider "aws" }}
52+
storage: {{ .Values.overpassApi.persistenceDisk.AWS_ElasticBlockStore_size }}
53+
{{- end }}
54+
{{- if eq .Values.cloudProvider "gcp" }}
55+
storage: {{ .Values.overpassApi.persistenceDisk.GCP_gcePersistentDisk_size }}
56+
{{- end }}
57+
{{- end }}
58+
{{- end }}

‎osm-seed/templates/overpass-api-service.yaml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{{- if .Values.tilerServer.enabled -}}
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: {{ template "osm-seed.fullname" . }}-overpass-api
6+
labels:
7+
app: {{ template "osm-seed.name" . }}
8+
component: overpass-api-service
9+
environment: {{ .Values.environment }}
10+
release: {{ .Release.Name }}
11+
{{- if eq .Values.cloudProvider "aws" }}
12+
annotations:
13+
service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "300"
14+
{{- if .Values.AWS_SSL_ARN }}
15+
service.beta.kubernetes.io/aws-load-balancer-ssl-cert: {{ .Values.AWS_SSL_ARN }}
16+
service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
17+
{{- end }}
18+
{{- end }}
19+
spec:
20+
# In case cloudProvider=aws
21+
{{- if eq .Values.cloudProvider "aws" }}
22+
type: LoadBalancer
23+
{{- end }}
24+
25+
# In case cloudProvider=gcp
26+
{{- if eq .Values.cloudProvider "gcp" }}
27+
type: LoadBalancer
28+
loadBalancerIP : {{ .Values.overpassApi.staticIp }}
29+
{{- end }}
30+
31+
# In case cloudProvider=minikube
32+
{{- if eq .Values.cloudProvider "minikube" }}
33+
type: NodePort
34+
{{- end }}
35+
36+
ports:
37+
- port: 80
38+
targetPort: 80
39+
protocol: TCP
40+
name: http
41+
{{- if .Values.AWS_SSL_ARN }}
42+
- port: 443
43+
targetPort: http
44+
protocol: TCP
45+
name: https
46+
{{- end }}
47+
48+
selector:
49+
app: {{ template "osm-seed.name" . }}
50+
release: {{ .Release.Name }}
51+
run: {{ .Release.Name }}-overpass-api
52+
{{- end }}

‎osm-seed/templates/overpass-api-statefulset.yaml

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
{{- if .Values.overpassApi.enabled -}}
2+
apiVersion: apps/v1
3+
kind: StatefulSet
4+
metadata:
5+
name: {{ template "osm-seed.fullname" . }}-overpass-api
6+
labels:
7+
app: {{ template "osm-seed.name" . }}
8+
component: overpass-api-statefulset
9+
environment: {{ .Values.environment }}
10+
release: {{ .Release.Name }}
11+
spec:
12+
replicas: 1
13+
selector:
14+
matchLabels:
15+
app: {{ template "osm-seed.name" . }}
16+
release: {{ .Release.Name }}
17+
run: {{ .Release.Name }}-overpass-api
18+
serviceName: {{ .Release.Name }}-overpass-api
19+
template:
20+
metadata:
21+
labels:
22+
app: {{ template "osm-seed.name" . }}
23+
release: {{ .Release.Name }}
24+
run: {{ .Release.Name }}-overpass-api
25+
spec:
26+
containers:
27+
- name: {{ .Chart.Name }}-overpass-api
28+
image: "{{ .Values.overpassApi.image.name }}:{{ .Values.overpassApi.image.tag }}"
29+
ports:
30+
- name: http
31+
containerPort: 80
32+
protocol: TCP
33+
# livenessProbe:
34+
# httpGet:
35+
# path: /
36+
# port: 80
37+
# initialDelaySeconds: 600 # 10 min, because the compile process takes time.
38+
# timeoutSeconds: 30
39+
{{- if .Values.overpassApi.resources.enabled }}
40+
resources:
41+
requests:
42+
memory: {{ .Values.overpassApi.resources.requests.memory }}
43+
cpu: {{ .Values.overpassApi.resources.requests.cpu }}
44+
limits:
45+
memory: {{ .Values.overpassApi.resources.limits.memory }}
46+
cpu: {{ .Values.overpassApi.resources.limits.cpu }}
47+
{{- end }}
48+
env:
49+
# overpass-api env variables
50+
- name: OVERPASS_META
51+
value: {{ .Values.overpassApi.env.OVERPASS_META | quote}}
52+
- name: OVERPASS_MODE
53+
value: {{ .Values.overpassApi.env.OVERPASS_MODE | quote}}
54+
- name: OVERPASS_PLANET_URL
55+
value: {{ .Values.overpassApi.env.OVERPASS_PLANET_URL | quote}}
56+
- name: OVERPASS_DIFF_URL
57+
value: {{ .Values.overpassApi.env.OVERPASS_DIFF_URL | quote}}
58+
- name: OVERPASS_RULES_LOAD
59+
value: {{ .Values.overpassApi.env.OVERPASS_RULES_LOAD | quote}}
60+
- name: OVERPASS_PLANET_PREPROCESS
61+
value: {{ .Values.overpassApi.env.OVERPASS_PLANET_PREPROCESS | quote}}
62+
volumeMounts:
63+
- mountPath: /db
64+
name: overpass-api-storage
65+
restartPolicy: Always
66+
volumes:
67+
- name: overpass-api-storage
68+
{{- if .Values.overpassApi.persistenceDisk.enabled }}
69+
persistentVolumeClaim:
70+
claimName: {{ .Release.Name }}-overpass-api-pv-claim
71+
{{- else }}
72+
emptyDir: {}
73+
{{- end }}
74+
{{- if .Values.overpassApi.nodeSelector.enabled }}
75+
nodeSelector:
76+
{{ .Values.overpassApi.nodeSelector.label_key }} : {{ .Values.overpassApi.nodeSelector.label_value }}
77+
{{- end }}
78+
{{- end }}

0 commit comments

Comments
 (0)