Merge pull request #191 from developmentseed/tiler_s3_cache

Ruben Lopez M · web-flow · commit e2a5961e3ec6 · 2021-05-21T13:57:08.000-05:00
S3 cache Deployment - Tiler
diff --git a/.env-tiler.example b/.env-tiler.example
@@ -19,6 +19,7 @@ POSTGRES_DB=tiler-osm
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=1234
 POSTGRES_PORT=5432
+POSTGRES_DB_MAX_CONNECTIONS=200
 
 # ====================================================================================================
 # tiler-imposm  evn variables  
diff --git a/.github/workflows/chartpress.yaml b/.github/workflows/chartpress.yaml
@@ -18,7 +18,7 @@ jobs:
       run: git config --global user.email "noreply@developmentseed.org" && git config --global user.name "Github Action"
     - name: Install Chartpress
       run: |
-        pip install -v chartpress
+        pip install -v chartpress && pip install six
     - name: Run Chartpress
       run: chartpress --push --publish-chart
       env:
diff --git a/images/tiler-db/Dockerfile b/images/tiler-db/Dockerfile
@@ -6,4 +6,5 @@ COPY ./config/docker-entrypoint.sh /usr/local/bin/
 RUN mkdir -p /docker-entrypoint-initdb.d
 COPY ./config/initdb_db.sh /docker-entrypoint-initdb.d/postgis.sh
 COPY ./config/update_db.sh /usr/local/bin
+COPY ./config/update_postgresql.sh /docker-entrypoint-initdb.d
 
diff --git a/images/tiler-db/config/update_postgresql.sh b/images/tiler-db/config/update_postgresql.sh
@@ -0,0 +1,26 @@
+#!/bin/sh
+
+# max_connections
+if [[ "${POSTGRES_DB_MAX_CONNECTIONS}X" != "X" ]]; then
+  sed -i -e"s/^.*max_connections =.*$/max_connections = $POSTGRES_DB_MAX_CONNECTIONS/" $PGDATA/postgresql.conf
+fi
+
+# shared_buffers
+if [[ "${POSTGRES_DB_SHARED_BUFFERS}X" != "X" ]]; then
+  sed -i -e"s/^.*shared_buffers =.*$/shared_buffers = $POSTGRES_DB_SHARED_BUFFERS/" $PGDATA/postgresql.conf
+fi
+
+# work_mem
+if [[ "${POSTGRES_DB_WORK_MEM}X" != "X" ]]; then
+  sed -i -e"s/^.*#work_mem =.*$/work_mem = $POSTGRES_DB_WORK_MEM/" $PGDATA/postgresql.conf
+fi
+
+# maintenance_work_mem
+if [[ "${POSTGRES_DB_MAINTENANCE_WORK_MEM}X" != "X" ]]; then
+  sed -i -e"s/^.*maintenance_work_mem =.*$/maintenance_work_mem = $POSTGRES_DB_MAINTENANCE_WORK_MEM/" $PGDATA/postgresql.conf
+fi
+
+# effective_cache_size
+if [[ "${POSTGRES_DB_EFFECTIVE_CACHE_SIZE}X" != "X" ]]; then
+  sed -i -e"s/^.*effective_cache_size =.*$/effective_cache_size = $POSTGRES_DB_EFFECTIVE_CACHE_SIZE/" $PGDATA/postgresql.conf
+fi
diff --git a/images/tiler-imposm/Dockerfile b/images/tiler-imposm/Dockerfile
@@ -1,81 +1,59 @@
-FROM ubuntu:16.04
+FROM osgeo/gdal:ubuntu-small-3.2.3
 
 RUN apt-get -y update
 RUN apt-get install -y \
-     \
     g++ \
-    git-core \
     libboost-dev \
     libboost-system-dev \
     libboost-filesystem-dev \
     libexpat1-dev \
     zlib1g-dev \
     libbz2-dev \
     libpq-dev \
-    libgeos-dev \
     libgeos++-dev \
     libproj-dev \
     libleveldb-dev \
     libgeos-dev \
     libprotobuf-dev \
-    libgeos++-dev \
-    libjson0-dev \
+    git-core \
     curl \
     wget \
     unzip \
     software-properties-common
 
-# Install python
-RUN add-apt-repository ppa:deadsnakes/ppa \
-        && apt-get update && \
-        apt-get update -y  && \
-        apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \
-        apt-get install -y git  && \
-        # update pip
-        python3.6 -m pip install pip --upgrade && \
-        python3.6 -m pip install wheel
+# # Install python
+RUN add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt-get install -y build-essential python3.6 python3.6-dev python3-pip && \
+    python3 -m pip install pip --upgrade && \
+    python3 -m pip install wheel
 
 # Install postgresql-client
-RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-    && sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee  /etc/apt/sources.list.d/pgdg.list' \
-    && apt-get update && apt-get install -y postgresql-client \
-    && rm -rf /var/lib/apt/lists/*
-
-# Gdal is required to process the natural earth files
-RUN add-apt-repository ppa:ubuntugis/ppa \
-    && apt-get -y update \
-    && apt-get install -y gdal-bin \
-    && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y postgresql-client && \
+    rm -rf /var/lib/apt/lists/*
 
 # Install go
-RUN add-apt-repository ppa:gophers/archive \
-    && apt-get -y update \
-    && apt-get install -y golang-1.10-go \
-    && cp /usr/lib/go-1.10/bin/go /usr/bin/go \
-    && cp /usr/lib/go-1.10/bin/gofmt /usr/bin/gofmt
+RUN wget -c https://dl.google.com/go/go1.10.linux-amd64.tar.gz -O - | tar -xz -C /usr/local
+ENV PATH $PATH:/usr/local/go/bin
 
-# Install imposm
-ENV GOPATH /usr/bin
-RUN mkdir -p go
 WORKDIR /go
-RUN export GOPATH=`pwd`
+ENV GOPATH /go
 RUN go get github.com/omniscale/imposm3
 RUN go install github.com/omniscale/imposm3/cmd/imposm
-RUN cp $GOPATH/bin/imposm /usr/bin/imposm
+ENV PATH $PATH:$GOPATH/bin
 
 # Install awscli  and gsutil  to get the pbf file
 RUN pip3 install -U setuptools
 RUN pip3 install awscli
 RUN pip3 install gsutil
 
-
 ENV IMPOSMDATA /mnt/data
-RUN mkdir -p "$IMPOSMDATA" && chmod 777 "$IMPOSMDATA"
-VOLUME /mnt/data
+RUN mkdir -p $IMPOSMDATA && chmod 777 -R $IMPOSMDATA
+WORKDIR $IMPOSMDATA
+VOLUME $IMPOSMDATA
 
-WORKDIR /mnt/data
 WORKDIR /osm
 COPY config config
 COPY scripts scripts
 COPY start.sh start.sh
-CMD ./start.sh
+CMD ./start.sh
diff --git a/images/tiler-imposm/start.sh b/images/tiler-imposm/start.sh
@@ -124,7 +124,6 @@ function importData () {
     echo "Import OSM Land"
     ./scripts/osm_land.sh
     echo "Import PBF file"
-
     if [ -z "$TILER_IMPORT_LIMIT" ]; then
         imposm import \
         -config $workDir/config.json \
@@ -149,12 +148,13 @@ function importData () {
     # These index will help speed up tegola tile generation
     psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" -a -f config/postgis_index.sql
 
+    touch /mnt/data/init_done
     # Update the DB
     updateData
 }
 
 
-echo "Connecting... to postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB"
+echo "Connecting to DB..."
 flag=true
 while "$flag" = true; do
     pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
@@ -163,21 +163,16 @@ while "$flag" = true; do
         hasData=$(psql "postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST/$POSTGRES_DB" \
         -c "SELECT count(*) FROM information_schema.tables WHERE table_schema = 'public'" | sed -n 3p | sed 's/ //g')
         # After import there are more than 70 tables
-        
-        if [ $hasData  \> 70 ]; then
+        echo "$hasData tables in the DB"
+        if ([ $hasData  \> 70 ] && [[ ! -f /mnt/data/init_done ]]); then
             echo "Update the DB with osm data"
             updateData
         else
-            if [[ ! -f /mnt/data/init_done ]]; then
-                echo "Import PBF data to DB"
-                getData
-                if [ -f $PBFFile ]; then
-                    echo "Start importing the data"
-                    importData \
-                    && touch /mnt/data/init_done
-                fi
-            else
-                echo "Data has already imported"
+            echo "Import PBF data to DB"
+            getData
+            if [ -f $PBFFile ]; then
+                echo "Start importing the data"
+                importData
             fi
         fi
 done
diff --git a/images/tiler-server/cache_cleaner.sh b/images/tiler-server/cache_cleaner.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+flag=true
+while "$flag" = true; do
+  pg_isready -h $POSTGRES_HOST -p 5432 >/dev/null 2>&2 || continue
+  flag=false
+  ./tile_cache_downloader.sh & ./expire-watcher.sh
+done
diff --git a/osm-seed/templates/tiler-db-statefulset.yaml b/osm-seed/templates/tiler-db-statefulset.yaml
@@ -45,6 +45,16 @@ spec:
               value: {{ default "postgres" .Values.tilerDb.env.POSTGRES_USER | quote }}
             - name: PGDATA
               value: {{ .Values.tilerDb.persistenceDisk.mountPath }}
+            - name: POSTGRES_DB_MAX_CONNECTIONS
+              value: {{ default 100 .Values.tilerDb.env.POSTGRES_DB_MAX_CONNECTIONS | quote }}
+            - name: POSTGRES_DB_SHARED_BUFFERS
+              value: {{ default "128MB" .Values.tilerDb.env.POSTGRES_DB_SHARED_BUFFERS | quote }}
+            - name: POSTGRES_DB_WORK_MEM
+              value: {{ default "4MB" .Values.tilerDb.env.POSTGRES_DB_WORK_MEM | quote }}
+            - name: POSTGRES_DB_MAINTENANCE_WORK_MEM
+              value: {{ default "64MB" .Values.tilerDb.env.POSTGRES_DB_MAINTENANCE_WORK_MEM | quote }}
+            - name: POSTGRES_DB_EFFECTIVE_CACHE_SIZE
+              value: {{ default "4GB" .Values.tilerDb.env.POSTGRES_DB_EFFECTIVE_CACHE_SIZE | quote }}
             - name: POD_IP
               valueFrom: { fieldRef: { fieldPath: status.podIP } }
           livenessProbe:
diff --git a/osm-seed/templates/tiler-server-cache-cleaner-deployment.yaml b/osm-seed/templates/tiler-server-cache-cleaner-deployment.yaml
@@ -0,0 +1,83 @@
+{{- if .Values.tilerServer.enabled -}}
+{{ if eq .Values.tilerServer.env.TILER_CACHE_TYPE "s3" }}
+{{- if .Values.tilerServerCacheCleaner.enabled -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-tiler-server-cache-cleaner
+  labels:
+    app: {{ template "osm-seed.name" . }}
+    component: tiler-server-cache-cleaner
+    environment: {{ .Values.environment }}
+    release: {{ .Release.Name }}
+spec:
+  replicas: {{ .Values.tilerServerCacheCleaner.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ template "osm-seed.name" . }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "osm-seed.name" . }}
+    spec:
+      containers:
+        - name: {{ .Release.Name }}-tiler-server-cache-cleaner-deployment
+          image: {{ .Values.tilerServer.image.name }}:{{ .Values.tilerServer.image.tag }}
+          command: [{{.Values.tilerServerCacheCleaner.command}}]
+          {{- if .Values.tilerServerCacheCleaner.resources.enabled }}
+          resources:
+            requests:
+              memory: {{ .Values.tilerServerCacheCleaner.resources.requests.memory }}
+              cpu: {{ .Values.tilerServerCacheCleaner.resources.requests.cpu }}
+            limits:
+              memory: {{ .Values.tilerServerCacheCleaner.resources.limits.memory }}
+              cpu: {{ .Values.tilerServerCacheCleaner.resources.limits.cpu }}
+          {{- end }}
+          env:
+              # tiler-db env variables
+            - name: POSTGRES_HOST
+              value: {{ .Release.Name }}-tiler-db
+            - name: POSTGRES_DB
+              value: {{ .Values.tilerDb.env.POSTGRES_DB }}
+            - name: POSTGRES_PASSWORD
+              value: {{ quote .Values.tilerDb.env.POSTGRES_PASSWORD }}
+            - name: POSTGRES_USER
+              value: {{ .Values.tilerDb.env.POSTGRES_USER }}
+            - name: POSTGRES_PORT
+              value: {{ .Values.tilerDb.env.POSTGRES_PORT | quote }}
+              # tiler-server env variables
+            - name: TILER_SERVER_PORT
+              value: {{ .Values.tilerServer.env.TILER_SERVER_PORT | quote }}
+            - name: TILER_CACHE_TYPE
+              value: {{ .Values.tilerServer.env.TILER_CACHE_TYPE }}
+            - name: TILER_CACHE_REGION
+              value: {{ .Values.tilerServer.env.TILER_CACHE_REGION }}
+            - name: TILER_CACHE_BUCKET
+              value: {{ .Values.tilerServer.env.TILER_CACHE_BUCKET }}
+            - name: TILER_CACHE_BASEPATH
+              value: {{ quote .Values.tilerServer.env.TILER_CACHE_BASEPATH }}
+            - name: TILER_CACHE_AWS_ACCESS_KEY_ID
+              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_ACCESS_KEY_ID }}
+            - name: TILER_CACHE_AWS_SECRET_ACCESS_KEY
+              value: {{ quote .Values.tilerServer.env.TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
+            - name: TILER_CACHE_MAX_ZOOM
+              value: {{ quote .Values.tilerServer.env.TILER_CACHE_MAX_ZOOM }}
+            - name: CLOUDPROVIDER
+              value: {{ .Values.cloudProvider }}
+            # In case cloudProvider=aws
+            {{- if eq .Values.cloudProvider "aws" }}
+            - name: AWS_S3_BUCKET
+              value: {{ .Values.AWS_S3_BUCKET }}
+            {{- end }}
+            # In case cloudProvider=gcp
+            {{- if eq .Values.cloudProvider "gcp" }}
+            - name: GCP_STORAGE_BUCKET
+              value: {{ .Values.GCP_STORAGE_BUCKET }}
+            {{- end }}
+      {{- if .Values.tilerServerCacheCleaner.nodeSelector.enabled }}
+      nodeSelector:
+        {{ .Values.tilerServerCacheCleaner.nodeSelector.label_key }} : {{ .Values.tilerServerCacheCleaner.nodeSelector.label_value }}
+      {{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/osm-seed/templates/tiler-server-deployment.yaml b/osm-seed/templates/tiler-server-deployment.yaml
diff --git a/osm-seed/templates/tiler-server-dp.yaml b/osm-seed/templates/tiler-server-dp.yaml
diff --git a/osm-seed/templates/tiler-server-statefulset.yaml b/osm-seed/templates/tiler-server-statefulset.yaml
diff --git a/osm-seed/values.yaml b/osm-seed/values.yaml
