Merge branch 'backup' into dev

Author: matthias_schaub

docker-compose.yaml

@@ -75,35 +75,34 @@ services:
             POSTGERS_PASSWORD: '${POSTGRES_PASSWORD}'
             POSTGRES_USER: 'mapswipe_workers'
             POSTGRES_DB: 'mapswipe'
+            PGDATA: '/var/lib/postgresql/mapswipe'
             WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
             GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
         # Set options for WAL-G (backup tool)
         command: postgres -c archive_mode=on -c archive_timeout=60 -c archive_command="/archive_command.sh %p"
         volumes:
-            - ./postgres-data:/var/lib/postgresql/data
+            - ./postgres-data:/var/lib/postgresql/mapswipe
         restart: unless-stopped
         ports:
           - "5432:5432"
         networks:
             - postgres
 
-    postgres_restore_backup:
-        container_name: postgres_restore_backup
+    postgres_recovery:
+        container_name: postgres_recovery
         build:
-            context: postgres/
-            dockerfile: Dockerfile-restore_backup
-            args:
-                WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
-                GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
+            context: postgres/recovery/
         environment:
             POSTGRES_PASSWORD: '${POSTGRES_PASSWORD}'
             POSTGRES_USER: 'mapswipe_workers'
             POSTGRES_DB: 'mapswipe'
-        command: postgres
+            PGDATA: '/var/lib/postgresql/mapswipe'
+            WALG_GS_PREFIX: '${WALG_GS_PREFIX}'
+            GOOGLE_APPLICATION_CREDENTIALS: '/serviceAccountKey.json'
         volumes:
-            - ./postgres-data:/var/lib/postgresql/restored_cluster
+            - ./pgdata_restored:/var/lib/postgresql/mapswipe
         ports:
-          - "5432:5432"
+          - "5445:5432"
 
     firebase_deploy:
         container_name: firebase_deploy

postgres/Dockerfile

@@ -4,12 +4,12 @@ COPY initdb.sql docker-entrypoint-initdb.d/
 COPY serviceAccountKey.json serviceAccountKey.json
 
 # Copy backup scripts and make them executable
-COPY wal-g/make_basebackup.sh make_basebackup.sh
-COPY wal-g/archive_command.sh archive_command.sh
+COPY backup/make_basebackup.sh make_basebackup.sh
+COPY backup/archive_command.sh archive_command.sh
 RUN chmod +x make_basebackup.sh
 RUN chmod +x archive_command.sh
 
-# Install wal-g (used by backup scripts)
+# Install wal-g (used by backup/recovery scripts)
 RUN apt-get update && apt-get install -y wget
 RUN wget https://github.com/wal-g/wal-g/releases/download/v0.2.9/wal-g.linux-amd64.tar.gz
 RUN tar -zxvf wal-g.linux-amd64.tar.gz

postgres/Dockerfile-restore_backup

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-export PGDATABSE=$POSTGRES_DB
+export PGDATABASE=$POSTGRES_DB
 export PGUSER=$POSTGRES_USER
 export PGPASSWORD=$POSTGRES_PASSWORD
 

postgres/wal-g/archive_command.sh postgres/backup/archive_command.shpostgres/wal-g/archive_command.sh renamed to postgres/backup/archive_command.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
-export PGDATABSE=$POSTGRES_DB
+export PGDATABASE=$POSTGRES_DB
 export PGUSER=$POSTGRES_USER
 export PGPASSWORD=$POSTGRES_PASSWORD
 
-wal-g backup-push /var/lib/postgresql/data
+wal-g backup-push /var/lib/postgresql/mapswipe

postgres/wal-g/make_basebackup.sh postgres/backup/make_basebackup.shpostgres/wal-g/make_basebackup.sh renamed to postgres/backup/make_basebackup.sh

@@ -102,10 +102,7 @@ CREATE TABLE IF NOT EXISTS results_temp (
     );
 
 --
--- USERS
+-- VIEWS
 --
--- create a read-only user for backups
-CREATE USER backup WITH PASSWORD 'backupuserpassword';
-GRANT CONNECT ON DATABASE mapswipe TO backup;
-GRANT USAGE ON SCHEMA public TO backup;
-GRANT SELECT ON ALL TABLES IN SCHEMA public TO backup;
+-- create views for statistics
+\i /docker-entrypoint-initdb.d/stat_views.sql

postgres/initdb.sql

@@ -0,0 +1,24 @@
+# Based on latest stable postgres.
+FROM mdillon/postgis
+
+COPY init_recovery.sh /usr/local/bin/
+COPY restore_command.sh .
+COPY recovery.conf .
+COPY serviceAccountKey.json .
+
+RUN chmod +x /usr/local/bin/init_recovery.sh
+RUN chmod +x restore_command.sh
+
+# Install wal-g (used by backup/restore scripts).
+RUN apt-get update && apt-get install -y wget
+RUN wget https://github.com/wal-g/wal-g/releases/download/v0.2.9/wal-g.linux-amd64.tar.gz
+RUN tar -zxvf wal-g.linux-amd64.tar.gz
+RUN mv wal-g /usr/local/bin/wal-g
+
+# Restore base backup,
+# set user permissions and
+# copy recovery.conf into data cluster.
+ENTRYPOINT ["init_recovery.sh"]
+# Run default Postgres/PostGIS entrypoint and
+# start Postgres.
+CMD ["docker-entrypoint.sh", "postgres"]

postgres/recovery/Dockerfile

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+# Restore base backup.
+# Will create a new Database Cluster.
+wal-g backup-fetch /var/lib/postgresql/mapswipe LATEST
+
+# Move recovery.conf to cluster to tell postgres where to fetch WAL from.
+mv /recovery.conf /var/lib/postgresql/mapswipe/
+
+chown -R postgres:postgres /var/lib/postgresql/mapswipe
+chmod 700 /var/lib/postgresql/mapswipe
+
+# Execute command given in Dockerfile ("docker-entrypoint.sh", "postgres").
+exec "$@"