Add offline backup script with setup recipe (#92)

* Add initial backup scripts

* Add requirements & basic script

* Move to backups folder

* Add timer target

* Add recipe for enabling backup service

* Lint

* Make sure timer is started on first run

* Add matrix webhook alerting for backup completion

Author: AetherUnbound

justfile

@@ -4,16 +4,17 @@ set dotenv-load := false
 default:
   @just --list --unsorted
 
-IS_PROD := env_var_or_default("IS_PROD", "")
+IS_PROD := env("IS_PROD", "")
 DOCKER_FILES := "--file=docker-compose.yml" + (
     if IS_PROD != "true" {" --file=docker-compose.dev.yml"}
     else {" --file=docker-compose.prod.yml"}
 )
-SERVICE := env_var_or_default("SERVICE", "scheduler")
+SERVICE := env("SERVICE", "scheduler")
+SCRIPT_DIR := justfile_directory() + "/techbloc_airflow/scripts"
 
 export PROJECT_PY_VERSION := `grep '# PYTHON' requirements_prod.txt | awk -F= '{print $2}'`
 export PROJECT_AIRFLOW_VERSION := `grep '^apache-airflow' requirements_prod.txt | awk -F= '{print $3}'`
-export SSH_DIRECTORY := env_var_or_default("SSH_DIRECTORY", "")
+export SSH_DIRECTORY := env("SSH_DIRECTORY", "")
 
 # Print the required Python version
 @py-version:
@@ -122,3 +123,21 @@ init:
 # Launch a pgcli shell on the postgres container (defaults to openledger) use "airflow" for airflow metastore
 db-shell:
     @just run bash -c 'sqlite3 /opt/airflow/db/airflow.db'
+
+# Set up the offsite backup systemd service (requires sudo)
+backup-setup:
+    git update-index --assume-unchanged techbloc_airflow/scripts/backups/techbloc-backup.env
+    @echo "Please edit the environment file at {{ SCRIPT_DIR }}/backups/techbloc-backup.env before continuing"
+    @read -p "Press Enter to continue" REPLY
+    sudo mkdir -p /opt/techbloc/
+    sudo chown $USER /opt/techbloc/
+    python3 -m venv /opt/techbloc/venv
+    /opt/techbloc/venv/bin/pip install -r {{ SCRIPT_DIR }}/backups/requirements.txt
+    cp {{ SCRIPT_DIR }}/backups/techbloc-backup.env /opt/techbloc/techbloc-backup.env
+    cp {{ SCRIPT_DIR }}/backups/techbloc-backup.py /opt/techbloc/techbloc-backup.py
+    sudo cp {{ SCRIPT_DIR }}/backups/techbloc-backup.service /etc/systemd/system/techbloc-backup.service
+    sudo cp {{ SCRIPT_DIR }}/backups/techbloc-backup.timer /etc/systemd/system/techbloc-backup.timer
+    sudo systemctl daemon-reload
+    sudo systemctl start techbloc-backup.timer
+    sudo systemctl enable techbloc-backup.timer
+    @echo "Backup service set up successfully!"

techbloc_airflow/scripts/backups/requirements.txt

@@ -0,0 +1,2 @@
+boto3
+requests

techbloc_airflow/scripts/backups/techbloc-backup.env

@@ -0,0 +1,7 @@
+# Environment file for techbloc-backups
+SPACES_ACCESS_KEY={spaces_access_key}
+SPACES_SECRET_KEY={spaces_secret_key}
+SPACES_BUCKET_NAME={spaces_bucket_name}
+DOWNLOAD_DIR={download_dir}
+MATRIX_WEBHOOK_URL=http://webhook
+MATRIX_WEBHOOK_API_KEY=api_key

techbloc_airflow/scripts/backups/techbloc-backup.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+
+import logging
+import os
+import socket
+
+import boto3
+import requests
+
+
+FILENAMES = [f"{name}-backup.tar.bz2" for name in ["matrix", "openoversight"]]
+DOWNLOAD_DIR = os.getenv("DOWNLOAD_DIR")
+SPACES_BUCKET_NAME = os.getenv("SPACES_BUCKET_NAME")
+SPACES_BASE_PREFIX = "monolith-backups"
+MATRIX_WEBHOOK_URL = os.getenv("MATRIX_WEBHOOK_URL")
+MATRIX_WEBHOOK_API_KEY = os.getenv("MATRIX_WEBHOOK_API_KEY")
+
+
+log = logging.getLogger(__name__)
+
+
+def get_s3_client():
+    return boto3.client(
+        "s3",
+        aws_access_key_id=os.getenv("SPACES_ACCESS_KEY"),
+        aws_secret_access_key=os.getenv("SPACES_SECRET_KEY"),
+        endpoint_url="https://sfo3.digitaloceanspaces.com",
+        region_name="sfo3",
+    )
+
+
+def alert_matrix(filename: str) -> None:
+    text = f"⬇ Backup on `{socket.gethostname()}` for `{filename}` complete."
+    log.info(f"Sending message to Matrix: {text}")
+    response = requests.post(
+        MATRIX_WEBHOOK_URL,
+        json={
+            "key": MATRIX_WEBHOOK_API_KEY,
+            "body": text,
+        },
+    )
+    response.raise_for_status()
+
+
+def download_files() -> None:
+    client = get_s3_client()
+    for filename in FILENAMES:
+        key = f"{SPACES_BASE_PREFIX}/{filename}"
+        download_path = f"{DOWNLOAD_DIR}/{filename}"
+        log.info(f"Downloading {filename} to {download_path}")
+
+        client.download_file(SPACES_BUCKET_NAME, key, download_path)
+        alert_matrix(filename)
+    log.info(f"Downloaded {len(FILENAMES)} files")
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        format="[%(asctime)s - %(name)s - %(lineno)3d][%(levelname)s] %(message)s",
+        level=logging.INFO,
+    )
+    download_files()

techbloc_airflow/scripts/backups/techbloc-backup.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=TechBloc Backup Service
+
+[Service]
+Type=oneshot
+ExecStart=/opt/techbloc/venv/bin/python3 /opt/techbloc/techbloc-backup.py
+EnvironmentFile=/opt/techbloc/techbloc-backup.env

techbloc_airflow/scripts/backups/techbloc-backup.timer

@@ -0,0 +1,9 @@
+[Unit]
+Description=TechBloc Backup Service (weekly)
+
+[Timer]
+OnCalendar=Sun *-*-* 05:00:00
+Persistent=true
+
+[Install]
+WantedBy=timers.target