Allow setting a common bucket for the environment (#90)

Author: AetherUnbound

.env.template

@@ -45,6 +45,8 @@ AIRFLOW_VAR_MATRIX_WEBHOOK_API_KEY=api_key
 
 S3_LOCAL_ENDPOINT=http://s3:5000
 SPACES_CONN_ID=aws_default
+# Optional bucket to use, necessary prefix in all cases for DigitalOcean Spaces
+# SPACES_BUCKET_NAME=bucketname
 
 ########################################################################################
 # Other config

techbloc_airflow/dags/constants.py

@@ -3,6 +3,11 @@
 
 SSH_MONOLITH_CONN_ID = "ssh_monolith"
 SPACES_CONN_ID = os.getenv("SPACES_CONN_ID")
+SPACES_BUCKET_NAME = os.getenv("SPACES_BUCKET_NAME", "")
+# DigitalOcean Spaces has a common shared bucket for everything, whereas locally we
+# have a separate bucket for each service. If the bucket name is defined, all keys
+# will be prefixed with that.
+SPACES_KEY_PREFIX = f"s3://{SPACES_BUCKET_NAME}/" if SPACES_BUCKET_NAME else "s3://"
 
 MATRIX_WEBHOOK_CONN_ID = "matrix_webhook"
 MATRIX_WEBHOOK_API_KEY = "matrix_webhook_api_key"

techbloc_airflow/dags/maintenance/backups/offsite_backups.py

@@ -13,12 +13,13 @@
 
 
 LOCAL_BACKUPS_FOLDER = "/opt/backups"
-BUCKET_NAME = "monolith-backups"
+KEY_BASE = constants.SPACES_KEY_PREFIX + "monolith-backups"
 
 
 @task_group
 def backup_service(config: OffsiteConfig):
     local_backup = f"{LOCAL_BACKUPS_FOLDER}/{config.filename}"
+    s3_key = f"{KEY_BASE}/{config.filename}"
 
     backup = SSHOperator(
         task_id=f"backup_{config.name}",
@@ -35,17 +36,14 @@ def backup_service(config: OffsiteConfig):
     copy_to_spaces = LocalFilesystemToS3Operator(
         task_id=f"copy_local_{config.name}_to_spaces",
         aws_conn_id=constants.SPACES_CONN_ID,
-        dest_bucket=BUCKET_NAME,
-        dest_key=config.filename,
+        dest_key=s3_key,
         replace=True,
         filename=local_backup,
     )
 
     @task(retries=3, retry_exponential_backoff=True)
     def notify_backup_complete():
-        matrix.send_message(
-            f"{config.name} backup complete at: `s3://{BUCKET_NAME}/{config.filename}`"
-        )
+        matrix.send_message(f"{config.name} backup complete at: `{s3_key}`")
 
     restart_if_failed = SSHOperator(
         task_id=f"start_if_failed_{config.name}",