add checksum validation

Author: NathanEne

backup_and_restore_database/backup_database/.env.example

@@ -4,6 +4,7 @@ DB_NAME=
 DB_USER=
 DB_PASSWORD=
 
+AWS_ENDPOINT_URL=
 AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_DEFAULT_REGION=ca-central-1

backup_and_restore_database/backup_database/Dockerfile

@@ -19,18 +19,23 @@ RUN apt-get update && \
     apt-get install -y \
         libldap-2.5-0 \
         libsasl2-2 \
+        curl unzip groff less \
         && \
-    pip install --no-cache-dir awscli && \
     rm -rf /var/lib/apt/lists/* && \
     apt-get clean
 
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install && \
+    rm -rf awscliv2.zip aws
+
 WORKDIR /usr/src/app
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1 \
     AWS_DEFAULT_REGION=ca-central-1 \
-    AWS_CLI_AUTO_PROMPT=off
-
+    AWS_RESPONSE_CHECKSUM_VALIDATION=when_required \
+    AWS_REQUEST_CHECKSUM_CALCULATION=when_required
 # Create and switch to non-root user
 RUN useradd -m appuser && \
     chown -R appuser:appuser /usr/src/app

backup_and_restore_database/backup_database/README.md

@@ -28,7 +28,7 @@ This directory contains a script and Docker setup to back up a PostgreSQL databa
 `docker build -t backup-to-s3 `
 
 2. Run the container:
-`docker run --env-file .env backup-to-s3`
+`docker run --rm --env-file .env backup-to-s3`
 
 ## How It Works
 1. The script uses pg_dump to create a PostgreSQL database dump in a custom binary format (-Fc).

backup_and_restore_database/backup_database/backup_and_upload_to_s3.py

@@ -8,10 +8,9 @@
 DB_NAME = os.environ["DB_NAME"]
 DB_USER = os.environ["DB_USER"]
 DB_PASSWORD = os.environ["DB_PASSWORD"]
+ENDPOINT_URL = os.environ["AWS_ENDPOINT_URL"]
 S3_PATH = f"s3://{os.getenv('BUCKET_NAME')}/{date.today()}-{DB_NAME}.dump"
-
 os.environ["PGPASSWORD"] = DB_PASSWORD
-
 pg_dump_cmd = [
     "pg_dump",
     "-h", DB_HOST,
@@ -22,24 +21,29 @@
     "-v"
 ]
 
-aws_cp_cmd = ["aws", "s3", "cp", "-", S3_PATH, "--only-show-errors"]
+aws_cp_cmd = [
+    "aws", "s3", "cp", "-", S3_PATH,
+    "--endpoint-url", ENDPOINT_URL,
+    "--debug",
+    "--expected-size", "60000000000"
+]
 
 print("Streaming dump into S3 bucket...")
 
-dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+dump_proc = subprocess.Popen(pg_dump_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 
-aws_proc = subprocess.Popen(aws_cp_cmd, stdin=dump_proc.stdout, stderr=subprocess.PIPE, text=True)
+aws_proc = subprocess.Popen(aws_cp_cmd, stdin=dump_proc.stdout, stderr=subprocess.PIPE)
 
 dump_proc.stdout.close()
 
 for line in dump_proc.stderr:
-    print(f"{datetime.now()} {line.strip()}")
+    print(f"{datetime.now()} {line.decode().strip()}")
 
 # Wait for both to finish and get final stderr from aws
 _, aws_stderr = aws_proc.communicate()
 
 if aws_proc.returncode != 0:
     print("[aws s3 cp] ERROR:")
-    print(aws_stderr)
+    print(aws_stderr.decode())
 if dump_proc.wait() != 0:
     print("pg_dump exited with a non-zero status.")