Enhance Rclone configuration with dynamic remote setup and update documentation

John Rogers · John Rogers · commit 1ba5ef32d3eb · 2025-04-01T15:04:07.000+01:00
diff --git a/.env.example b/.env.example
@@ -27,20 +27,53 @@ PGADMIN_EMAIL=your_pgadmin_email@example.com
 PGADMIN_PASSWORD=your_pgadmin_password
 
 # --- Backup Agent Configuration (Rclone & Retention) ---
-# Name of the rclone remote configured in your rclone.conf (e.g., "mygdrive", "s3remote")
-# Leave blank to disable cloud upload.
+# --- Backup Agent: Active Rclone Destination ---
+# Name of the rclone remote (configured below or manually in rclone.conf) to use for uploads.
+# This remote MUST be defined either via the RCLONE_REMOTE_<N> variables below OR manually in rclone_config/rclone.conf
+# Leave blank to disable cloud upload for this backup run.
 RCLONE_REMOTE_NAME=
 
-# Path within the rclone remote where backups should be stored (e.g., "resolve_backups/production")
-# Leave blank to disable cloud upload.
+# Path within the *active* rclone remote where backups should be stored (e.g., "resolve_backups/production")
+# Leave blank if RCLONE_REMOTE_NAME is blank.
 RCLONE_REMOTE_PATH=
 
 # Number of days to keep local backups in the ./backups volume (default is 7 if not set)
 BACKUP_RETENTION_DAYS=7
 
-# --- Optional: Google Drive Service Account for Rclone ---
-# If RCLONE_REMOTE_NAME is set and you want to use Google Drive via Service Account:
-# Paste the *entire content* of your Google Cloud Service Account JSON key file here.
-# Ensure it's enclosed in single quotes if it contains special characters, or handle quoting appropriately.
-# Leave blank if not using Service Account or if rclone.conf is manually configured.
-RCLONE_SERVICE_ACCOUNT_CREDENTIALS=
+# --- Optional: Dynamic Rclone Remote Configuration ---
+# The entrypoint script can dynamically configure rclone remotes based on these variables.
+# Define remotes sequentially starting with N=1 (RCLONE_REMOTE_1_...).
+# The entrypoint will stop looking when it doesn't find RCLONE_REMOTE_<N>_NAME.
+
+# --- Example 1: Google Drive using Service Account ---
+# RCLONE_REMOTE_1_NAME=my_google_drive_backup
+# RCLONE_REMOTE_1_TYPE=drive
+# # Paste the *entire content* of your Google Cloud Service Account JSON key file below.
+# # Ensure it's enclosed in single quotes if it contains special characters, or handle quoting appropriately.
+# RCLONE_REMOTE_1_PARAM_SERVICE_ACCOUNT_CREDENTIALS='{ "type": "service_account", ... }'
+# # Optional: Specify a Team Drive ID if needed
+# # RCLONE_REMOTE_1_PARAM_TEAM_DRIVE=YOUR_TEAM_DRIVE_ID
+
+# --- Example 2: AWS S3 ---
+# RCLONE_REMOTE_2_NAME=my_s3_backup
+# RCLONE_REMOTE_2_TYPE=s3
+# RCLONE_REMOTE_2_PARAM_PROVIDER=AWS # Or other S3-compatible provider
+# RCLONE_REMOTE_2_PARAM_ACCESS_KEY_ID=YOUR_AWS_ACCESS_KEY_ID
+# RCLONE_REMOTE_2_PARAM_SECRET_ACCESS_KEY=YOUR_AWS_SECRET_ACCESS_KEY
+# RCLONE_REMOTE_2_PARAM_REGION=us-east-1 # Or your desired AWS region
+# # Optional: Specify storage class, ACL, etc.
+# # RCLONE_REMOTE_2_PARAM_STORAGE_CLASS=STANDARD_IA
+# # RCLONE_REMOTE_2_PARAM_ACL=private
+
+# --- Example 3: Backblaze B2 ---
+# RCLONE_REMOTE_3_NAME=my_b2_backup
+# RCLONE_REMOTE_3_TYPE=b2
+# RCLONE_REMOTE_3_PARAM_ACCOUNT=YOUR_B2_ACCOUNT_ID_OR_APPLICATION_KEY_ID
+# RCLONE_REMOTE_3_PARAM_KEY=YOUR_B2_APPLICATION_KEY
+# # Optional: Specify endpoint if needed (e.g., for specific regions)
+# # RCLONE_REMOTE_3_PARAM_ENDPOINT=s3.us-west-000.backblazeb2.com
+
+# --- Example 4: Local Filesystem (useful for testing) ---
+# RCLONE_REMOTE_4_NAME=local_test_backup
+# RCLONE_REMOTE_4_TYPE=local
+# # No parameters usually needed for 'local' type unless specifying nounc, case_sensitive etc.
diff --git a/.gitignore b/.gitignore
@@ -33,4 +33,7 @@ Thumbs.db
 *.swo
 
 # Coverage
-/backups
+/backups
+
+#plan
+PLAN_rclone_config.md
diff --git a/README.md b/README.md
@@ -27,7 +27,7 @@ graph TD
         BackupAgent["laterbase-backup-agent<br>(Hourly pg_dump, rclone upload, retention)"]
         PgAdminUI[("laterbase-pgadmin<br>pgAdmin 4 UI")]
         BackupVolume["Local Backups backups"]
-        RcloneConfigVolume[Rclone Config<br>./rclone_config]
+        RcloneConfigVolume[Rclone Config<br>./rclone_config<br>(Dynamically Generated)]
     end
 
     subgraph "Cloud Storage (Optional)"
@@ -65,9 +65,15 @@ graph TD
     *   Replace `YOUR_PGADMIN_PASSWORD_HERE` with the password you want for the pgAdmin login.
     *   Adjust `PRIMARY_PORT` or `PRIMARY_USER` if they differ from the defaults (5432, postgres).
     *   Optionally, uncomment and set `POSTGRES_USER`, `POSTGRES_DB`, or `PGDATA` under the "Standby Server Configuration" section to override the defaults used by the standby service.
-    *   **Rclone Upload (Optional):**
-        *   Set `RCLONE_REMOTE_NAME` to the name of the remote you have configured in your `rclone.conf` file (e.g., `mygdrive`, `s3remote`). Leave blank to disable uploads.
-        *   Set `RCLONE_REMOTE_PATH` to the directory path within your rclone remote where backups should be stored (e.g., `resolve_backups/production`). Leave blank to disable uploads.
+    *   **Backup Agent - Active Rclone Destination (Optional):**
+        *   Set `RCLONE_REMOTE_NAME` to the name of the rclone remote you want the backup agent to **use** for uploads (e.g., `my_google_drive_backup`, `my_s3_backup`). This remote must be defined either dynamically (see next section) or manually in `./rclone_config/rclone.conf`. Leave blank to disable cloud uploads.
+        *   Set `RCLONE_REMOTE_PATH` to the directory path within the *active* rclone remote where backups should be stored (e.g., `resolve_backups/production`). Leave blank if `RCLONE_REMOTE_NAME` is blank.
+    *   **Backup Agent - Dynamic Rclone Remote Configuration (Optional):**
+        *   Instead of manually creating `./rclone_config/rclone.conf`, you can define remotes directly in the `.env` file using a specific format. The backup agent's entrypoint script will automatically create the necessary configuration inside the container when it starts.
+        *   Use the `RCLONE_REMOTE_<N>_...` variables as shown in `.env.example` (where `N` is a number starting from 1).
+        *   Define `RCLONE_REMOTE_<N>_NAME` (e.g., `my_s3_backup`) and `RCLONE_REMOTE_<N>_TYPE` (e.g., `s3`).
+        *   Add provider-specific parameters using `RCLONE_REMOTE_<N>_PARAM_<KEY>` (e.g., `RCLONE_REMOTE_1_PARAM_ACCESS_KEY_ID=...`). The `<KEY>` should be the lowercase version of the parameter name rclone expects (e.g., `access_key_id`, `secret_access_key`, `service_account_credentials`). Refer to `rclone config create --help` or the rclone documentation for specific provider parameters.
+        *   See `.env.example` for detailed examples for Google Drive, S3, B2, etc.
     *   **Backup Retention (Optional):**
         *   Set `BACKUP_RETENTION_DAYS` to the number of days you want to keep local backups in the `./backups` directory. Defaults to 7 if not set.
 
@@ -125,14 +131,26 @@ graph TD
         mkdir backups
         ```
 
-4.  **Rclone Configuration (Optional):**
-    *   If you want to use the cloud upload feature:
-        *   Create a directory named `rclone_config` in the same directory as `docker-compose.yml`:
+4.  **Rclone Configuration (Optional - Choose One Method):**
+
+    *   **Method A: Dynamic Configuration via `.env` (Recommended)**
+        *   Define your desired rclone remotes directly in the `.env` file using the `RCLONE_REMOTE_<N>_...` variables as described in the `.env` File section above and shown in `.env.example`.
+        *   The `laterbase-backup-agent` container will automatically generate the necessary `/config/rclone.conf` file internally when it starts based on these environment variables.
+        *   You still need to create the host directory for persistence, although the file inside will be managed by the container:
+            ```bash
+            mkdir rclone_config
+            ```
+        *   Ensure the `RCLONE_REMOTE_NAME` variable in `.env` matches one of the `RCLONE_REMOTE_<N>_NAME` values you defined.
+
+    *   **Method B: Manual `rclone.conf` File**
+        *   If you prefer to manage the `rclone.conf` file manually or have complex configurations not easily represented by environment variables:
+        *   Create the directory:
             ```bash
             mkdir rclone_config
             ```
-        *   Place your configured `rclone.conf` file inside the `./rclone_config/` directory. The backup container will mount this directory read-only at `/config/rclone/rclone.conf`. (Note: path inside container updated)
-        *   Ensure the remote name you use in `rclone.conf` matches the `RCLONE_REMOTE_NAME` set in your `.env` file.
+        *   Place your fully configured `rclone.conf` file inside `./rclone_config/`.
+        *   **Important:** If using this method, **do not** set any `RCLONE_REMOTE_<N>_...` variables in your `.env` file, as the entrypoint script might overwrite or conflict with your manual configuration.
+        *   Ensure the remote name you want to use matches the `RCLONE_REMOTE_NAME` set in your `.env` file.
         *   **Security Note:** The `rclone.conf` file contains sensitive credentials. Ensure appropriate file permissions are set on the host machine (`chmod 600 ./rclone_config/rclone.conf`).
 
 ## Usage
@@ -172,7 +190,7 @@ graph TD
 *   `app/prepare_primary_db.sh`: **(New)** Script to automate granting replication role and creating the replication slot on the primary server via `psql`. Run manually before starting services.
 *   `app/setup_standby.sh`: Script run inside the standby container on first start to perform the initial base backup and configure replication.
 *   `backup/backup.sh`: Script run by cron inside the backup agent container to perform hourly `pg_dump` backups, optionally upload via rclone, and manage retention.
-*   `backup/entrypoint.sh`: Entrypoint for backup container, sets up cron.
+*   `backup/entrypoint.sh`: Entrypoint for backup container. Dynamically configures rclone based on `RCLONE_REMOTE_<N>_...` environment variables (if present) before starting the cron daemon (managed by Ofelia).
 *   `backup/crontab.txt`: Defines the cron schedule for `backup.sh`.
 *   `./backups/` (Directory to be created): Host directory where local backup files (`.sql.gz`) will be stored by the backup agent.
 *   `./rclone_config/` (Directory to be created, optional): Host directory containing the `rclone.conf` file for cloud uploads.
diff --git a/backup/entrypoint.sh b/backup/entrypoint.sh
@@ -11,36 +11,80 @@ mkdir -p /config
 touch "$CONFIG_FILE"
 # Logging will go to stdout/stderr now
 
-# Check if remote name and credentials are provided
-if [ -n "$RCLONE_REMOTE_NAME" ] && [ -n "$SERVICE_ACCOUNT_CREDS" ]; then
-    echo "$(date): RCLONE_SERVICE_ACCOUNT_CREDENTIALS detected. Attempting to configure rclone remote '$RCLONE_REMOTE_NAME'." >&1
+# --- Dynamic Rclone Configuration Loop ---
+echo "$(date): Starting dynamic rclone configuration..." >&1
+N=1
+while true; do
+    # Construct variable names for the Nth remote
+    name_var_name="RCLONE_REMOTE_${N}_NAME"
+    type_var_name="RCLONE_REMOTE_${N}_TYPE"
 
-    # Check if the remote already exists in the config file
-    # Use rclone config show to check for a specific remote
-    if ! rclone config show "$RCLONE_REMOTE_NAME:" --config "$CONFIG_FILE" > /dev/null 2>&1; then
-        echo "$(date): Remote '$RCLONE_REMOTE_NAME' not found in $CONFIG_FILE. Creating..." >&1
+    # Use indirect expansion to get the values
+    remote_name="${!name_var_name}"
+    remote_type="${!type_var_name}"
 
-        # Create the remote using service account credentials
-        # Note: rclone expects the credentials directly, not a file path here
+    # If NAME is not set, we've processed all defined remotes
+    if [ -z "$remote_name" ]; then
+        echo "$(date): No more RCLONE_REMOTE_${N}_NAME found. Configuration loop finished." >&1
+        break
+    fi
+
+    # TYPE is mandatory if NAME is set
+    if [ -z "$remote_type" ]; then
+        echo "$(date): Error: RCLONE_REMOTE_${N}_NAME ('$remote_name') is set, but RCLONE_REMOTE_${N}_TYPE is missing. Skipping remote ${N}." >&2
+        N=$((N + 1))
+        continue
+    fi
+
+    echo "$(date): Processing remote ${N}: Name='$remote_name', Type='$remote_type'" >&1
+
+    # Check if the remote already exists
+    if rclone config show "$remote_name:" --config "$CONFIG_FILE" > /dev/null 2>&1; then
+        echo "$(date): Remote '$remote_name' already exists in $CONFIG_FILE. Skipping creation." >&1
+    else
+        # Remote does not exist, proceed with creation
+        echo "$(date): Remote '$remote_name' not found in $CONFIG_FILE. Creating..." >&1
+        declare -a rclone_params=() # Use an array for parameters
+
+        # Find all parameters for the current remote N
+        param_prefix="RCLONE_REMOTE_${N}_PARAM_"
+        # Use process substitution with env and grep for portability
+        while IFS='=' read -r env_var_full env_var_value; do
+            if [ -n "$env_var_value" ]; then # Ensure value is not empty
+                # Extract the rclone key part (e.g., SERVICE_ACCOUNT_CREDENTIALS)
+                rclone_key_upper=${env_var_full#"$param_prefix"}
+                # Convert to lowercase for rclone config command
+                rclone_key_lower=$(echo "$rclone_key_upper" | tr '[:upper:]' '[:lower:]')
+                # Add the key=value pair to the array
+                rclone_params+=("${rclone_key_lower}=${env_var_value}")
+            fi
+        done < <(env | grep "^${param_prefix}")
+
+        # Check if any parameters were found
+        if [ ${#rclone_params[@]} -eq 0 ]; then
+            echo "$(date): Warning: No parameters (RCLONE_REMOTE_${N}_PARAM_*) found for remote '$remote_name'. Creating with type only." >&2
+        fi
+
+        # Construct and execute the command
+        echo "$(date): Running: rclone config create \"$remote_name\" \"$remote_type\" [params...] --config \"$CONFIG_FILE\"" >&1
         rclone config create \
-            "$RCLONE_REMOTE_NAME" \
-            drive \
-            scope=drive \
-            service_account_credentials="$SERVICE_ACCOUNT_CREDS" \
+            "$remote_name" \
+            "$remote_type" \
+            "${rclone_params[@]}" \
             --config "$CONFIG_FILE"
 
+        # Check exit status
         if [ $? -eq 0 ]; then
-            echo "$(date): Successfully created rclone remote '$RCLONE_REMOTE_NAME' using Service Account." >&1
+            echo "$(date): Successfully created rclone remote '$remote_name' (type: $remote_type)." >&1
         else
-            echo "$(date): Error: Failed to create rclone remote '$RCLONE_REMOTE_NAME' using Service Account. Check credentials and permissions." >&2
+            echo "$(date): Error: Failed to create rclone remote '$remote_name' (type: $remote_type). Check parameters and permissions." >&2
             # Consider exiting if creation fails: exit 1
         fi
-    else
-        echo "$(date): Rclone remote '$RCLONE_REMOTE_NAME' already exists in $CONFIG_FILE. Skipping creation." >&1
-    fi
-else
-    echo "$(date): RCLONE_SERVICE_ACCOUNT_CREDENTIALS not set or RCLONE_REMOTE_NAME is empty. Skipping dynamic rclone configuration." >&1
-fi
+    fi # End if remote exists check
+
+    # Move to the next potential remote
+    N=$((N + 1))
+done
 
 echo "$(date): Entrypoint configuration check finished." >&1
 exit 0
