Fix GCS bucket dependency and improve Azure test script

moizpgedge · moizpgedge · commit 52d9b23b66a1 · 2025-12-03T19:04:00.000+05:00
- Fix repo-gcs-bucket to depend directly on repo-type=gcs instead of repo-gcs-key-type
  This prevents repo-gcs-bucket from being incorrectly required when using other
  storage types (posix, s3, azure, sftp)
- Regenerate parse.auto.c.inc with corrected dependency
- Improve Azure blob storage test script:
  - Add PostgreSQL path detection and config update (matching AMI test)
  - Add recovery mode detection and automatic promotion
  - Add repo2 configuration verification
  - Add archiving check before backups
  - Improve volume cleanup with force flag
diff --git a/src/build/config/config.yaml b/src/build/config/config.yaml
@@ -2223,7 +2223,9 @@ option:
     group: repo
     command: repo-type
     depend:
-      option: repo-gcs-key-type
+      option: repo-type
+      list:
+        - gcs
 
   repo-gcs-endpoint:
     section: global
diff --git a/src/config/parse.auto.c.inc b/src/config/parse.auto.c.inc
@@ -6164,7 +6164,8 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
             (                                                                                                 // opt/repo-gcs-bucket
                 PARSE_RULE_OPTIONAL_DEPEND                                                                    // opt/repo-gcs-bucket
                 (                                                                                             // opt/repo-gcs-bucket
-                    PARSE_RULE_VAL_OPT(RepoGcsKeyType),                                                       // opt/repo-gcs-bucket
+                    PARSE_RULE_VAL_OPT(RepoType),                                                             // opt/repo-gcs-bucket
+                    PARSE_RULE_VAL_STRID(Gcs),                                                                // opt/repo-gcs-bucket
                 ),                                                                                            // opt/repo-gcs-bucket
             ),                                                                                                // opt/repo-gcs-bucket
         ),                                                                                                    // opt/repo-gcs-bucket
@@ -6242,7 +6243,8 @@ static const ParseRuleOption parseRuleOption[CFG_OPTION_TOTAL] =
             (                                                                                               // opt/repo-gcs-endpoint
                 PARSE_RULE_OPTIONAL_DEPEND                                                                  // opt/repo-gcs-endpoint
                 (                                                                                           // opt/repo-gcs-endpoint
-                    PARSE_RULE_VAL_OPT(RepoGcsKeyType),                                                     // opt/repo-gcs-endpoint
+                    PARSE_RULE_VAL_OPT(RepoType),                                                           // opt/repo-gcs-endpoint
+                    PARSE_RULE_VAL_STRID(Gcs),                                                              // opt/repo-gcs-endpoint
                 ),                                                                                          // opt/repo-gcs-endpoint
                                                                                                             // opt/repo-gcs-endpoint
                 PARSE_RULE_OPTIONAL_DEFAULT                                                                 // opt/repo-gcs-endpoint
@@ -11732,6 +11734,8 @@ static const uint8_t optionResolveOrder[] =
     cfgOptRepoBlockSizeSuper,                                                                                   // opt-resolve-order
     cfgOptRepoBlockSizeSuperFull,                                                                               // opt-resolve-order
     cfgOptRepoCipherPass,                                                                                       // opt-resolve-order
+    cfgOptRepoGcsBucket,                                                                                        // opt-resolve-order
+    cfgOptRepoGcsEndpoint,                                                                                      // opt-resolve-order
     cfgOptRepoGcsKeyType,                                                                                       // opt-resolve-order
     cfgOptRepoHost,                                                                                             // opt-resolve-order
     cfgOptRepoHostConfig,                                                                                       // opt-resolve-order
@@ -11777,8 +11781,6 @@ static const uint8_t optionResolveOrder[] =
     cfgOptPgHostKeyFile,                                                                                        // opt-resolve-order
     cfgOptPgHostPort,                                                                                           // opt-resolve-order
     cfgOptRepoAzureKey,                                                                                         // opt-resolve-order
-    cfgOptRepoGcsBucket,                                                                                        // opt-resolve-order
-    cfgOptRepoGcsEndpoint,                                                                                      // opt-resolve-order
     cfgOptRepoGcsKey,                                                                                           // opt-resolve-order
     cfgOptRepoHostCaFile,                                                                                       // opt-resolve-order
     cfgOptRepoHostCaPath,                                                                                       // opt-resolve-order
diff --git a/test/azure/azure-pgbackrest.sh b/test/azure/azure-pgbackrest.sh
@@ -195,7 +195,9 @@ cleanup_docker() {
     docker rm "$CONTAINER" 2>/dev/null || true
     
     echo "Removing volumes..."
-    docker volume rm pgdata pgrepo 2>/dev/null || true
+    docker volume rm -f pgdata pgrepo 2>/dev/null || true
+    # Wait a moment to ensure volumes are fully removed
+    sleep 1
     
     # Check and free up port 5433 if in use
     if lsof -ti:5433 >/dev/null 2>&1; then
@@ -629,6 +631,25 @@ test_azure_blob() {
     echo ""
     print_success "PostgreSQL is ready (took ${ELAPSED}s)"
     
+    # Check if PostgreSQL is in recovery mode and promote if needed
+    RECOVERY_STATUS=$(docker exec "$CONTAINER" psql -U postgres -d postgres -t -c "SELECT pg_is_in_recovery();" 2>/dev/null | xargs || echo "t")
+    if [ "$RECOVERY_STATUS" = "t" ]; then
+        print_warning "PostgreSQL is in recovery mode, promoting..."
+        docker exec "$CONTAINER" psql -U postgres -d postgres -c "SELECT pg_wal_replay_resume();" >/dev/null 2>&1 || true
+        sleep 2
+        # Wait for PostgreSQL to be ready again after promotion
+        ELAPSED=0
+        until docker exec "$CONTAINER" pg_isready -U postgres >/dev/null 2>&1; do
+            if [ $ELAPSED -ge 30 ]; then
+                print_error "PostgreSQL failed to start after promotion"
+                return 1
+            fi
+            sleep 1
+            ELAPSED=$((ELAPSED + 1))
+        done
+        print_success "PostgreSQL promoted successfully"
+    fi
+    
     # Create test data (this will also download Northwind if needed)
     create_test_data "$CONTAINER"
     
@@ -672,6 +693,170 @@ test_azure_blob() {
     # Clean local backups
     docker exec "$CONTAINER" rm -rf /var/lib/pgbackrest/archive/* /var/lib/pgbackrest/backup/* 2>/dev/null || true
     
+    # Debug: Check PostgreSQL data directory and pgBackRest config
+    print_header "Debug: Check PostgreSQL Paths"
+    echo "PGDATA environment:"
+    docker exec "$CONTAINER" bash -c 'echo "PGDATA=$PGDATA"'
+    echo ""
+    echo "PostgreSQL data directory locations:"
+    docker exec "$CONTAINER" bash -c 'ls -la /var/lib/postgresql/ 2>/dev/null || echo "Cannot list /var/lib/postgresql"'
+    echo ""
+    echo "pgBackRest config pg1-path:"
+    docker exec "$CONTAINER" bash -c 'grep "pg1-path" /etc/pgbackrest/pgbackrest.conf || echo "pg1-path not found in config"'
+    echo ""
+    echo "Actual PostgreSQL data directory (from postgres process):"
+    ACTUAL_DATA_DIR=$(docker exec "$CONTAINER" bash -c 'psql -U postgres -d postgres -t -c "SHOW data_directory;" 2>/dev/null | xargs' || echo "")
+    if [ -n "$ACTUAL_DATA_DIR" ]; then
+        echo "Found data directory: $ACTUAL_DATA_DIR"
+        echo ""
+        echo "Updating pgBackRest config with correct path..."
+        # Remove old pg1-path and add new one with correct directory
+        # Use /tmp for temp file and Python to avoid all permission issues
+        docker exec -e DATA_DIR="$ACTUAL_DATA_DIR" "$CONTAINER" bash -c "python3 <<'PYEOF'
+import sys
+import os
+
+data_dir = os.environ['DATA_DIR']
+config_file = '/etc/pgbackrest/pgbackrest.conf'
+tmp_file = '/tmp/pgbackrest.conf.tmp'
+
+# Read current config
+with open(config_file, 'r') as f:
+    lines = f.readlines()
+
+# Process lines: remove old pg1-path, add new one after [demo]
+output = []
+in_demo = False
+pg1_added = False
+
+for line in lines:
+    stripped = line.strip()
+    
+    # Track when we enter [demo] section
+    if stripped == '[demo]':
+        in_demo = True
+        output.append(line)
+        continue
+    
+    # Track when we leave [demo] section
+    if stripped.startswith('[') and stripped != '[demo]':
+        if in_demo and not pg1_added:
+            output.append('pg1-path=' + data_dir + '\n')
+            pg1_added = True
+        in_demo = False
+        output.append(line)
+        continue
+    
+    # Skip old pg1-path lines
+    if stripped.startswith('pg1-path='):
+        continue
+    
+    # Add pg1-path after [demo] when we hit first empty line or end
+    if in_demo and not pg1_added and (stripped == '' or lines.index(line) == len(lines) - 1):
+        output.append('pg1-path=' + data_dir + '\n')
+        pg1_added = True
+    
+    output.append(line)
+
+# If [demo] section exists but pg1-path was never added
+if in_demo and not pg1_added:
+    output.append('pg1-path=' + data_dir + '\n')
+
+# If [demo] section doesn't exist, add it
+if '[demo]' not in ''.join(output):
+    output.append('\n[demo]\n')
+    output.append('pg1-path=' + data_dir + '\n')
+
+# Write to temp file
+with open(tmp_file, 'w') as f:
+    f.writelines(output)
+
+# Copy back and set permissions
+import shutil
+import pwd
+import grp
+
+shutil.copy(tmp_file, config_file)
+postgres_uid = pwd.getpwnam('postgres').pw_uid
+postgres_gid = grp.getgrnam('postgres').gr_gid
+os.chown(config_file, postgres_uid, postgres_gid)
+os.chmod(config_file, 0o640)
+os.remove(tmp_file)
+PYEOF
+"
+        echo "Updated config:"
+        docker exec "$CONTAINER" bash -c 'grep "pg1-path" /etc/pgbackrest/pgbackrest.conf'
+    else
+        print_warning "Could not determine PostgreSQL data directory, using default"
+    fi
+    echo ""
+    
+    # Verify repo2 is configured
+    print_header "Verify Azure (repo2) Configuration"
+    if ! docker exec "$CONTAINER" grep -q "repo2-type=azure" /etc/pgbackrest/pgbackrest.conf; then
+        echo "repo2 not found in config, running configure-azure.sh..."
+        docker exec "$CONTAINER" bash -lc "/usr/local/bin/configure-azure.sh" || echo "configure-azure.sh returned error, will configure manually"
+        
+        # Check if it worked
+        if ! docker exec "$CONTAINER" grep -q "repo2-type=azure" /etc/pgbackrest/pgbackrest.conf; then
+            echo "configure-azure.sh didn't add repo2, configuring manually..."
+            docker exec -e AZURE_ACCOUNT="$AZURE_ACCOUNT" -e AZURE_CONTAINER="$AZURE_CONTAINER" -e AZURE_KEY_TYPE="$AZURE_KEY_TYPE" -e AZURE_REPO_PATH="$AZURE_REPO_PATH" "$CONTAINER" bash -c "
+                AZURE_REPO_PATH=\${AZURE_REPO_PATH:-/demo-repo}
+                TMP_FILE=/tmp/pgbackrest_repo2.$$
+                cat /etc/pgbackrest/pgbackrest.conf > \$TMP_FILE
+                echo '' >> \$TMP_FILE
+                echo 'repo2-type=azure' >> \$TMP_FILE
+                echo \"repo2-azure-account=\${AZURE_ACCOUNT}\" >> \$TMP_FILE
+                echo \"repo2-azure-container=\${AZURE_CONTAINER}\" >> \$TMP_FILE
+                echo \"repo2-azure-key-type=\${AZURE_KEY_TYPE}\" >> \$TMP_FILE
+                echo \"repo2-azure-key=\${AZURE_KEY}\" >> \$TMP_FILE
+                echo \"repo2-path=\${AZURE_REPO_PATH}\" >> \$TMP_FILE
+                echo 'repo2-retention-full=4' >> \$TMP_FILE
+                cp \$TMP_FILE /etc/pgbackrest/pgbackrest.conf
+                chown postgres:postgres /etc/pgbackrest/pgbackrest.conf
+                chmod 640 /etc/pgbackrest/pgbackrest.conf
+                rm -f \$TMP_FILE
+            "
+        fi
+        echo "Config after configuration:"
+        docker exec "$CONTAINER" grep -E "repo2|azure" /etc/pgbackrest/pgbackrest.conf || echo "No repo2 config found"
+    else
+        echo "repo2 configuration found:"
+        docker exec "$CONTAINER" grep -E "repo2|azure" /etc/pgbackrest/pgbackrest.conf
+    fi
+    echo ""
+    
+    # Ensure archiving is enabled before backups
+    print_header "Ensure PostgreSQL Archiving Enabled"
+    ARCHIVE_MODE=$(docker exec "$CONTAINER" psql -U postgres -At -c "show archive_mode")
+
+    if [ "$ARCHIVE_MODE" != "on" ]; then
+        echo "archive_mode is currently $ARCHIVE_MODE - enabling..."
+        docker exec "$CONTAINER" bash -lc '
+            set -e
+            PGDATA_DIR=${PGDATA:-/var/lib/postgresql/data}
+            {
+                echo ""
+                echo "# pgBackRest archiving configuration"
+                echo "archive_mode = on"
+                echo "archive_command = '\''pgbackrest --stanza=demo archive-push %p'\''"
+                echo "archive_timeout = 60"
+                echo "wal_level = replica"
+                echo "max_wal_senders = 3"
+                echo "max_replication_slots = 3"
+            } >> "$PGDATA_DIR/postgresql.conf"
+        '
+
+        docker restart "$CONTAINER" >/dev/null
+        echo "Waiting for PostgreSQL to restart with archiving enabled..."
+        until docker exec "$CONTAINER" pg_isready -U postgres >/dev/null 2>&1; do
+            sleep 1
+        done
+    else
+        echo "archive_mode already enabled"
+    fi
+    echo ""
+    
     # Create stanza
     print_header "Create Stanza (Azure Blob Storage)"
     docker exec "$CONTAINER" pgbackrest --stanza=demo stanza-create
@@ -727,10 +912,27 @@ test_azure_blob() {
       -v pgrepo:/var/lib/pgbackrest \
       "$IMAGE" \
       -lc "/usr/local/bin/configure-azure.sh || true; \
+           if ! grep -q \"repo2-type=azure\" /etc/pgbackrest/pgbackrest.conf; then \
+             AZURE_REPO_PATH=\${AZURE_REPO_PATH:-/demo-repo}; \
+             TMP_FILE=/tmp/pgbackrest_repo2.\$\$; \
+             cat /etc/pgbackrest/pgbackrest.conf > \$TMP_FILE; \
+             echo '' >> \$TMP_FILE; \
+             echo 'repo2-type=azure' >> \$TMP_FILE; \
+             echo \"repo2-azure-account=\${AZURE_ACCOUNT}\" >> \$TMP_FILE; \
+             echo \"repo2-azure-container=\${AZURE_CONTAINER}\" >> \$TMP_FILE; \
+             echo \"repo2-azure-key-type=\${AZURE_KEY_TYPE}\" >> \$TMP_FILE; \
+             echo \"repo2-azure-key=\${AZURE_KEY}\" >> \$TMP_FILE; \
+             echo \"repo2-path=\${AZURE_REPO_PATH}\" >> \$TMP_FILE; \
+             echo 'repo2-retention-full=4' >> \$TMP_FILE; \
+             cp \$TMP_FILE /etc/pgbackrest/pgbackrest.conf; \
+             chown postgres:postgres /etc/pgbackrest/pgbackrest.conf; \
+             chmod 640 /etc/pgbackrest/pgbackrest.conf; \
+             rm -f \$TMP_FILE; \
+           fi; \
            DATA_DIR=\${ACTUAL_DATA_DIR:-/var/lib/postgresql/data}; \
            echo \"Restoring to data directory: \$DATA_DIR\"; \
            rm -rf \"\$DATA_DIR\"/* && \
-           pgbackrest --stanza=demo restore --set='$BACKUP_LABEL_AZURE' --type=immediate --pg1-path=\"\$DATA_DIR\""
+           pgbackrest --repo=2 --stanza=demo restore --set='$BACKUP_LABEL_AZURE' --type=immediate --pg1-path=\"\$DATA_DIR\""
     
     print_success "Restore complete"
     
