Skip to content

Commit 2272925

Browse files
rafbgarciarafbgarcia
committed
fix: crunchybridge integration
1 parent 596a1ff commit 2272925

File tree

2 files changed

+87
-82
lines changed

2 files changed

+87
-82
lines changed

‎internal/server/restore_handlers.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -340,12 +340,20 @@ func (s *Server) applyAnonymization(c *gin.Context) {
340340
Str("restore_name", restore.Name).
341341
Msg("Manually triggering anonymization")
342342

343+
// Determine the target database name based on restore type:
344+
// - Crunchy Bridge: uses the database name from config (pgBackRest restores all databases)
345+
// - Logical: uses the source database name from connection string (restored to database with same name as source)
346+
targetDatabase := config.DatabaseName // Default for logical restores (extracted from connection string)
347+
if config.CrunchyBridgeAPIKey != "" {
348+
// Crunchy Bridge restore - use the configured database name
349+
targetDatabase = config.CrunchyBridgeDatabaseName
350+
}
351+
343352
// Apply anonymization rules
344-
pgPort := postgresVersionToPort(config.PostgresVersion)
345353
rulesApplied, err := anonymize.Apply(c.Request.Context(), s.db, anonymize.ApplyParams{
346-
DatabaseName: restore.Name,
354+
DatabaseName: targetDatabase,
347355
PostgresVersion: config.PostgresVersion,
348-
PostgresPort: pgPort,
356+
PostgresPort: restore.Port,
349357
}, s.logger)
350358
if err != nil {
351359
s.logger.Error().Err(err).Str("restore_id", restoreID).Msg("Failed to apply anonymization")

‎internal/workers/crunchy_bridge_restore.sh‎

Lines changed: 76 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -42,16 +42,18 @@ die() {
4242
fi
4343

4444
# Destroy ZFS dataset if it was created
45-
if sudo zfs list "${ZFS_DATASET}" >/dev/null 2>&1; then
46-
log "Destroying ZFS dataset..."
47-
sudo zfs destroy -r "${ZFS_DATASET}" 2>/dev/null || log "Warning: Could not destroy ZFS dataset"
48-
fi
45+
# if sudo zfs list "${ZFS_DATASET}" >/dev/null 2>&1; then
46+
# log "Destroying ZFS dataset..."
47+
# sudo zfs destroy -r "${ZFS_DATASET}" 2>/dev/null || log "Warning: Could not destroy ZFS dataset"
48+
# fi
49+
log "WARNING: ZFS dataset ${ZFS_DATASET} left intact for debugging (not destroyed)"
4950

5051
# Clean up pgBackRest config
51-
if [ -f "${PGBACKREST_CONF}" ]; then
52-
log "Cleaning up pgBackRest config..."
53-
rm -f "${PGBACKREST_CONF}" 2>/dev/null || log "Warning: Could not remove pgBackRest config"
54-
fi
52+
# if [ -f "${PGBACKREST_CONF}" ]; then
53+
# log "Cleaning up pgBackRest config..."
54+
# rm -f "${PGBACKREST_CONF}" 2>/dev/null || log "Warning: Could not remove pgBackRest config"
55+
# fi
56+
log "WARNING: pgBackRest config ${PGBACKREST_CONF} left intact for debugging (not deleted)"
5557

5658
# Write failure marker
5759
echo '__BRANCHD_RESTORE_FAILED__' >> "${RESTORE_LOG}"
@@ -129,80 +131,75 @@ sudo -u postgres cp /etc/postgresql-common/ssl/server.key "${DATA_DIR}/"
129131
sudo -u postgres chmod 0600 "${DATA_DIR}/server.key"
130132
sudo -u postgres chmod 0644 "${DATA_DIR}/server.crt"
131133

132-
# Replace postgresql.conf with a clean config optimized for ephemeral dev branches
133-
# This removes all Crunchy Bridge specific settings and production-oriented configs
134-
log "Creating clean postgresql.conf for dev branch..."
135-
sudo -u postgres tee "${DATA_DIR}/postgresql.conf" > /dev/null << 'EOF'
136-
# Connection Settings
137-
listen_addresses = '127.0.0.1'
138-
port = ${PG_PORT}
139-
max_connections = 1000
134+
# Backup original Crunchy Bridge config for reference
135+
sudo -u postgres cp "${DATA_DIR}/postgresql.conf" "${DATA_DIR}/postgresql.conf.crunchybridge"
136+
137+
log "Extracting recovery-critical parameters from conf.d before removing include_dir..."
138+
139+
# Extract recovery-critical parameters from conf.d files
140+
# These MUST be >= the primary server's values for recovery to succeed
141+
MAX_CONNECTIONS=$(sudo -u postgres grep -h "^max_connections" "${DATA_DIR}/conf.d/"*.conf 2>/dev/null | tail -1 | sed "s/.*=\s*['\"]*//" | sed "s/['\"].*//" || echo "100")
142+
MAX_WORKER_PROCESSES=$(sudo -u postgres grep -h "^max_worker_processes" "${DATA_DIR}/conf.d/"*.conf 2>/dev/null | tail -1 | sed "s/.*=\s*['\"]*//" | sed "s/['\"].*//" || echo "8")
143+
MAX_WAL_SENDERS=$(sudo -u postgres grep -h "^max_wal_senders" "${DATA_DIR}/conf.d/"*.conf 2>/dev/null | tail -1 | sed "s/.*=\s*['\"]*//" | sed "s/['\"].*//" || echo "10")
144+
MAX_PREPARED_XACTS=$(sudo -u postgres grep -h "^max_prepared_transactions" "${DATA_DIR}/conf.d/"*.conf 2>/dev/null | tail -1 | sed "s/.*=\s*['\"]*//" | sed "s/['\"].*//" || echo "0")
145+
MAX_LOCKS_PER_XACT=$(sudo -u postgres grep -h "^max_locks_per_transaction" "${DATA_DIR}/conf.d/"*.conf 2>/dev/null | tail -1 | sed "s/.*=\s*['\"]*//" | sed "s/['\"].*//" || echo "64")
146+
147+
log "Extracted parameters: max_connections=${MAX_CONNECTIONS}, max_worker_processes=${MAX_WORKER_PROCESSES}"
148+
149+
log "Modifying postgresql.conf for dev branch..."
150+
151+
# Remove problematic lines that would prevent startup
152+
sudo -u postgres sed -i \
153+
-e '/^include_dir/d' \
154+
-e '/^pgpodman\./d' \
155+
-e '/^pg_parquet\./d' \
156+
-e '/^cron\.use_background_workers/d' \
157+
-e '/^ssl_ca_file/d' \
158+
-e '/^archive_mode/d' \
159+
-e '/^archive_command/d' \
160+
-e '/^archive_timeout/d' \
161+
"${DATA_DIR}/postgresql.conf"
162+
163+
sudo -u postgres sed -i \
164+
"s/shared_preload_libraries = 'pgaudit,pgpodman,anon,pg_squeeze,pg_parquet,pg_cron,pg_stat_statements'/shared_preload_libraries = 'pgaudit,pg_stat_statements'/" \
165+
"${DATA_DIR}/postgresql.conf"
166+
167+
# Change log destination from syslog to stderr for easier debugging
168+
sudo -u postgres sed -i \
169+
"s/log_destination = 'syslog'/log_destination = 'stderr'/" \
170+
"${DATA_DIR}/postgresql.conf"
171+
172+
# Add log_directory since we changed from syslog
173+
if ! grep -q "^log_directory" "${DATA_DIR}/postgresql.conf"; then
174+
echo "log_directory = 'log'" | sudo -u postgres tee -a "${DATA_DIR}/postgresql.conf" > /dev/null
175+
fi
140176

141-
# TLS/SSL
142-
ssl = on
177+
# Disable archive mode for dev branches
178+
echo "archive_mode = off" | sudo -u postgres tee -a "${DATA_DIR}/postgresql.conf" > /dev/null
179+
180+
# Override network settings for local-only access (append at end to override any earlier settings)
181+
sudo -u postgres tee -a "${DATA_DIR}/postgresql.conf" > /dev/null << EOF
182+
183+
# Branchd overrides for dev branch
184+
port = ${PG_PORT}
185+
listen_addresses = '127.0.0.1'
143186
ssl_cert_file = 'server.crt'
144187
ssl_key_file = 'server.key'
145-
ssl_min_protocol_version = 'TLSv1.2'
146-
147-
# Authentication
148-
password_encryption = scram-sha-256
149-
150-
# Resource Limits
151-
# Note: These parameters are set high to support pgBackRest recovery from production backups.
152-
# PostgreSQL requires these values to be >= the source database during WAL replay.
153-
shared_buffers = 128MB
154-
work_mem = 8MB
155-
maintenance_work_mem = 64MB
156-
effective_cache_size = 512MB
157-
max_worker_processes = 200
158-
max_parallel_workers_per_gather = 2
159-
max_parallel_workers = 4
160-
max_prepared_transactions = 100
161-
max_locks_per_transaction = 256
162-
max_pred_locks_per_transaction = 256
163-
164-
# WAL Settings
165-
wal_level = logical
166-
max_wal_senders = 20
167-
max_replication_slots = 0
168-
max_wal_size = 512MB
169-
min_wal_size = 80MB
170-
171-
# Extensions
172-
# Note: pgaudit is included because Crunchy Bridge databases have it installed
173-
shared_preload_libraries = 'pg_stat_statements,pgaudit'
174-
175-
# Logging
176-
logging_collector = on
177-
log_destination = 'stderr'
178-
log_directory = 'log'
179-
log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
180-
log_rotation_age = 1d
181-
log_rotation_size = 100MB
182-
log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
183-
log_timezone = 'UTC'
184-
185-
# Performance
186-
random_page_cost = 1.1
187-
effective_io_concurrency = 200
188-
189-
# Statistics
190-
track_io_timing = on
191-
track_functions = pl
192-
193-
# Locale
194-
datestyle = 'iso, mdy'
195-
timezone = 'UTC'
196-
lc_messages = 'en_US.UTF-8'
197-
lc_monetary = 'en_US.UTF-8'
198-
lc_numeric = 'en_US.UTF-8'
199-
lc_time = 'en_US.UTF-8'
200-
default_text_search_config = 'pg_catalog.english'
188+
189+
# Recovery-critical parameters from Crunchy Bridge conf.d
190+
# These must be >= primary server values for recovery to succeed
191+
max_connections = ${MAX_CONNECTIONS}
192+
max_worker_processes = ${MAX_WORKER_PROCESSES}
193+
max_wal_senders = ${MAX_WAL_SENDERS}
194+
max_prepared_transactions = ${MAX_PREPARED_XACTS}
195+
max_locks_per_transaction = ${MAX_LOCKS_PER_XACT}
196+
197+
# Performance parameters optimized for dev environment
198+
shared_buffers = '128MB'
199+
huge_pages = try # Changed from 'on' - VM may not have huge pages configured
201200
EOF
202201

203-
# Substitute the port variable
204-
sudo -u postgres sed -i "s/\${PG_PORT}/${PG_PORT}/g" "${DATA_DIR}/postgresql.conf"
205-
log "postgresql.conf created"
202+
log "postgresql.conf configured"
206203

207204
# Configure pg_hba.conf for local access only
208205
sudo -u postgres tee "${DATA_DIR}/pg_hba.conf" > /dev/null << EOF
@@ -226,12 +223,12 @@ Requires=zfs-mount.service
226223
Type=forking
227224
User=postgres
228225
Group=postgres
229-
ExecStart=${PG_BIN}/pg_ctl start -D ${DATA_DIR} -l ${DATA_DIR}/postgresql.log
226+
ExecStart=${PG_BIN}/pg_ctl start -t 3600 -D ${DATA_DIR} -l ${DATA_DIR}/postgresql.log
230227
ExecStop=${PG_BIN}/pg_ctl stop -D ${DATA_DIR} -m fast
231228
ExecReload=${PG_BIN}/pg_ctl reload -D ${DATA_DIR}
232229
KillMode=mixed
233230
KillSignal=SIGINT
234-
TimeoutStartSec=300
231+
TimeoutStartSec=3600
235232
TimeoutStopSec=300
236233
Restart=on-failure
237234
RestartSec=5s

0 commit comments

Comments
 (0)