test: crunchybridge integration

Author: rafbgarcia

bin/e2e

@@ -7,18 +7,27 @@ set -euo pipefail
 # Hardcoded test database connection string (PostgreSQL 16 superuser)
 # It's a very small database with only a users table with 3 users.
 PG_16_CONN="postgres://postgres:hCazJJexWqQCzRghehroazRkj5nDbGMpW9JjjtoL5lBFPGi6auHbQ8jZHUZYEluO@p.mgdruzjwa5hk7elhmlnug4somq.db.postgresbridge.com:5432/quic_test"
+CRUNCHYBRIDGE_API_KEY="cbkey_16we5jO0whSk843bLuVa7jdni90TaB2lmdVw091SYUkZrn"
+CRUNCHYBRIDGE_CLUSTER_NAME="quic-e2e"
+CRUNCHYBRIDGE_DATABASE_NAME="quic_test"
 
 # Default PostgreSQL version
 PG_VERSION="${TEST_POSTGRES_VERSION:-16}"
 
 # Use hardcoded connection string if not set in environment
 export TEST_CONNECTION_STRING="${TEST_CONNECTION_STRING:-$PG_16_CONN}"
 
+# Export Crunchy Bridge credentials for e2e tests
+export CRUNCHYBRIDGE_API_KEY="${CRUNCHYBRIDGE_API_KEY}"
+export CRUNCHYBRIDGE_CLUSTER_NAME="${CRUNCHYBRIDGE_CLUSTER_NAME}"
+export CRUNCHYBRIDGE_DATABASE_NAME="${CRUNCHYBRIDGE_DATABASE_NAME}"
+
 echo "========================================="
 echo "Branchd E2E Test Runner"
 echo "========================================="
 echo "PostgreSQL version: ${PG_VERSION}"
 echo "Connection: ${TEST_CONNECTION_STRING%%@*}@***"
+echo "Crunchy Bridge cluster: ${CRUNCHYBRIDGE_CLUSTER_NAME}"
 echo ""
 
 # Check for Terraform variables
@@ -49,13 +58,6 @@ terraform init -upgrade >/dev/null
 terraform apply -auto-approve \
     -var "postgres_version=${PG_VERSION}"
 
-# Get VM details
-PUBLIC_IP=$(terraform output -raw public_ip)
-API_URL=$(terraform output -raw api_url)
-echo "VM provisioned: ${PUBLIC_IP}"
-echo "API URL: ${API_URL}"
-echo "Infrastructure setup complete (PostgreSQL, ZFS, Redis, Caddy)"
-
 cd ../..
 
 echo ""
@@ -66,14 +68,14 @@ export TEST_POSTGRES_VERSION="${PG_VERSION}"
 
 # Run tests with verbose output
 # BuildAndDeploy is called by the test itself
-go test ./tests/e2e -v -timeout 10m
+# Running only Crunchy Bridge test
+go test ./tests/e2e -v -timeout 5m -run TestCrunchyBridgeIntegration
 
 echo ""
 echo "========================================="
 echo "Tests completed successfully!"
 echo "========================================="
 echo ""
-echo "VM is still running at: ${PUBLIC_IP}"
 echo "To destroy the VM:"
 echo "  cd tests/terraform && terraform destroy -auto-approve"
 echo ""

internal/branches/create-branch.sh

@@ -282,12 +282,9 @@ echo "Cleaning up PostgreSQL files..."
 sudo -u postgres rm -f "${BRANCH_PGDATA}/postmaster.pid"
 sudo -u postgres rm -f "${BRANCH_PGDATA}/postgresql.auto.conf"
 
-# Copy config files from system location to data directory
-# Ubuntu's pg_createcluster stores configs in /etc/postgresql/, but branches need them in the data dir
-echo "Copying PostgreSQL config files to data directory..."
-sudo -u postgres cp "/etc/postgresql/${PG_VERSION}/main/postgresql.conf" "${BRANCH_PGDATA}/postgresql.conf"
-sudo -u postgres cp "/etc/postgresql/${PG_VERSION}/main/pg_hba.conf" "${BRANCH_PGDATA}/pg_hba.conf.bak"
-sudo -u postgres cp "/etc/postgresql/${PG_VERSION}/main/pg_ident.conf" "${BRANCH_PGDATA}/pg_ident.conf"
+# Config files already exist in the ZFS clone from the source restore
+# No need to copy - they were cloned from the restore's data directory
+echo "Using PostgreSQL config files from cloned restore..."
 
 # Minimal update to postgresql.conf for the new port.
 # Note: SSL configuration is inherited from the main cluster via ZFS clone

internal/server/anon_rules_handlers.go

@@ -14,7 +14,7 @@ import (
 type CreateAnonRuleRequest struct {
 	Table    string          `json:"table" binding:"required"`
 	Column   string          `json:"column" binding:"required"`
-	Template json.RawMessage `json:"template" binding:"required"`
+	Template json.RawMessage `json:"template" binding:"required" swaggertype:"string" example:"\"user_${index}@example.com\""`
 	Type     string          `json:"type"` // Optional: "text", "integer", "boolean", "null" - overrides auto-detection
 }
 

internal/server/config_handlers.go

@@ -25,30 +25,36 @@ type OnboardingDatabaseRequest struct {
 
 // ConfigResponse represents the configuration response
 type ConfigResponse struct {
-	ID                   string     `json:"id"`
-	ConnectionString     string     `json:"connection_string"`
-	PostgresVersion      string     `json:"postgres_version"`
-	SchemaOnly           bool       `json:"schema_only"`
-	RefreshSchedule      string     `json:"refresh_schedule"`
-	BranchPostgresqlConf string     `json:"branch_postgresql_conf"`
-	DatabaseName         string     `json:"database_name"`
-	Domain               string     `json:"domain"`
-	LetsEncryptEmail     string     `json:"lets_encrypt_email"`
-	MaxRestores          int        `json:"max_restores"`
-	LastRefreshedAt      *time.Time `json:"last_refreshed_at"`
-	NextRefreshAt        *time.Time `json:"next_refresh_at"`
-	CreatedAt            time.Time  `json:"created_at"`
+	ID                        string     `json:"id"`
+	ConnectionString          string     `json:"connection_string"`
+	PostgresVersion           string     `json:"postgres_version"`
+	SchemaOnly                bool       `json:"schema_only"`
+	RefreshSchedule           string     `json:"refresh_schedule"`
+	BranchPostgresqlConf      string     `json:"branch_postgresql_conf"`
+	DatabaseName              string     `json:"database_name"`
+	Domain                    string     `json:"domain"`
+	LetsEncryptEmail          string     `json:"lets_encrypt_email"`
+	MaxRestores               int        `json:"max_restores"`
+	LastRefreshedAt           *time.Time `json:"last_refreshed_at"`
+	NextRefreshAt             *time.Time `json:"next_refresh_at"`
+	CreatedAt                 time.Time  `json:"created_at"`
+	CrunchyBridgeAPIKey       string     `json:"crunchy_bridge_api_key"`
+	CrunchyBridgeClusterName  string     `json:"crunchy_bridge_cluster_name"`
+	CrunchyBridgeDatabaseName string     `json:"crunchy_bridge_database_name"`
 }
 
 // UpdateConfigRequest represents the request to update configuration
 type UpdateConfigRequest struct {
-	ConnectionString string `json:"connectionString"`
-	PostgresVersion  string `json:"postgresVersion"`
-	SchemaOnly       *bool  `json:"schemaOnly"`
-	RefreshSchedule  string `json:"refreshSchedule"`
-	Domain           string `json:"domain"`
-	LetsEncryptEmail string `json:"letsEncryptEmail"`
-	MaxRestores      *int   `json:"maxRestores"`
+	ConnectionString          string `json:"connectionString"`
+	PostgresVersion           string `json:"postgresVersion"`
+	SchemaOnly                *bool  `json:"schemaOnly"`
+	RefreshSchedule           string `json:"refreshSchedule"`
+	Domain                    string `json:"domain"`
+	LetsEncryptEmail          string `json:"letsEncryptEmail"`
+	MaxRestores               *int   `json:"maxRestores"`
+	CrunchyBridgeAPIKey       string `json:"crunchyBridgeApiKey"`
+	CrunchyBridgeClusterName  string `json:"crunchyBridgeClusterName"`
+	CrunchyBridgeDatabaseName string `json:"crunchyBridgeDatabaseName"`
 }
 
 // @Summary Get configuration
@@ -72,19 +78,22 @@ func (s *Server) getConfig(c *gin.Context) {
 	}
 
 	c.JSON(http.StatusOK, ConfigResponse{
-		ID:                   config.ID,
-		ConnectionString:     redactConnectionString(config.ConnectionString),
-		PostgresVersion:      config.PostgresVersion,
-		SchemaOnly:           config.SchemaOnly,
-		RefreshSchedule:      config.RefreshSchedule,
-		BranchPostgresqlConf: config.BranchPostgresqlConf,
-		DatabaseName:         config.DatabaseName,
-		Domain:               config.Domain,
-		LetsEncryptEmail:     config.LetsEncryptEmail,
-		MaxRestores:          config.MaxRestores,
-		LastRefreshedAt:      config.LastRefreshedAt,
-		NextRefreshAt:        config.NextRefreshAt,
-		CreatedAt:            config.CreatedAt,
+		ID:                        config.ID,
+		ConnectionString:          redactConnectionString(config.ConnectionString),
+		PostgresVersion:           config.PostgresVersion,
+		SchemaOnly:                config.SchemaOnly,
+		RefreshSchedule:           config.RefreshSchedule,
+		BranchPostgresqlConf:      config.BranchPostgresqlConf,
+		DatabaseName:              config.DatabaseName,
+		Domain:                    config.Domain,
+		LetsEncryptEmail:          config.LetsEncryptEmail,
+		MaxRestores:               config.MaxRestores,
+		LastRefreshedAt:           config.LastRefreshedAt,
+		NextRefreshAt:             config.NextRefreshAt,
+		CreatedAt:                 config.CreatedAt,
+		CrunchyBridgeAPIKey:       redactSecret(config.CrunchyBridgeAPIKey),
+		CrunchyBridgeClusterName:  config.CrunchyBridgeClusterName,
+		CrunchyBridgeDatabaseName: config.CrunchyBridgeDatabaseName,
 	})
 }
 
@@ -117,6 +126,29 @@ func (s *Server) updateConfig(c *gin.Context) {
 		return
 	}
 
+	// Update Crunchy Bridge configuration if provided
+	if req.CrunchyBridgeAPIKey != "" {
+		config.CrunchyBridgeAPIKey = req.CrunchyBridgeAPIKey
+	}
+	if req.CrunchyBridgeClusterName != "" {
+		config.CrunchyBridgeClusterName = req.CrunchyBridgeClusterName
+	}
+	if req.CrunchyBridgeDatabaseName != "" {
+		config.CrunchyBridgeDatabaseName = req.CrunchyBridgeDatabaseName
+	}
+
+	// Clear connection string if Crunchy Bridge fields are being set
+	if req.CrunchyBridgeAPIKey != "" && req.ConnectionString == "" {
+		config.ConnectionString = ""
+	}
+
+	// Clear Crunchy Bridge fields if connection string is being set
+	if req.ConnectionString != "" && req.CrunchyBridgeAPIKey == "" {
+		config.CrunchyBridgeAPIKey = ""
+		config.CrunchyBridgeClusterName = ""
+		config.CrunchyBridgeDatabaseName = ""
+	}
+
 	// Update connection string if provided
 	if req.ConnectionString != "" {
 		// Validate new connection string
@@ -238,22 +270,33 @@ func (s *Server) updateConfig(c *gin.Context) {
 	s.logger.Info().Str("config_id", config.ID).Msg("Configuration updated")
 
 	c.JSON(http.StatusOK, ConfigResponse{
-		ID:                   config.ID,
-		ConnectionString:     redactConnectionString(config.ConnectionString),
-		PostgresVersion:      config.PostgresVersion,
-		SchemaOnly:           config.SchemaOnly,
-		RefreshSchedule:      config.RefreshSchedule,
-		BranchPostgresqlConf: config.BranchPostgresqlConf,
-		DatabaseName:         config.DatabaseName,
-		Domain:               config.Domain,
-		LetsEncryptEmail:     config.LetsEncryptEmail,
-		MaxRestores:          config.MaxRestores,
-		LastRefreshedAt:      config.LastRefreshedAt,
-		NextRefreshAt:        config.NextRefreshAt,
-		CreatedAt:            config.CreatedAt,
+		ID:                        config.ID,
+		ConnectionString:          redactConnectionString(config.ConnectionString),
+		PostgresVersion:           config.PostgresVersion,
+		SchemaOnly:                config.SchemaOnly,
+		RefreshSchedule:           config.RefreshSchedule,
+		BranchPostgresqlConf:      config.BranchPostgresqlConf,
+		DatabaseName:              config.DatabaseName,
+		Domain:                    config.Domain,
+		LetsEncryptEmail:          config.LetsEncryptEmail,
+		MaxRestores:               config.MaxRestores,
+		LastRefreshedAt:           config.LastRefreshedAt,
+		NextRefreshAt:             config.NextRefreshAt,
+		CreatedAt:                 config.CreatedAt,
+		CrunchyBridgeAPIKey:       redactSecret(config.CrunchyBridgeAPIKey),
+		CrunchyBridgeClusterName:  config.CrunchyBridgeClusterName,
+		CrunchyBridgeDatabaseName: config.CrunchyBridgeDatabaseName,
 	})
 }
 
+// redactSecret replaces a secret value with *** if it's not empty
+func redactSecret(secret string) string {
+	if secret == "" {
+		return ""
+	}
+	return "***"
+}
+
 // redactConnectionString replaces the password with *** in a PostgreSQL connection string
 func redactConnectionString(connStr string) string {
 	if connStr == "" {

internal/workers/crunchy_bridge_restore.sh

@@ -125,20 +125,84 @@ log "Configuring PostgreSQL..."
 # Copy TLS certificates (shared across all clusters)
 sudo -u postgres cp /etc/postgresql-common/ssl/server.crt "${DATA_DIR}/"
 sudo -u postgres cp /etc/postgresql-common/ssl/server.key "${DATA_DIR}/"
-
-# Update postgresql.conf - append our settings
-sudo -u postgres tee -a "${DATA_DIR}/postgresql.conf" > /dev/null << EOF
-
-# Branchd settings
-port = ${PG_PORT}
+# Fix permissions on server.key (PostgreSQL requires 0600)
+sudo -u postgres chmod 0600 "${DATA_DIR}/server.key"
+sudo -u postgres chmod 0644 "${DATA_DIR}/server.crt"
+
+# Replace postgresql.conf with a clean config optimized for ephemeral dev branches
+# This removes all Crunchy Bridge specific settings and production-oriented configs
+log "Creating clean postgresql.conf for dev branch..."
+sudo -u postgres tee "${DATA_DIR}/postgresql.conf" > /dev/null << 'EOF'
+# Connection Settings
 listen_addresses = '127.0.0.1'
+port = ${PG_PORT}
+max_connections = 1000
 
 # TLS/SSL
 ssl = on
 ssl_cert_file = 'server.crt'
 ssl_key_file = 'server.key'
+ssl_min_protocol_version = 'TLSv1.2'
+
+# Authentication
+password_encryption = scram-sha-256
+
+# Resource Limits
+# Note: These parameters are set high to support pgBackRest recovery from production backups.
+# PostgreSQL requires these values to be >= the source database during WAL replay.
+shared_buffers = 128MB
+work_mem = 8MB
+maintenance_work_mem = 64MB
+effective_cache_size = 512MB
+max_worker_processes = 200
+max_parallel_workers_per_gather = 2
+max_parallel_workers = 4
+max_prepared_transactions = 100
+max_locks_per_transaction = 256
+max_pred_locks_per_transaction = 256
+
+# WAL Settings
+wal_level = logical
+max_wal_senders = 20
+max_replication_slots = 0
+max_wal_size = 512MB
+min_wal_size = 80MB
+
+# Extensions
+shared_preload_libraries = 'pg_stat_statements'
+
+# Logging
+logging_collector = on
+log_destination = 'stderr'
+log_directory = 'log'
+log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log'
+log_rotation_age = 1d
+log_rotation_size = 100MB
+log_line_prefix = '%t [%p]: [%l-1] user=%u,db=%d,app=%a,client=%h '
+log_timezone = 'UTC'
+
+# Performance
+random_page_cost = 1.1
+effective_io_concurrency = 200
+
+# Statistics
+track_io_timing = on
+track_functions = pl
+
+# Locale
+datestyle = 'iso, mdy'
+timezone = 'UTC'
+lc_messages = 'en_US.UTF-8'
+lc_monetary = 'en_US.UTF-8'
+lc_numeric = 'en_US.UTF-8'
+lc_time = 'en_US.UTF-8'
+default_text_search_config = 'pg_catalog.english'
 EOF
 
+# Substitute the port variable
+sudo -u postgres sed -i "s/\${PG_PORT}/${PG_PORT}/g" "${DATA_DIR}/postgresql.conf"
+log "postgresql.conf created"
+
 # Configure pg_hba.conf for local access only
 sudo -u postgres tee "${DATA_DIR}/pg_hba.conf" > /dev/null << EOF
 # TYPE  DATABASE        USER            ADDRESS                 METHOD
@@ -180,8 +244,8 @@ log "Systemd service created"
 
 # 7. Start PostgreSQL cluster
 log "Starting PostgreSQL cluster..."
-sudo systemctl enable "${SERVICE_NAME}"
-sudo systemctl start "${SERVICE_NAME}"
+sudo systemctl enable "${SERVICE_NAME}" || die "Failed to enable systemd service"
+sudo systemctl start "${SERVICE_NAME}" || die "Failed to start systemd service"
 
 # Wait for PostgreSQL to be ready
 log "Waiting for PostgreSQL to be ready..."