Cluster: Add config option to sync and drop ProxySQL user accounts #98

Signed-off-by: Michael Mayer <michael@photoprism.app>

Author: lastzero

internal/config/flags.go

@@ -938,6 +938,12 @@ var Flags = CliFlags{
 			EnvVars: EnvVars("DATABASE_PROVISION_DSN"),
 			Hidden:  true,
 		}}, {
+		Flag: &cli.StringFlag{
+			Name:    "database-provision-proxy-dsn",
+			Usage:   "ProxySQL admin `DSN` (port 6032 by default) for keeping user accounts in sync",
+			EnvVars: EnvVars("DATABASE_PROVISION_PROXY_DSN"),
+			Hidden:  true,
+		}}, {
 		Flag: &cli.StringFlag{
 			Name:    "ffmpeg-bin",
 			Usage:   "FFmpeg `COMMAND` for video transcoding and thumbnail extraction",

internal/config/options.go

@@ -199,6 +199,7 @@ func (c *Config) Report() (rows [][]string, cols []string) {
 			{"database-provision-driver", c.options.DatabaseProvisionDriver},
 			{"database-provision-prefix", c.DatabaseProvisionPrefix()},
 			{"database-provision-dsn", maskDatabaseProvisionDSN(c.options.DatabaseProvisionDSN)},
+			{"database-provision-proxy-dsn", maskDatabaseProvisionDSN(c.options.DatabaseProvisionProxyDSN)},
 		}...)
 	}
 

internal/config/report.go

@@ -70,3 +70,43 @@ The provisioner package manages per-node MariaDB schemas and users for cluster d
 
 - Fast pass: `go test ./internal/service/cluster/provisioner -count=1`
 - End-to-end sanity with API: `go test ./internal/api -run 'ClusterNodesRegister' -count=1` (ensures the API cleanup helper stays aligned with the provisioner)
+
+## ProxySQL Integration
+
+Use ProxySQL to verify tenant provisioning stays in sync with the proxy in addition to MariaDB. The unit test suite ships with an opt-in integration test (`TestEnsureCredentials_ProxySQLIntegration`) that exercises the full flow once ProxySQL is available locally.
+
+### One-Time Setup (inside the dev container)
+
+1. Download and install ProxySQL (v3.0.2 shown here):
+   ```bash
+   cd /tmp
+   curl -fL -o proxysql_3.0.2-debian12_amd64.deb https://github.com/sysown/proxysql/releases/download/v3.0.2/proxysql_3.0.2-debian12_amd64.deb
+   sudo dpkg -i proxysql_3.0.2-debian12_amd64.deb
+   ```
+2. Start ProxySQL as a daemon using the default config (/etc/proxysql.cnf ships with admin `admin:admin`):
+   ```bash
+   sudo proxysql --config /etc/proxysql.cnf --pidfile /tmp/proxysql.pid --daemon
+   ```
+3. Confirm the admin listener is reachable:
+   ```bash
+   sudo mysql --protocol=TCP --host=127.0.0.1 --port=6032 --user=admin --password=admin -e 'SELECT 1'
+   ```
+
+The bundled MariaDB instance (credentials in `.my.cnf` at the repo root) is sufficient as a backend; no extra ProxySQL configuration is required for the integration test.
+
+### Running the Integration Test
+
+1. When ProxySQL is running, toggle the test via an environment variable:
+   ```bash
+   PHOTOPRISM_TEST_PROXYSQL=1 go test ./internal/service/cluster/provisioner -run TestEnsureCredentials_ProxySQLIntegration -count=1
+   ```
+   - Override the admin DSN with `PHOTOPRISM_TEST_PROXYSQL_DSN=user:pass@tcp(host:6032)/` if you changed the default credentials or port.
+2. The test provisions a tenant, verifies the ProxySQL `mysql_users` row, reruns the idempotent ensure path, and exercises `DropCredentials`. Cleanup hooks remove both the MariaDB schema/user and the ProxySQL account.
+
+### Tearing Down / Restarting
+
+- Stop ProxySQL when finished:
+  ```bash
+  sudo kill "$(cat /tmp/proxysql.pid)"
+  ```
+- To restart, re-run the daemon command from the setup section. The generated SSL materials live under `/var/lib/proxysql`.

internal/service/cluster/provisioner/README.md

@@ -118,6 +118,18 @@ func EnsureCredentials(ctx context.Context, conf *config.Config, nodeUUID, nodeN
 		return out, created, err
 	}
 
+	// 7) Provision ProxySQL user account if ProvisionProxyDSN is set.
+	if ProvisionProxyDSN != "" {
+		proxyPass := ""
+		if rotate || created {
+			proxyPass = dbPass
+		}
+
+		if err = SyncProxyUser(ctx, ProvisionProxyDSN, dbName, dbUser, proxyPass, ProvisionProxyOptions); err != nil {
+			return out, created, fmt.Errorf("proxysql: %w", err)
+		}
+	}
+
 	// Compose credentials.
 	out.Host = DatabaseHost
 	out.Port = DatabasePort
@@ -167,6 +179,12 @@ func DropCredentials(ctx context.Context, dbName, user string) error {
 		}
 	}
 
+	if ProvisionProxyDSN != "" && user != "" {
+		if err := DropProxyUser(ctx, ProvisionProxyDSN, user); err != nil {
+			errs = append(errs, fmt.Sprintf("proxysql: %v", err))
+		}
+	}
+
 	if len(errs) > 0 {
 		return fmt.Errorf("drop credentials: %s", strings.Join(errs, "; "))
 	}

internal/service/cluster/provisioner/credentials.go

@@ -0,0 +1,136 @@
+package provisioner
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"strings"
+)
+
+const (
+	// DefaultProxyHostgroup routes tenant connections to the primary (writer) backend hostgroup.
+	DefaultProxyHostgroup = 10
+	// DefaultProxyFrontend enables clients to authenticate through ProxySQL; required for tenant users.
+	DefaultProxyFrontend = 1
+	// DefaultProxyBackend keeps tenant users from authenticating against upstream servers directly.
+	DefaultProxyBackend = 0
+	// DefaultProxyMaxConnections caps concurrent connections per tenant to avoid exhausting ProxySQL.
+	DefaultProxyMaxConnections = 200
+	// DefaultProxyUseSSL toggles ProxySQL's SSL flag for tenant accounts (0 = disabled by default).
+	DefaultProxyUseSSL = 0
+	// DefaultProxyComment labels provisioned users so operators can distinguish auto-managed accounts.
+	DefaultProxyComment = "Portal provisioned tenant"
+)
+
+// ProxyOptions describes the ProxySQL mysql_users attributes to apply when syncing tenant accounts.
+type ProxyOptions struct {
+	Hostgroup      int
+	Frontend       int
+	Backend        int
+	MaxConnections int
+	UseSSL         int
+	Comment        string
+}
+
+// ProvisionProxyDSN specifies the optional ProxySQL admin DSN (port 6032 by default) for keeping user accounts in sync.
+var ProvisionProxyDSN = ""
+
+// ProvisionProxyOptions stores the current defaults used when synchronizing ProxySQL tenant accounts.
+var ProvisionProxyOptions = ProxyOptions{
+	Hostgroup:      DefaultProxyHostgroup,
+	Frontend:       DefaultProxyFrontend,
+	Backend:        DefaultProxyBackend,
+	MaxConnections: DefaultProxyMaxConnections,
+	UseSSL:         DefaultProxyUseSSL,
+	Comment:        DefaultProxyComment,
+}
+
+// SyncProxyUser ensures the ProxySQL mysql_users entry matches the provided schema and credentials.
+// When pass is empty the existing password is preserved, allowing non-rotating syncs that only adjust metadata.
+func SyncProxyUser(ctx context.Context, proxyDSN, schema, user, pass string, opts ProxyOptions) error {
+	db, err := sql.Open("mysql", normalizeProxyDSN(proxyDSN))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	password := pass
+	if password == "" {
+		if err := db.QueryRowContext(ctx, "SELECT password FROM mysql_users WHERE username = ?", user).Scan(&password); err != nil {
+			if errors.Is(err, sql.ErrNoRows) {
+				return errors.New("proxysql: existing user not found and password not provided")
+			}
+			return err
+		}
+	}
+
+	if opts.Comment == "" {
+		opts.Comment = DefaultProxyComment
+	}
+
+	if _, err = db.ExecContext(ctx, "DELETE FROM mysql_users WHERE username = ?", user); err != nil {
+		return err
+	}
+
+	if _, err = db.ExecContext(ctx, `
+		INSERT INTO mysql_users (
+			username, password, active, use_ssl, default_hostgroup,
+			default_schema, schema_locked, transaction_persistent,
+			fast_forward, backend, frontend, max_connections, attributes, comment
+		) VALUES (
+			?, ?, 1, ?, ?, ?,
+			0, 1,
+			0, ?, ?, ?, '{}', ?
+		)
+	`, user, password, opts.UseSSL, opts.Hostgroup, schema, opts.Backend, opts.Frontend, opts.MaxConnections, opts.Comment); err != nil {
+		return err
+	}
+
+	return applyProxySQL(ctx, db)
+}
+
+// DropProxyUser removes the mysql_users record for a tenant and reloads ProxySQL runtime/disk.
+func DropProxyUser(ctx context.Context, proxyDSN, user string) error {
+	db, err := sql.Open("mysql", normalizeProxyDSN(proxyDSN))
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+
+	if _, err = db.ExecContext(ctx, "DELETE FROM mysql_users WHERE username = ?", user); err != nil {
+		return err
+	}
+
+	return applyProxySQL(ctx, db)
+}
+
+// applyProxySQL reloads mysql_users into ProxySQL runtime and persists the changes to disk.
+func applyProxySQL(ctx context.Context, db *sql.DB) error {
+	for _, stmt := range []string{
+		"LOAD MYSQL USERS TO RUNTIME",
+		"SAVE MYSQL USERS TO DISK",
+	} {
+		if _, err := db.ExecContext(ctx, stmt); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// normalizeProxyDSN adds interpolateParams to ProxySQL admin DSNs when missing so prepared statements work.
+func normalizeProxyDSN(dsn string) string {
+	if dsn == "" || strings.Contains(dsn, "interpolateParams=") {
+		return dsn
+	}
+
+	sep := "?"
+	if strings.Contains(dsn, "?") {
+		if strings.HasSuffix(dsn, "?") || strings.HasSuffix(dsn, "&") {
+			sep = ""
+		} else {
+			sep = "&"
+		}
+	}
+
+	return dsn + sep + "interpolateParams=true"
+}

internal/service/cluster/provisioner/proxysql.go

@@ -0,0 +1,127 @@
+package provisioner
+
+import (
+	"context"
+	"database/sql"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/photoprism/photoprism/internal/config"
+)
+
+func TestEnsureCredentials_ProxySQLIntegration(t *testing.T) {
+	if os.Getenv("PHOTOPRISM_TEST_PROXYSQL") == "" {
+		t.Skip("PHOTOPRISM_TEST_PROXYSQL not set; skipping ProxySQL integration test")
+	}
+
+	ctx := context.Background()
+
+	proxyDSN := os.Getenv("PHOTOPRISM_TEST_PROXYSQL_DSN")
+	if proxyDSN == "" {
+		proxyDSN = "admin:admin@tcp(127.0.0.1:6032)/"
+	}
+
+	adminDB, err := sql.Open("mysql", normalizeProxyDSN(proxyDSN))
+	if err != nil {
+		t.Skipf("proxy DSN not openable: %v", err)
+	}
+	t.Cleanup(func() { _ = adminDB.Close() })
+
+	pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+	if err := adminDB.PingContext(pingCtx); err != nil {
+		cancel()
+		t.Skipf("proxy DSN not reachable: %v", err)
+	}
+	cancel()
+
+	origDSN := ProvisionProxyDSN
+	origOpts := ProvisionProxyOptions
+	ProvisionProxyDSN = proxyDSN
+	ProvisionProxyOptions = ProxyOptions{
+		Hostgroup:      DefaultProxyHostgroup,
+		Frontend:       DefaultProxyFrontend,
+		Backend:        DefaultProxyBackend,
+		MaxConnections: DefaultProxyMaxConnections,
+		UseSSL:         DefaultProxyUseSSL,
+		Comment:        DefaultProxyComment,
+	}
+	t.Cleanup(func() {
+		ProvisionProxyDSN = origDSN
+		ProvisionProxyOptions = origOpts
+	})
+
+	conf := config.NewConfig(config.CliTestContext())
+	conf.Options().ClusterUUID = time.Now().UTC().Format("20060102-150405.000000000")
+
+	nodeUUID := "11111111-1111-4111-8111-333333333333"
+	nodeName := "pp-proxy-itest"
+
+	creds, _, err := EnsureCredentials(ctx, conf, nodeUUID, nodeName, true)
+	if err != nil {
+		t.Fatalf("EnsureCredentials with ProxySQL error: %v", err)
+	}
+
+	t.Cleanup(func() {
+		if creds.Name == "" || creds.User == "" {
+			return
+		}
+		if dropErr := DropCredentials(ctx, creds.Name, creds.User); dropErr != nil {
+			t.Logf("cleanup drop credentials: %v", dropErr)
+		}
+	})
+
+	var defaultSchema, comment string
+	var frontend, backend, maxConnections int
+
+	err = adminDB.QueryRowContext(ctx, `
+		SELECT default_schema, frontend, backend, max_connections, comment
+		  FROM mysql_users WHERE username = ?
+	`, creds.User).Scan(&defaultSchema, &frontend, &backend, &maxConnections, &comment)
+	if err != nil {
+		t.Fatalf("mysql_users lookup: %v", err)
+	}
+
+	if defaultSchema != creds.Name {
+		t.Fatalf("expected default_schema %q, got %q", creds.Name, defaultSchema)
+	}
+	if frontend != ProvisionProxyOptions.Frontend {
+		t.Fatalf("expected frontend=%d, got %d", ProvisionProxyOptions.Frontend, frontend)
+	}
+	if backend != ProvisionProxyOptions.Backend {
+		t.Fatalf("expected backend=%d, got %d", ProvisionProxyOptions.Backend, backend)
+	}
+	if maxConnections != ProvisionProxyOptions.MaxConnections {
+		t.Fatalf("expected max_connections=%d, got %d", ProvisionProxyOptions.MaxConnections, maxConnections)
+	}
+	if comment != ProvisionProxyOptions.Comment {
+		t.Fatalf("expected comment %q, got %q", ProvisionProxyOptions.Comment, comment)
+	}
+
+	if _, _, err := EnsureCredentials(ctx, conf, nodeUUID, nodeName, false); err != nil {
+		t.Fatalf("EnsureCredentials (rotate=false) error: %v", err)
+	}
+
+	var count int
+	if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM mysql_users WHERE username = ?", creds.User).Scan(&count); err != nil {
+		t.Fatalf("count mysql_users: %v", err)
+	}
+	if count != 1 {
+		t.Fatalf("expected mysql_users count 1 after idempotent ensure, got %d", count)
+	}
+
+	nodeUser := creds.User
+	nodeSchema := creds.Name
+
+	if err := DropCredentials(ctx, nodeSchema, nodeUser); err != nil {
+		t.Fatalf("DropCredentials error: %v", err)
+	}
+	creds.Name, creds.User = "", ""
+
+	if err := adminDB.QueryRowContext(ctx, "SELECT COUNT(*) FROM mysql_users WHERE username = ?", nodeUser).Scan(&count); err != nil {
+		t.Fatalf("count mysql_users after drop: %v", err)
+	}
+	if count != 0 {
+		t.Fatalf("expected mysql_users count 0 after drop, got %d", count)
+	}
+}