feat: config reloading with fsnotify, poller fallback, and signals (#2161)

**Summary**
This PR introduces additional configuration reloading options.
Previously, Auth only supported fsnotify for watching `.env` changes,
which fails on some filesystems. This implementation adds a poller
fallback mode and a signal-triggered reloads.

**Config reloader**

* Multi-source triggers:
  * **fsnotify** for `.env` changes where supported
* **poller fallback** that diffs directory contents for filesystems
without notification support
* **POSIX signals** (default `SIGUSR1`) for operator initiated reloads
(i.e. `systemctl reoad`)
* Debounce via `GracePeriodInterval` to coalesce bursts of changes into
a single reload.
* Invalid configs are logged but ignored, last good configuration
continues to serve.

**Wiring & observability**

* Update `serve_cmd` to construct `NewReloader` from
`ReloadingConfiguration` and log enabled modes/intervals.
* Graceful shutdown handled independently of reloads.
* Errors from watchers and pollers logged clearly, fallbacks engaged
automatically when possible.

**Configuration knobs** (`GOTRUE_RELOADING_*`)

* `NotifyEnabled` (default true)
* `PollerEnabled` (default true), `PollerInterval` (default 10s)
* `SignalEnabled` (default false), `SignalNumber` (default 10 / SIGUSR1)
* `GracePeriodInterval` (default 5s)

**Tests**

* End-to-end reloader tests: fsnotify happy path, poller fallback,
signal path, debounce, invalid configs.
* Poller unit tests for detection semantics and concurrency guard.

---------

Co-authored-by: Chris Stockton <[email protected]>

Author: cstockton

cmd/serve_cmd.go

@@ -103,19 +103,40 @@ func serve(ctx context.Context) {
 		go func() {
 			defer wg.Done()
 
+			rc := config.Reloading
+			le := logrus.WithFields(logrus.Fields{
+				"component":             "reloader",
+				"notify_enabled":        rc.NotifyEnabled,
+				"poller_enabled":        rc.PollerEnabled,
+				"poller_interval":       rc.PollerInterval.String(),
+				"signal_enabled":        rc.SignalEnabled,
+				"signal_number":         rc.SignalNumber,
+				"grace_period_duration": rc.GracePeriodInterval.String(),
+			})
+			le.Info("starting configuration reloader")
+
+			var err error
+			defer func() {
+				exitFn := le.Info
+				if err != nil {
+					exitFn = le.WithError(err).Error
+				}
+				exitFn("config reloader is exiting")
+			}()
+
 			fn := func(latestCfg *conf.GlobalConfiguration) {
-				log.Info("reloading api with new configuration")
+				le.Info("reloading api with new configuration")
 
 				// When config is updated we notify the apiworker.
 				wrk.ReloadConfig(latestCfg)
 
 				// Create a new API version with the updated config.
 				latestAPI := api.NewAPIWithVersion(
-					config, db, utilities.Version,
+					latestCfg, db, utilities.Version,
 
 					// Create a new mailer with existing template cache.
 					api.WithMailer(
-						templatemailer.FromConfig(config, mrCache),
+						templatemailer.FromConfig(latestCfg, mrCache),
 					),
 
 					// Persist existing rate limiters.
@@ -128,9 +149,9 @@ func serve(ctx context.Context) {
 				ah.Store(latestAPI)
 			}
 
-			rl := reloader.NewReloader(watchDir)
-			if err := rl.Watch(ctx, fn); err != nil {
-				log.WithError(err).Error("watcher is exiting")
+			rl := reloader.NewReloader(rc, watchDir)
+			if err = rl.Watch(ctx, fn); err != nil {
+				log.WithError(err).Error("config reloader is exiting")
 			}
 		}()
 	}

internal/conf/configuration.go

@@ -266,6 +266,39 @@ type ExperimentalConfiguration struct {
 	ProvidersWithOwnLinkingDomain []string `split_words:"true"`
 }
 
+// ReloadingConfiguration holds the configuration values for runtime
+// configuration reloads. These are startup configuration values meaning
+// they do not react to live config reloads.
+//
+// IMPORTANT:
+// * You must provide the --config-dir flag for these settings to take effect.
+// * These config values are for startup, they remain static through reloads.
+type ReloadingConfiguration struct {
+
+	// If notify reloading is enabled the auth server will attempt to use the
+	// filesystems notification support to watch for config updates.
+	NotifyEnabled bool `json:"notify_enabled" split_words:"true" default:"true"`
+
+	// When notify reloading fails, fallback to filesystem polling if this
+	// setting is enabled.
+	PollerEnabled bool `json:"poller_enabled" split_words:"false" default:"false"`
+
+	// This determines how often to poll the filesystem when notify is disabled.
+	PollerInterval time.Duration `json:"poller_interval" split_words:"true" default:"10s"`
+
+	// If signal reloading is enabled the auth server will listen for the
+	// given SignalNumber and reload the config when received. This may be
+	// used to configure `systemd reload` support, by default the SIGUSR1 linux
+	// signal number of 10 is used.
+	SignalEnabled bool `json:"signal_enabled" split_words:"true" default:"false"`
+	SignalNumber  int  `json:"signal_number" split_words:"true" default:"10"`
+
+	// When at least one reloader is enabled this flag determines how much idle
+	// time must pass before triggering a reload. This ensures a single
+	// auth server config reload operation during a burst of config updates.
+	GracePeriodInterval time.Duration `json:"grace_period_interval" split_words:"true" default:"5s"`
+}
+
 // GlobalConfiguration holds all the configuration that applies to all instances.
 type GlobalConfiguration struct {
 	API           APIConfiguration
@@ -307,6 +340,7 @@ type GlobalConfiguration struct {
 	CORS            CORSConfiguration        `json:"cors"`
 
 	Experimental ExperimentalConfiguration `json:"experimental"`
+	Reloading    ReloadingConfiguration    `json:"reloading"`
 }
 
 type CORSConfiguration struct {

internal/reloader/poller.go

@@ -0,0 +1,181 @@
+package reloader
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/fs"
+	"maps"
+	"os"
+	"strings"
+	"sync"
+	"time"
+)
+
+const (
+	pollerMaxScan  = 1
+	pollerMaxFiles = 1000
+)
+
+type poller struct {
+	pollMu    sync.Mutex
+	dir       string
+	cur, prev *pollerState
+}
+
+type pollerFile struct {
+	name string
+	size int64
+	mode fs.FileMode
+	mod  time.Time
+	dir  bool
+}
+
+type pollerState struct {
+	updatedAt time.Time
+	files     map[string]*pollerFile
+}
+
+func (o *pollerState) reset() { clear(o.files) }
+
+func newPollerState() *pollerState {
+	return &pollerState{
+		files: make(map[string]*pollerFile),
+	}
+}
+
+func newPollerFile(fi fs.FileInfo) *pollerFile {
+	return &pollerFile{
+		name: fi.Name(),
+		size: fi.Size(),
+		mode: fi.Mode(),
+		mod:  fi.ModTime(),
+		dir:  fi.IsDir(),
+	}
+}
+
+func newPoller(watchDir string) *poller {
+	return &poller{
+		dir:  watchDir,
+		cur:  newPollerState(),
+		prev: newPollerState(),
+	}
+}
+
+func (o *poller) watch(
+	ctx context.Context,
+	ival time.Duration,
+	notifyFn func(),
+	errFn func(error),
+) error {
+	tr := time.NewTicker(ival)
+	defer tr.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-tr.C:
+			changed, err := o.poll(ctx)
+			if err != nil {
+				errFn(err)
+				continue
+			}
+			if changed {
+				notifyFn()
+			}
+		}
+	}
+}
+
+func (o *poller) poll(ctx context.Context) (bool, error) {
+	if ok := o.pollMu.TryLock(); !ok {
+		const msg = "reloader: poller: concurrent calls to poll are invalid"
+		return false, errors.New(msg)
+	}
+	defer o.pollMu.Unlock()
+
+	if err := ctx.Err(); err != nil {
+		return false, err
+	}
+
+	o.prev, o.cur = o.cur, o.prev
+	if err := o.scan(ctx, o.cur); err != nil {
+		return false, err
+	}
+	o.cur.updatedAt = time.Now()
+	m1, m2 := o.prev.files, o.cur.files
+
+	if o.prev.updatedAt.IsZero() {
+		return false, nil
+	}
+
+	eq := maps.EqualFunc(m1, m2, func(v1, v2 *pollerFile) bool {
+		return *v1 == *v2
+	})
+	return !eq, nil
+}
+
+func (o *poller) scan(
+	ctx context.Context,
+	ps *pollerState,
+) error {
+	o.cur.reset()
+
+	f, err := os.Open(o.dir)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	return o.scanFile(ctx, ps, f)
+}
+
+func (o *poller) scanFile(
+	ctx context.Context,
+	ps *pollerState,
+	f fs.ReadDirFile,
+) error {
+	fi, err := f.Stat()
+	if err != nil {
+		return fmt.Errorf("poller: %w", err)
+	}
+	if !fi.IsDir() {
+		return fmt.Errorf("poller: %q is not a directory", o.dir)
+	}
+
+	for range pollerMaxFiles / pollerMaxScan {
+		if err := ctx.Err(); err != nil {
+			return err
+		}
+
+		ents, err := f.ReadDir(pollerMaxScan)
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return fmt.Errorf("poller: error reading dir %q: %w", o.dir, err)
+		}
+		o.scanEntries(ps, ents)
+	}
+	return fmt.Errorf("poller: %q has too many files", o.dir)
+}
+
+func (o *poller) scanEntries(ps *pollerState, ents []fs.DirEntry) {
+	for _, ent := range ents {
+		fi, err := ent.Info()
+		if err != nil {
+			continue
+		}
+		if fi.IsDir() {
+			continue
+		}
+		if !strings.HasSuffix(ent.Name(), ".env") {
+			continue
+		}
+
+		pf := newPollerFile(fi)
+		ps.files[pf.name] = pf
+	}
+}