wfe/ra: Periodically load rate limit overrides from the database (#8407)

Add a `refreshOverrides` func to the `ratelimits.limitRegistry` struct.
Instead of populating the static `overrides` field once when creating an
instance of the struct, call the new func at startup and then every 30
minutes.

Emit relevant logs and metrics from `limitRegistry`.

Add an `OverridesFromDB` limiter config flag (for RA & WFE) to read
overrides from the DB instead of a file.

Flatten `newLimitRegistry.*()` methods' logic into their sole caller,
`NewTransactionBuilder()`.

Rename `loadDefaults()` & `loadOverrides()`, appending `FromFile` for
clarity/consistency.

test: Add ra-sct-provider dependency on SA.

**Important for deployment:** If the `OverridesFromDB` config flag is
enabled, an RA now depends on the SA in order to load overrides. The RA
must be added as a gRPC client of `sa.StorageAuthorityReadOnly`.

*CPS Compliance Review:* `OverridesFromDB` only controls how we load
rate limit overrides, which has no compliance implications beyond
general API availability (e.g. for revocation). I've checked our CP/CPS
to confirm we make no related stipulations.

Fixes #8382

Author: jprenken

cmd/boulder-ra/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"os"
+	"time"
 
 	"github.com/jmhodges/clock"
 
@@ -69,13 +70,17 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides passed in this file must be
-			// identical to those in the WFE.
+			// neither this field nor OverridesFromDB is set, all requesters
+			// will be subject to the default rate limits. Overrides passed in
+			// this file must be identical to those in the WFE.
 			//
 			// Note: At this time, only the Failed Authorizations overrides are
 			// necessary in the RA.
 			Overrides string
+
+			// OverridesFromDB causes the WFE and RA to retrieve rate limit overrides
+			// from the database, instead of from a file.
+			OverridesFromDB bool
 		}
 
 		// MaxNames is the maximum number of subjectAltNames in a single cert.
@@ -271,8 +276,18 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, scope)
 		limiter, err = ratelimits.NewLimiter(clk, source, scope)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides)
+		if c.RA.Limiter.OverridesFromDB {
+			if c.RA.Limiter.Overrides != "" {
+				cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
+			}
+			saroc := sapb.NewStorageAuthorityReadOnlyClient(saConn)
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.RA.Limiter.Defaults, saroc.GetEnabledRateLimitOverrides, scope, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides, scope, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overrideRefresherShutdown := txnBuilder.NewRefresher(30 * time.Minute)
+		defer overrideRefresherShutdown()
 	}
 
 	rai := ra.NewRegistrationAuthorityImpl(

cmd/boulder-wfe2/main.go

@@ -151,11 +151,15 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides for the Failed Authorizations
-			// overrides passed in this file must be identical to those in the
-			// RA.
+			// neither this field nor OverridesFromDB is set, all requesters
+			// will be subject to the default rate limits. Overrides for the
+			// Failed Authorizations overrides passed in this file must be
+			// identical to those in the RA.
 			Overrides string
+
+			// OverridesFromDB causes the WFE and RA to retrieve rate limit
+			// overrides from the database, instead of from a file.
+			OverridesFromDB bool
 		}
 
 		// CertProfiles is a map of acceptable certificate profile names to
@@ -326,6 +330,7 @@ func main() {
 	var limiter *ratelimits.Limiter
 	var txnBuilder *ratelimits.TransactionBuilder
 	var limiterRedis *bredis.Ring
+	overridesRefresherShutdown := func() {}
 	if c.WFE.Limiter.Defaults != "" {
 		// Setup rate limiting.
 		limiterRedis, err = bredis.NewRingFromConfig(*c.WFE.Limiter.Redis, stats, logger)
@@ -334,8 +339,16 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides)
+		if c.WFE.Limiter.OverridesFromDB {
+			if c.WFE.Limiter.Overrides != "" {
+				cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
+			}
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.WFE.Limiter.Defaults, sac.GetEnabledRateLimitOverrides, stats, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides, stats, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overridesRefresherShutdown = txnBuilder.NewRefresher(30 * time.Minute)
 	}
 
 	var accountGetter wfe2.AccountGetter
@@ -413,6 +426,7 @@ func main() {
 	defer func() {
 		ctx, cancel := context.WithTimeout(context.Background(), c.WFE.ShutdownStopTimeout.Duration)
 		defer cancel()
+		overridesRefresherShutdown()
 		_ = srv.Shutdown(ctx)
 		_ = tlsSrv.Shutdown(ctx)
 		limiterRedis.StopLookups()

cmd/sfe/main.go

@@ -223,7 +223,7 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "")
+		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "", stats, logger)
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
 	}
 

ra/ra.go

@@ -85,6 +85,7 @@ type RegistrationAuthorityImpl struct {
 	maxContactsPerReg int
 	limiter           *ratelimits.Limiter
 	txnBuilder        *ratelimits.TransactionBuilder
+	started           time.Time
 	finalizeTimeout   time.Duration
 	drainWG           sync.WaitGroup
 
@@ -110,6 +111,15 @@ type RegistrationAuthorityImpl struct {
 
 var _ rapb.RegistrationAuthorityServer = (*RegistrationAuthorityImpl)(nil)
 
+// Health implements our grpc.checker interface. This method will be called
+// periodically to set the gRPC service's healthpb.Health.Check() status.
+func (ra *RegistrationAuthorityImpl) Health(ctx context.Context) error {
+	if ra.txnBuilder.Ready() || time.Since(ra.started) > time.Second*10 {
+		return nil
+	}
+	return errors.New("waiting for overrides")
+}
+
 // NewRegistrationAuthorityImpl constructs a new RA object.
 func NewRegistrationAuthorityImpl(
 	clk clock.Clock,
@@ -234,6 +244,7 @@ func NewRegistrationAuthorityImpl(
 		keyPolicy:                 keyPolicy,
 		limiter:                   limiter,
 		txnBuilder:                txnBuilder,
+		started:                   clk.Now(),
 		publisher:                 pubc,
 		finalizeTimeout:           finalizeTimeout,
 		ctpolicy:                  ctp,

ra/ra_test.go

@@ -363,7 +363,7 @@ func initAuthorities(t *testing.T) (*DummyValidationAuthority, sapb.StorageAutho
 	rlSource := ratelimits.NewInmemSource()
 	limiter, err := ratelimits.NewLimiter(fc, rlSource, stats)
 	test.AssertNotError(t, err, "making limiter")
-	txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "")
+	txnBuilder, err := ratelimits.NewTransactionBuilderFromFiles("../test/config-next/wfe2-ratelimit-defaults.yml", "", metrics.NoopRegisterer, log)
 	test.AssertNotError(t, err, "making transaction composer")
 
 	testKeyPolicy, err := goodkey.NewPolicy(nil, nil)
@@ -708,7 +708,7 @@ func TestPerformValidation_FailedValidationsTriggerPauseIdentifiersRatelimit(t *
 			Burst:  1,
 			Count:  1,
 			Period: config.Duration{Duration: time.Hour * 24}},
-	})
+	}, nil, metrics.NoopRegisterer, blog.NewMock())
 	test.AssertNotError(t, err, "making transaction composer")
 	ra.txnBuilder = txnBuilder
 
@@ -967,7 +967,7 @@ func TestDeactivateAuthorization_Pausing(t *testing.T) {
 			Burst:  1,
 			Count:  1,
 			Period: config.Duration{Duration: time.Hour * 24}},
-	})
+	}, nil, metrics.NoopRegisterer, blog.NewMock())
 	test.AssertNotError(t, err, "making transaction composer")
 	ra.txnBuilder = txnBuilder