WFE: Add custom balancer implementation which routes nonce redemption RPCs by prefix (#6618)

Assign nonce prefixes for each nonce-service by taking the first eight
characters of the the base64url encoded HMAC-SHA256 hash of the RPC
listening address using a provided key. The provided key must be same
across all boulder-wfe and nonce-service instances.
- Add a custom `grpc-go` load balancer implementation (`nonce`) which
can route nonce redemption RPC messages by matching the prefix to the
derived prefix of the nonce-service instance which created it.
- Modify the RPC client constructor to allow the operator to override
the default load balancer implementation (`round_robin`).
- Modify the `srv` RPC resolver to accept a comma separated list of
targets to be resolved.
- Remove unused nonce-service `-prefix` flag.

Fixes #6404

Author: beautifulentropy

cmd/boulder-wfe2/main.go

@@ -18,9 +18,10 @@ import (
 	"github.com/letsencrypt/boulder/features"
 	"github.com/letsencrypt/boulder/goodkey"
 	bgrpc "github.com/letsencrypt/boulder/grpc"
+	"github.com/letsencrypt/boulder/grpc/noncebalancer"
 	"github.com/letsencrypt/boulder/issuance"
 	blog "github.com/letsencrypt/boulder/log"
-	noncepb "github.com/letsencrypt/boulder/nonce/proto"
+	"github.com/letsencrypt/boulder/nonce"
 	rapb "github.com/letsencrypt/boulder/ra/proto"
 	sapb "github.com/letsencrypt/boulder/sa/proto"
 	"github.com/letsencrypt/boulder/wfe2"
@@ -50,16 +51,36 @@ type Config struct {
 
 		RAService *cmd.GRPCClientConfig
 		SAService *cmd.GRPCClientConfig
-		// GetNonceService contains a gRPC config for any nonce-service instances
-		// which we want to retrieve nonces from. In a multi-DC deployment this
-		// should refer to any local nonce-service instances.
+
+		// GetNonceService is a gRPC config which contains a single SRV name
+		// used to lookup nonce-service instances used exclusively for nonce
+		// creation. In a multi-DC deployment this should refer to local
+		// nonce-service instances only.
 		GetNonceService *cmd.GRPCClientConfig
+
 		// RedeemNonceServices contains a map of nonce-service prefixes to
 		// gRPC configs we want to use to redeem nonces. In a multi-DC deployment
 		// this should contain all nonce-services from all DCs as we want to be
 		// able to redeem nonces generated at any DC.
+		//
+		// DEPRECATED: See RedeemNonceService, below.
+		// TODO (#6610) Remove this after all configs have migrated to
+		// `RedeemNonceService`.
 		RedeemNonceServices map[string]cmd.GRPCClientConfig
 
+		// RedeemNonceService is a gRPC config which contains a list of SRV
+		// names used to lookup nonce-service instances used exclusively for
+		// nonce redemption. In a multi-DC deployment this should contain both
+		// local and remote nonce-service instances.
+		RedeemNonceService *cmd.GRPCClientConfig
+
+		// NoncePrefixKey is a secret used for deriving the prefix of each nonce
+		// instance. It should contain 256 bits of random data to be suitable as
+		// an HMAC-SHA256 key (e.g. the output of `openssl rand -hex 32`). In a
+		// multi-DC deployment this value should be the same across all
+		// boulder-wfe and nonce-service instances.
+		NoncePrefixKey cmd.PasswordConfig
+
 		// CertificateChains maps AIA issuer URLs to certificate filenames.
 		// Certificates are read into the chain in the order they are defined in the
 		// slice of filenames.
@@ -280,7 +301,7 @@ func loadChain(certFiles []string) (*issuance.Certificate, []byte, error) {
 	return certs[0], buf.Bytes(), nil
 }
 
-func setupWFE(c Config, scope prometheus.Registerer, clk clock.Clock) (rapb.RegistrationAuthorityClient, sapb.StorageAuthorityReadOnlyClient, noncepb.NonceServiceClient, map[string]noncepb.NonceServiceClient) {
+func setupWFE(c Config, scope prometheus.Registerer, clk clock.Clock) (rapb.RegistrationAuthorityClient, sapb.StorageAuthorityReadOnlyClient, nonce.Getter, map[string]nonce.Redeemer, nonce.Redeemer, string) {
 	tlsConfig, err := c.WFE.TLS.Load()
 	cmd.FailOnError(err, "TLS config")
 
@@ -292,21 +313,56 @@ func setupWFE(c Config, scope prometheus.Registerer, clk clock.Clock) (rapb.Regi
 	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to SA")
 	sac := sapb.NewStorageAuthorityReadOnlyClient(saConn)
 
-	var rns noncepb.NonceServiceClient
-	npm := map[string]noncepb.NonceServiceClient{}
-	if c.WFE.GetNonceService != nil {
-		rnsConn, err := bgrpc.ClientSetup(c.WFE.GetNonceService, tlsConfig, scope, clk)
-		cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to get nonce service")
-		rns = noncepb.NewNonceServiceClient(rnsConn)
+	// TODO(#6610) Refactor these checks.
+	if c.WFE.RedeemNonceService != nil && c.WFE.RedeemNonceServices != nil {
+		cmd.Fail("Only one of 'redeemNonceService' or 'redeemNonceServices' should be configured.")
+	}
+	if c.WFE.RedeemNonceService == nil && c.WFE.RedeemNonceServices == nil {
+		cmd.Fail("One of 'redeemNonceService' or 'redeemNonceServices' must be configured.")
+	}
+	if c.WFE.RedeemNonceService != nil && c.WFE.NoncePrefixKey.PasswordFile == "" {
+		cmd.Fail("'noncePrefixKey' must be configured if 'redeemNonceService' is configured.")
+	}
+	if c.WFE.GetNonceService == nil {
+		cmd.Fail("'getNonceService' must be configured")
+	}
+
+	var rncKey string
+	if c.WFE.NoncePrefixKey.PasswordFile != "" {
+		rncKey, err = c.WFE.NoncePrefixKey.Pass()
+		cmd.FailOnError(err, "Failed to load noncePrefixKey")
+	}
+
+	getNonceConn, err := bgrpc.ClientSetup(c.WFE.GetNonceService, tlsConfig, scope, clk)
+	cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to get nonce service")
+	gnc := nonce.NewGetter(getNonceConn)
+
+	var rnc nonce.Redeemer
+	var npm map[string]nonce.Redeemer
+	if c.WFE.RedeemNonceService != nil {
+		// Dispatch nonce redemption RPCs dynamically.
+		if c.WFE.RedeemNonceService.SRVResolver != noncebalancer.SRVResolverScheme {
+			cmd.Fail(fmt.Sprintf(
+				"'redeemNonceService.SRVResolver' must be set to %q", noncebalancer.SRVResolverScheme),
+			)
+		}
+		redeemNonceConn, err := bgrpc.ClientSetup(c.WFE.RedeemNonceService, tlsConfig, scope, clk)
+		cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to redeem nonce service")
+		rnc = nonce.NewRedeemer(redeemNonceConn)
+	} else {
+		// Dispatch nonce redpemption RPCs using a static mapping.
+		//
+		// TODO(#6610) Remove code below and the `npm` mapping.
+		npm = make(map[string]nonce.Redeemer)
 		for prefix, serviceConfig := range c.WFE.RedeemNonceServices {
 			serviceConfig := serviceConfig
 			conn, err := bgrpc.ClientSetup(&serviceConfig, tlsConfig, scope, clk)
 			cmd.FailOnError(err, "Failed to load credentials and create gRPC connection to redeem nonce service")
-			npm[prefix] = noncepb.NewNonceServiceClient(conn)
+			npm[prefix] = nonce.NewRedeemer(conn)
 		}
 	}
 
-	return rac, sac, rns, npm
+	return rac, sac, gnc, npm, rnc, rncKey
 }
 
 type errorWriter struct {
@@ -391,7 +447,7 @@ func main() {
 
 	clk := cmd.Clock()
 
-	rac, sac, rns, npm := setupWFE(c, stats, clk)
+	rac, sac, gnc, npm, rnc, npKey := setupWFE(c, stats, clk)
 
 	kp, err := goodkey.NewKeyPolicy(&c.WFE.GoodKey, sac.KeyBlocked)
 	cmd.FailOnError(err, "Unable to create key policy")
@@ -434,15 +490,17 @@ func main() {
 		kp,
 		allCertChains,
 		issuerCerts,
-		rns,
-		npm,
 		logger,
 		c.WFE.Timeout.Duration,
 		c.WFE.StaleTimeout.Duration,
 		authorizationLifetime,
 		pendingAuthorizationLifetime,
 		rac,
 		sac,
+		gnc,
+		npm,
+		rnc,
+		npKey,
 		accountGetter,
 	)
 	cmd.FailOnError(err, "Unable to create WFE")

cmd/config.go

@@ -16,6 +16,7 @@ import (
 	"github.com/go-sql-driver/mysql"
 	"github.com/honeycombio/beeline-go"
 	"github.com/letsencrypt/boulder/core"
+	"google.golang.org/grpc/resolver"
 )
 
 // PasswordConfig contains a path to a file containing a password.
@@ -245,12 +246,20 @@ func (d *ConfigDuration) UnmarshalYAML(unmarshal func(interface{}) error) error
 	return nil
 }
 
+// ServiceDomain contains the service and domain name the gRPC client will use
+// to construct a SRV DNS query to lookup backends.
+type ServiceDomain struct {
+	Service string
+	Domain  string
+}
+
 // GRPCClientConfig contains the information necessary to setup a gRPC client
 // connection. The following field combinations are allowed:
 //
 // ServerIPAddresses, [Timeout]
 // ServerAddress, [Timeout], [DNSAuthority], [HostOverride]
-// SRVLookup, [Timeout], [DNSAuthority], [HostOverride]
+// SRVLookup, [Timeout], [DNSAuthority], [HostOverride], [SRVResolver]
+// SRVLookups, [Timeout], [DNSAuthority], [HostOverride], [SRVResolver]
 type GRPCClientConfig struct {
 	// DNSAuthority is a single <hostname|IPv4|[IPv6]>:<port> of the DNS server
 	// to be used for resolution of gRPC backends. If the address contains a
@@ -290,10 +299,21 @@ type GRPCClientConfig struct {
 	// $ dig @10.55.55.10 -t SRV _foo._tcp.service.consul +short
 	// 1 1 8080 0a585858.addr.dc1.consul.
 	// 1 1 8080 0a4d4d4d.addr.dc1.consul.
-	SRVLookup *struct {
-		Service string
-		Domain  string
-	}
+	SRVLookup *ServiceDomain
+
+	// SRVLookups allows you to pass multiple SRV records to the gRPC client.
+	// The gRPC client will resolves each SRV record and use the results to
+	// construct a list of backends to connect to. For more details, see the
+	// documentation for the SRVLookup field. Note: while you can pass multiple
+	// targets to the gRPC client using this field, all of the targets will use
+	// the same HostOverride and TLS configuration.
+	SRVLookups []*ServiceDomain
+
+	// SRVResolver is an optional override to indicate that a specific
+	// implementation of the SRV resolver should be used. The default is 'srv'
+	// For more details, see the documentation in:
+	// grpc/internal/resolver/dns/dns_resolver.go.
+	SRVResolver string
 
 	// ServerAddress is a single <hostname|IPv4|[IPv6]>:<port> or `:<port>` that
 	// the gRPC client will, if necessary, resolve via DNS and then connect to.
@@ -359,6 +379,10 @@ func (c *GRPCClientConfig) MakeTargetAndHostOverride() (string, string, error) {
 		return fmt.Sprintf("dns://%s/%s", c.DNSAuthority, c.ServerAddress), hostOverride, nil
 
 	} else if c.SRVLookup != nil {
+		scheme, err := c.makeSRVScheme()
+		if err != nil {
+			return "", "", err
+		}
 		if c.ServerIPAddresses != nil {
 			return "", "", errors.New(
 				"both 'SRVLookup' and 'serverIPAddresses' in gRPC client config. Only one should be provided",
@@ -371,19 +395,53 @@ func (c *GRPCClientConfig) MakeTargetAndHostOverride() (string, string, error) {
 		if c.HostOverride != "" {
 			hostOverride = c.HostOverride
 		}
-		return fmt.Sprintf("srv://%s/%s", c.DNSAuthority, targetHost), hostOverride, nil
+		return fmt.Sprintf("%s://%s/%s", scheme, c.DNSAuthority, targetHost), hostOverride, nil
+
+	} else if c.SRVLookups != nil {
+		scheme, err := c.makeSRVScheme()
+		if err != nil {
+			return "", "", err
+		}
+		if c.ServerIPAddresses != nil {
+			return "", "", errors.New(
+				"both 'SRVLookups' and 'serverIPAddresses' in gRPC client config. Only one should be provided",
+			)
+		}
+		// Lookup backends using multiple DNS SRV records.
+		var targetHosts []string
+		for _, s := range c.SRVLookups {
+			targetHosts = append(targetHosts, s.Service+"."+s.Domain)
+		}
+		if c.HostOverride != "" {
+			hostOverride = c.HostOverride
+		}
+		return fmt.Sprintf("%s://%s/%s", scheme, c.DNSAuthority, strings.Join(targetHosts, ",")), hostOverride, nil
 
 	} else {
 		if c.ServerIPAddresses == nil {
 			return "", "", errors.New(
-				"neither 'serverAddress', 'SRVLookup' nor 'serverIPAddresses' in gRPC client config. One should be provided",
+				"neither 'serverAddress', 'SRVLookup', 'SRVLookups' nor 'serverIPAddresses' in gRPC client config. One should be provided",
 			)
 		}
 		// Specify backends as a list of IP addresses.
 		return "static:///" + strings.Join(c.ServerIPAddresses, ","), "", nil
 	}
 }
 
+// makeSRVScheme returns the scheme to use for SRV lookups. If the SRVResolver
+// field is empty, it returns "srv". Otherwise it checks that the specified
+// SRVResolver is registered with the gRPC runtime and returns it.
+func (c *GRPCClientConfig) makeSRVScheme() (string, error) {
+	if c.SRVResolver == "" {
+		return "srv", nil
+	}
+	rb := resolver.Get(c.SRVResolver)
+	if rb == nil {
+		return "", fmt.Errorf("resolver %q is not registered", c.SRVResolver)
+	}
+	return c.SRVResolver, nil
+}
+
 // GRPCServerConfig contains the information needed to start a gRPC server.
 type GRPCServerConfig struct {
 	Address string `json:"address"`

cmd/nonce-service/main.go

@@ -2,6 +2,8 @@ package notmain
 
 import (
 	"flag"
+	"fmt"
+	"net"
 
 	"github.com/honeycombio/beeline-go"
 
@@ -15,18 +17,53 @@ type Config struct {
 	NonceService struct {
 		cmd.ServiceConfig
 
-		MaxUsed     int
+		MaxUsed int
+		// TODO(#6610): Remove once we've moved to derivable prefixes by
+		// default.
 		NoncePrefix string
 
+		// UseDerivablePrefix indicates whether to use a nonce prefix derived
+		// from the gRPC listening address. If this is false, the nonce prefix
+		// will be the value of the NoncePrefix field. If this is true, the
+		// NoncePrefixKey field is required.
+		//
+		// TODO(#6610): Remove once we've moved to derivable prefixes by
+		// default.
+		UseDerivablePrefix bool
+
+		// NoncePrefixKey is a secret used for deriving the prefix of each nonce
+		// instance. It should contain 256 bits (32 bytes) of random data to be
+		// suitable as an HMAC-SHA256 key (e.g. the output of `openssl rand -hex
+		// 32`). In a multi-DC deployment this value should be the same across
+		// all boulder-wfe and nonce-service instances. This is only used if
+		// UseDerivablePrefix is true.
+		//
+		// TODO(#6610): Edit this comment once we've moved to derivable prefixes
+		// by default.
+		NoncePrefixKey cmd.PasswordConfig
+
 		Syslog  cmd.SyslogConfig
 		Beeline cmd.BeelineConfig
 	}
 }
 
+func derivePrefix(key string, grpcAddr string) (string, error) {
+	host, port, err := net.SplitHostPort(grpcAddr)
+	if err != nil {
+		return "", fmt.Errorf("parsing gRPC listen address: %w", err)
+	}
+	if host != "" && port != "" {
+		hostIP := net.ParseIP(host)
+		if hostIP == nil {
+			return "", fmt.Errorf("parsing IP from gRPC listen address: %w", err)
+		}
+	}
+	return nonce.DerivePrefix(grpcAddr, key), nil
+}
+
 func main() {
 	grpcAddr := flag.String("addr", "", "gRPC listen address override")
 	debugAddr := flag.String("debug-addr", "", "Debug server address override")
-	prefixOverride := flag.String("prefix", "", "Override the configured nonce prefix")
 	configFile := flag.String("config", "", "File path to the configuration file for this service")
 	flag.Parse()
 
@@ -40,8 +77,22 @@ func main() {
 	if *debugAddr != "" {
 		c.NonceService.DebugAddr = *debugAddr
 	}
-	if *prefixOverride != "" {
-		c.NonceService.NoncePrefix = *prefixOverride
+
+	// TODO(#6610): Remove once we've moved to derivable prefixes by default.
+	if c.NonceService.NoncePrefix != "" && c.NonceService.UseDerivablePrefix {
+		cmd.Fail("Cannot set both 'noncePrefix' and 'useDerivablePrefix'")
+	}
+
+	// TODO(#6610): Remove once we've moved to derivable prefixes by default.
+	if c.NonceService.UseDerivablePrefix && c.NonceService.NoncePrefixKey.PasswordFile == "" {
+		cmd.Fail("Cannot set 'noncePrefixKey' without 'useDerivablePrefix'")
+	}
+
+	if c.NonceService.UseDerivablePrefix && c.NonceService.NoncePrefixKey.PasswordFile != "" {
+		key, err := c.NonceService.NoncePrefixKey.Pass()
+		cmd.FailOnError(err, "Failed to load 'noncePrefixKey' file.")
+		c.NonceService.NoncePrefix, err = derivePrefix(key, c.NonceService.GRPC.Address)
+		cmd.FailOnError(err, "Failed to derive nonce prefix")
 	}
 
 	bc, err := c.NonceService.Beeline.Load()