Merge remote-tracking branch 'origin/main' into add-vitess

Author: beautifulentropy

ca/ca_test.go

@@ -134,21 +134,19 @@ func newCAArgs(t *testing.T) *caArgs {
 	test.AssertNotError(t, err, "Couldn't set identifier policy")
 
 	legacy, err := issuance.NewProfile(issuance.ProfileConfig{
-		IncludeCRLDistributionPoints: true,
-		MaxValidityPeriod:            config.Duration{Duration: time.Hour * 24 * 90},
-		MaxValidityBackdate:          config.Duration{Duration: time.Hour},
-		IgnoredLints:                 []string{"w_subject_common_name_included"},
+		MaxValidityPeriod:   config.Duration{Duration: time.Hour * 24 * 90},
+		MaxValidityBackdate: config.Duration{Duration: time.Hour},
+		IgnoredLints:        []string{"w_subject_common_name_included"},
 	})
 	test.AssertNotError(t, err, "Loading test profile")
 	modern, err := issuance.NewProfile(issuance.ProfileConfig{
-		OmitCommonName:               true,
-		OmitKeyEncipherment:          true,
-		OmitClientAuth:               true,
-		OmitSKID:                     true,
-		IncludeCRLDistributionPoints: true,
-		MaxValidityPeriod:            config.Duration{Duration: time.Hour * 24 * 6},
-		MaxValidityBackdate:          config.Duration{Duration: time.Hour},
-		IgnoredLints:                 []string{"w_ext_subject_key_identifier_missing_sub_cert"},
+		OmitCommonName:      true,
+		OmitKeyEncipherment: true,
+		OmitClientAuth:      true,
+		OmitSKID:            true,
+		MaxValidityPeriod:   config.Duration{Duration: time.Hour * 24 * 6},
+		MaxValidityBackdate: config.Duration{Duration: time.Hour},
+		IgnoredLints:        []string{"w_ext_subject_key_identifier_missing_sub_cert"},
 	})
 	test.AssertNotError(t, err, "Loading test profile")
 	profiles := map[string]*issuance.Profile{
@@ -161,7 +159,6 @@ func newCAArgs(t *testing.T) *caArgs {
 		issuers[i], err = issuance.LoadIssuer(issuance.IssuerConfig{
 			Active:     true,
 			IssuerURL:  fmt.Sprintf("http://not-example.com/i/%s", name),
-			OCSPURL:    "http://not-example.com/o",
 			CRLURLBase: fmt.Sprintf("http://not-example.com/c/%s/", name),
 			CRLShards:  10,
 			Location: issuance.IssuerLoc{
@@ -886,7 +883,6 @@ func TestPickIssuer_Inactive(t *testing.T) {
 		issuer, err := issuance.LoadIssuer(issuance.IssuerConfig{
 			Active:     i%2 == 0,
 			IssuerURL:  fmt.Sprintf("http://not-example.com/i/%s", name),
-			OCSPURL:    "http://not-example.com/o",
 			CRLURLBase: fmt.Sprintf("http://not-example.com/c/%s/", name),
 			CRLShards:  10,
 			Location: issuance.IssuerLoc{

cmd/boulder-ca/main.go

@@ -12,7 +12,6 @@ import (
 	"github.com/letsencrypt/boulder/ca"
 	capb "github.com/letsencrypt/boulder/ca/proto"
 	"github.com/letsencrypt/boulder/cmd"
-	"github.com/letsencrypt/boulder/config"
 	"github.com/letsencrypt/boulder/ctpolicy/loglist"
 	"github.com/letsencrypt/boulder/features"
 	"github.com/letsencrypt/boulder/goodkey"
@@ -75,12 +74,6 @@ type Config struct {
 		// configurations.
 		MaxNames int `validate:"required,min=1,max=100"`
 
-		// LifespanOCSP is how long OCSP responses are valid for. Per the BRs,
-		// Section 4.9.10, it MUST NOT be more than 10 days. Default 96h.
-		//
-		// Deprecated: TODO(#8345): Remove this.
-		LifespanOCSP config.Duration
-
 		// GoodKey is an embedded config stanza for the goodkey library.
 		GoodKey goodkey.Config
 
@@ -89,17 +82,6 @@ type Config struct {
 		// OCSP and CRL audit log emission. Recommended to be around 4000.
 		OCSPLogMaxLength int
 
-		// Maximum period (in Go duration format) to wait to accumulate a max-length
-		// OCSP audit log line. We will emit a log line at least once per period,
-		// if there is anything to be logged. Keeping this low minimizes the risk
-		// of losing logs during a catastrophic failure. Making it too high
-		// means logging more often than necessary, which is inefficient in terms
-		// of bytes and log system resources.
-		// Recommended to be around 500ms.
-		//
-		// Deprecated: TODO(#8345): Remove this.
-		OCSPLogPeriod config.Duration
-
 		// CTLogListFile is the path to a JSON file on disk containing the set of
 		// all logs trusted by Chrome. The file must match the v3 log list schema:
 		// https://www.gstatic.com/ct/log_list/v3/log_list_schema.json
@@ -108,11 +90,7 @@ type Config struct {
 		// DisableCertService causes the CertificateAuthority gRPC service to not
 		// start, preventing any certificates or precertificates from being issued.
 		DisableCertService bool
-		// DisableOCSPService causes the OCSPGenerator gRPC service to not start,
-		// preventing any OCSP responses from being issued.
-		//
-		// Deprecated: TODO(#8345): Remove this.
-		DisableOCSPService bool
+
 		// DisableCRLService causes the CRLGenerator gRPC service to not start,
 		// preventing any CRLs from being issued.
 		DisableCRLService bool

cmd/boulder-ra/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"flag"
 	"os"
+	"time"
 
 	"github.com/jmhodges/clock"
 
@@ -44,9 +45,6 @@ type Config struct {
 		CAService        *cmd.GRPCClientConfig
 		PublisherService *cmd.GRPCClientConfig
 
-		// Deprecated: TODO(#8345): Remove this.
-		AkamaiPurgerService *cmd.GRPCClientConfig
-
 		// Deprecated: TODO(#8349): Remove this when removing the corresponding
 		// service from the CA.
 		OCSPService *cmd.GRPCClientConfig
@@ -69,13 +67,17 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides passed in this file must be
-			// identical to those in the WFE.
+			// neither this field nor OverridesFromDB is set, all requesters
+			// will be subject to the default rate limits. Overrides passed in
+			// this file must be identical to those in the WFE.
 			//
 			// Note: At this time, only the Failed Authorizations overrides are
 			// necessary in the RA.
 			Overrides string
+
+			// OverridesFromDB causes the WFE and RA to retrieve rate limit overrides
+			// from the database, instead of from a file.
+			OverridesFromDB bool
 		}
 
 		// MaxNames is the maximum number of subjectAltNames in a single cert.
@@ -100,15 +102,6 @@ type Config struct {
 		// default.
 		DefaultProfileName string `validate:"required"`
 
-		// MustStapleAllowList specified the path to a YAML file containing a
-		// list of account IDs permitted to request certificates with the OCSP
-		// Must-Staple extension.
-		//
-		// Deprecated: This field no longer has any effect, all Must-Staple requests
-		// are rejected.
-		// TODO(#8345): Remove this field.
-		MustStapleAllowList string `validate:"omitempty"`
-
 		// GoodKey is an embedded config stanza for the goodkey library.
 		GoodKey goodkey.Config
 
@@ -271,8 +264,18 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, scope)
 		limiter, err = ratelimits.NewLimiter(clk, source, scope)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides)
+		if c.RA.Limiter.OverridesFromDB {
+			if c.RA.Limiter.Overrides != "" {
+				cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
+			}
+			saroc := sapb.NewStorageAuthorityReadOnlyClient(saConn)
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.RA.Limiter.Defaults, saroc.GetEnabledRateLimitOverrides, scope, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.RA.Limiter.Defaults, c.RA.Limiter.Overrides, scope, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overrideRefresherShutdown := txnBuilder.NewRefresher(30 * time.Minute)
+		defer overrideRefresherShutdown()
 	}
 
 	rai := ra.NewRegistrationAuthorityImpl(

cmd/boulder-wfe2/main.go

@@ -151,11 +151,15 @@ type Config struct {
 
 			// Overrides is a path to a YAML file containing overrides for the
 			// default rate limits. See: ratelimits/README.md for details. If
-			// this field is not set, all requesters will be subject to the
-			// default rate limits. Overrides for the Failed Authorizations
-			// overrides passed in this file must be identical to those in the
-			// RA.
+			// neither this field nor OverridesFromDB is set, all requesters
+			// will be subject to the default rate limits. Overrides for the
+			// Failed Authorizations overrides passed in this file must be
+			// identical to those in the RA.
 			Overrides string
+
+			// OverridesFromDB causes the WFE and RA to retrieve rate limit
+			// overrides from the database, instead of from a file.
+			OverridesFromDB bool
 		}
 
 		// CertProfiles is a map of acceptable certificate profile names to
@@ -326,6 +330,7 @@ func main() {
 	var limiter *ratelimits.Limiter
 	var txnBuilder *ratelimits.TransactionBuilder
 	var limiterRedis *bredis.Ring
+	overridesRefresherShutdown := func() {}
 	if c.WFE.Limiter.Defaults != "" {
 		// Setup rate limiting.
 		limiterRedis, err = bredis.NewRingFromConfig(*c.WFE.Limiter.Redis, stats, logger)
@@ -334,8 +339,16 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides)
+		if c.WFE.Limiter.OverridesFromDB {
+			if c.WFE.Limiter.Overrides != "" {
+				cmd.Fail("OverridesFromDB and an overrides file were both defined, but are mutually exclusive")
+			}
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromDatabase(c.WFE.Limiter.Defaults, sac.GetEnabledRateLimitOverrides, stats, logger)
+		} else {
+			txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.WFE.Limiter.Defaults, c.WFE.Limiter.Overrides, stats, logger)
+		}
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
+		overridesRefresherShutdown = txnBuilder.NewRefresher(30 * time.Minute)
 	}
 
 	var accountGetter wfe2.AccountGetter
@@ -413,6 +426,7 @@ func main() {
 	defer func() {
 		ctx, cancel := context.WithTimeout(context.Background(), c.WFE.ShutdownStopTimeout.Duration)
 		defer cancel()
+		overridesRefresherShutdown()
 		_ = srv.Shutdown(ctx)
 		_ = tlsSrv.Shutdown(ctx)
 		limiterRedis.StopLookups()

cmd/crl-updater/main.go

@@ -72,21 +72,6 @@ type Config struct {
 		// of magnitude greater than our p99 update latency.
 		UpdateTimeout config.Duration `validate:"-"`
 
-		// TemporallyShardedSerialPrefixes is a list of prefixes that were used to
-		// issue certificates with no CRLDistributionPoints extension, and which are
-		// therefore temporally sharded. If it's non-empty, the CRL Updater will
-		// require matching serials when querying by temporal shard. When querying
-		// by explicit shard, any prefix is allowed.
-		//
-		// This should be set to the current set of serial prefixes in production.
-		// When deploying explicit sharding (i.e. the CRLDistributionPoints extension),
-		// the CAs should be configured with a new set of serial prefixes that haven't
-		// been used before.
-		//
-		// Deprecated: The updater no longer supports temporal sharding.
-		// TODO(#8345): Remove this.
-		TemporallyShardedSerialPrefixes []string
-
 		// MaxParallelism controls how many workers may be running in parallel.
 		// A higher value reduces the total time necessary to update all CRL shards
 		// that this updater is responsible for, but also increases the memory used

cmd/email-exporter/main.go

@@ -45,7 +45,7 @@ type Config struct {
 		ClientSecret cmd.PasswordConfig
 
 		// SalesforceBaseURL is the base URL for the Salesforce API. (e.g.,
-		// "https://login.salesforce.com")
+		// "https://company.salesforce.com")
 		SalesforceBaseURL string `validate:"required"`
 
 		// PardotBaseURL is the base URL for the Pardot API. (e.g.,
@@ -100,7 +100,7 @@ func main() {
 		cache = email.NewHashedEmailCache(c.EmailExporter.EmailCacheSize, scope)
 	}
 
-	pardotClient, err := email.NewPardotClientImpl(
+	sfClient, err := email.NewSalesforceClientImpl(
 		clk,
 		c.EmailExporter.PardotBusinessUnit,
 		clientId,
@@ -109,7 +109,7 @@ func main() {
 		c.EmailExporter.PardotBaseURL,
 	)
 	cmd.FailOnError(err, "Creating Pardot API client")
-	exporterServer := email.NewExporterImpl(pardotClient, cache, c.EmailExporter.PerDayLimit, c.EmailExporter.MaxConcurrentRequests, scope, logger)
+	exporterServer := email.NewExporterImpl(sfClient, cache, c.EmailExporter.PerDayLimit, c.EmailExporter.MaxConcurrentRequests, scope, logger)
 
 	tlsConfig, err := c.EmailExporter.TLS.Load(scope)
 	cmd.FailOnError(err, "Loading email-exporter TLS config")

cmd/sfe/main.go

@@ -223,7 +223,7 @@ func main() {
 		source := ratelimits.NewRedisSource(limiterRedis.Ring, clk, stats)
 		limiter, err = ratelimits.NewLimiter(clk, source, stats)
 		cmd.FailOnError(err, "Failed to create rate limiter")
-		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "")
+		txnBuilder, err = ratelimits.NewTransactionBuilderFromFiles(c.SFE.Limiter.Defaults, "", stats, logger)
 		cmd.FailOnError(err, "Failed to create rate limits transaction builder")
 	}
 

email/exporter.go

@@ -39,10 +39,11 @@ type ExporterImpl struct {
 
 	maxConcurrentRequests int
 	limiter               *rate.Limiter
-	client                PardotClient
+	client                SalesforceClient
 	emailCache            *EmailCache
 	emailsHandledCounter  prometheus.Counter
 	pardotErrorCounter    prometheus.Counter
+	caseErrorCounter      prometheus.Counter
 	log                   blog.Logger
 }
 
@@ -55,7 +56,7 @@ var _ emailpb.ExporterServer = (*ExporterImpl)(nil)
 // is assigned 40% (20,000 requests), it should also receive 40% of the max
 // concurrent requests (e.g., 2 out of 5). For more details, see:
 // https://developer.salesforce.com/docs/marketing/pardot/guide/overview.html?q=rate%20limits
-func NewExporterImpl(client PardotClient, cache *EmailCache, perDayLimit float64, maxConcurrentRequests int, scope prometheus.Registerer, logger blog.Logger) *ExporterImpl {
+func NewExporterImpl(client SalesforceClient, cache *EmailCache, perDayLimit float64, maxConcurrentRequests int, scope prometheus.Registerer, logger blog.Logger) *ExporterImpl {
 	limiter := rate.NewLimiter(rate.Limit(perDayLimit/86400.0), maxConcurrentRequests)
 
 	emailsHandledCounter := prometheus.NewCounter(prometheus.CounterOpts{
@@ -70,6 +71,12 @@ func NewExporterImpl(client PardotClient, cache *EmailCache, perDayLimit float64
 	})
 	scope.MustRegister(pardotErrorCounter)
 
+	caseErrorCounter := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "email_exporter_case_errors",
+		Help: "Total number of errors encountered when sending Cases to the Salesforce REST API",
+	})
+	scope.MustRegister(caseErrorCounter)
+
 	impl := &ExporterImpl{
 		maxConcurrentRequests: maxConcurrentRequests,
 		limiter:               limiter,
@@ -78,6 +85,7 @@ func NewExporterImpl(client PardotClient, cache *EmailCache, perDayLimit float64
 		emailCache:            cache,
 		emailsHandledCounter:  emailsHandledCounter,
 		pardotErrorCounter:    pardotErrorCounter,
+		caseErrorCounter:      caseErrorCounter,
 		log:                   logger,
 	}
 	impl.wake = sync.NewCond(&impl.Mutex)
@@ -116,6 +124,33 @@ func (impl *ExporterImpl) SendContacts(ctx context.Context, req *emailpb.SendCon
 	return &emptypb.Empty{}, nil
 }
 
+// SendCase immediately submits a new Case to the Salesforce REST API using the
+// provided details. Any retries are handled internally by the SalesforceClient.
+// The following fields are required: Origin, Subject, ContactEmail.
+func (impl *ExporterImpl) SendCase(ctx context.Context, req *emailpb.SendCaseRequest) (*emptypb.Empty, error) {
+	if core.IsAnyNilOrZero(req, req.Origin, req.Subject, req.ContactEmail) {
+		return nil, berrors.InternalServerError("incomplete gRPC request message")
+	}
+
+	err := impl.client.SendCase(Case{
+		Origin:        req.Origin,
+		Subject:       req.Subject,
+		Description:   req.Description,
+		ContactEmail:  req.ContactEmail,
+		Organization:  req.Organization,
+		AccountId:     req.AccountId,
+		RateLimitName: req.RateLimitName,
+		RateLimitTier: req.RateLimitTier,
+		UseCase:       req.UseCase,
+	})
+	if err != nil {
+		impl.caseErrorCounter.Inc()
+		return nil, berrors.InternalServerError("sending Case to the Salesforce REST API: %s", err)
+	}
+
+	return &emptypb.Empty{}, nil
+}
+
 // Start begins asynchronous processing of the email queue. When the parent
 // daemonCtx is cancelled the queue will be drained and the workers will exit.
 func (impl *ExporterImpl) Start(daemonCtx context.Context) {