SA: gRPC methods for leasing CRL shards (#6940)

Add two new methods, LeaseCRLShard and UpdateCRLShard, to the SA gRPC
interface. These methods work in concert both to prevent multiple
instances of crl-updater from stepping on each others toes, and to lay
the groundwork for a less bursty version of crl-updater in the future.

Introduce a new database table, crlShards, which tracks the thisUpdate
and nextUpdate timestamps of each CRL shard for each issuer. It also has
a column "leasedUntil", which is also a timestamp. Grant the SA user
read-write access to this table.

LeaseCRLShard updates the leasedUntil column of the identified shard to
the given time. It returns an error if the identified shard's
leasedUntil timestamp is already in the future. This provides a
mechanism for crl-updater instances to "lick the cookie", so to speak,
marking CRL shards as "taken" so that multiple crl-updater instances
don't attempt to work on the same shard at the same time. Using a
timestamp has the added benefit that leases are guaranteed to expire,
ensuring that we don't accidentally fail to work on a shard forever.

LeaseCRLShard has a second mode of operation, when a range of potential
shards is given in the request, rather than a single shard. In this
mode, it returns the shard (within the given range) whose thisUpdate
timestamp is oldest. (Shards with no thisUpdate timestamp, including
because the requested range includes shard indices the database doesn't
yet know about, count as older than any shard with any thisUpdate
timestamp.) This allows crl-updater instances which don't care which
shard they're working on to do the most urgent work first.

UpdateCRLShard updates the thisUpdate and nextUpdate timestamps of the
identified shard. This closes the loop with the second mode of
LeaseCRLShard above: by updating the thisUpdate timestamp, the method
marks the shard as no longer urgently needing to be worked on.

IN-9220 tracks creating this table in staging and production
Part of #6897

Author: aarongable

mocks/mocks.go

@@ -245,7 +245,7 @@ func (sa *StorageAuthorityReadOnly) GetCertificateStatus(_ context.Context, req
 			Status: string(core.OCSPStatusRevoked),
 		}, nil
 	} else {
-		return nil, errors.New("No cert status")
+		return nil, errors.New("no cert status")
 	}
 }
 
@@ -355,7 +355,6 @@ func (sa *StorageAuthority) DeactivateRegistration(_ context.Context, _ *sapb.Re
 
 // NewOrderAndAuthzs is a mock
 func (sa *StorageAuthority) NewOrderAndAuthzs(_ context.Context, req *sapb.NewOrderAndAuthzsRequest, _ ...grpc.CallOption) (*corepb.Order, error) {
-	rand.Seed(time.Now().UnixNano())
 	response := &corepb.Order{
 		// Fields from the input new order request.
 		RegistrationID:   req.NewOrder.RegistrationID,
@@ -556,7 +555,7 @@ func (sa *StorageAuthorityReadOnly) GetAuthorization2(ctx context.Context, id *s
 		authz.ID = fmt.Sprintf("%d", authzIdExpired)
 		return bgrpc.AuthzToPB(authz)
 	case authzIdErrorResult:
-		return nil, fmt.Errorf("Unspecified database error")
+		return nil, fmt.Errorf("unspecified database error")
 	case authzIdDiffAccount:
 		exp := sa.clk.Now().AddDate(100, 0, 0)
 		authz.RegistrationID = 2
@@ -593,6 +592,16 @@ func (sa *StorageAuthorityReadOnly) IncidentsForSerial(ctx context.Context, req
 	return &sapb.Incidents{}, nil
 }
 
+// LeaseCRLShard is a mock.
+func (sa *StorageAuthority) LeaseCRLShard(ctx context.Context, req *sapb.LeaseCRLShardRequest, _ ...grpc.CallOption) (*sapb.LeaseCRLShardResponse, error) {
+	return nil, errors.New("unimplemented")
+}
+
+// UpdateCRLShard is a mock.
+func (sa *StorageAuthority) UpdateCRLShard(ctx context.Context, req *sapb.UpdateCRLShardRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) {
+	return nil, errors.New("unimplemented")
+}
+
 // Publisher is a mock
 type PublisherClient struct {
 	// empty

sa/database.go

@@ -281,6 +281,7 @@ func initTables(dbMap *gorp.DbMap) {
 	dbMap.AddTableWithName(keyHashModel{}, "keyHashToSerial").SetKeys(true, "ID")
 	dbMap.AddTableWithName(incidentModel{}, "incidents").SetKeys(true, "ID")
 	dbMap.AddTable(incidentSerialModel{})
+	dbMap.AddTableWithName(crlShardModel{}, "crlShards").SetKeys(true, "ID")
 
 	// Read-only maps used for selecting subsets of columns.
 	dbMap.AddTableWithName(CertStatusMetadata{}, "certificateStatus")

sa/db-next/boulder_sa/20230519000000_CrlShards.sql

@@ -0,0 +1,18 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+CREATE TABLE `crlShards` (
+  `id` bigint(20) UNSIGNED NOT NULL AUTO_INCREMENT,
+  `issuerID` bigint(20) NOT NULL,
+  `idx` int UNSIGNED NOT NULL,
+  `thisUpdate` datetime,
+  `nextUpdate` datetime,
+  `leasedUntil` datetime NOT NULL,
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `shardID` (`issuerID`, `idx`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+DROP TABLE `crlShards`;

sa/db-users/boulder_sa.sql

@@ -32,6 +32,7 @@ GRANT SELECT,INSERT ON keyHashToSerial TO 'sa'@'localhost';
 GRANT SELECT,INSERT ON blockedKeys TO 'sa'@'localhost';
 GRANT SELECT,INSERT,UPDATE ON newOrdersRL TO 'sa'@'localhost';
 GRANT SELECT ON incidents TO 'sa'@'localhost';
+GRANT SELECT,INSERT,UPDATE ON crlShards TO 'sa'@'localhost';
 
 GRANT SELECT ON certificates TO 'sa_ro'@'localhost';
 GRANT SELECT ON certificateStatus TO 'sa_ro'@'localhost';
@@ -50,6 +51,7 @@ GRANT SELECT ON keyHashToSerial TO 'sa_ro'@'localhost';
 GRANT SELECT ON blockedKeys TO 'sa_ro'@'localhost';
 GRANT SELECT ON newOrdersRL TO 'sa_ro'@'localhost';
 GRANT SELECT ON incidents TO 'sa_ro'@'localhost';
+GRANT SELECT ON crlShards TO 'sa_ro'@'localhost';
 
 -- OCSP Responder
 GRANT SELECT ON certificateStatus TO 'ocsp_resp'@'localhost';

sa/model.go

@@ -1168,3 +1168,14 @@ func namesForOrder(s db.Selector, orderID int64) ([]string, error) {
 	}
 	return reversedNames, nil
 }
+
+// crlShardModel represents one row in the crlShards table. The ThisUpdate and
+// NextUpdate fields are pointers because they are NULL-able columns.
+type crlShardModel struct {
+	ID          int64      `db:"id"`
+	IssuerID    int64      `db:"issuerID"`
+	Idx         int        `db:"idx"`
+	ThisUpdate  *time.Time `db:"thisUpdate"`
+	NextUpdate  *time.Time `db:"nextUpdate"`
+	LeasedUntil time.Time  `db:"leasedUntil"`
+}