Progress on reading orders from new schema

Author: aarongable

core/util.go

@@ -27,9 +27,10 @@ import (
 	"unicode"
 
 	"github.com/go-jose/go-jose/v4"
-	"github.com/letsencrypt/boulder/identifier"
 	"google.golang.org/protobuf/types/known/durationpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
+
+	"github.com/letsencrypt/boulder/identifier"
 )
 
 const Unspecified = "Unspecified"

sa/database.go

@@ -291,6 +291,10 @@ func initTables(dbMap *borp.DbMap) {
 	dbMap.AddTableWithName(revokedCertModel{}, "revokedCertificates").SetKeys(true, "ID")
 	dbMap.AddTableWithName(replacementOrderModel{}, "replacementOrders").SetKeys(true, "ID")
 	dbMap.AddTableWithName(pausedModel{}, "paused")
+	dbMap.AddTableWithName(orders2Model{}, "orders2")
+	dbMap.AddTableWithName(authorizationsModel{}, "authorizations")
+	dbMap.AddTableWithName(validationsModel{}, "validations")
+	dbMap.AddTableWithName(authzReuseModel{}, "authzReuse")
 
 	// Read-only maps used for selecting subsets of columns.
 	dbMap.AddTableWithName(CertStatusMetadata{}, "certificateStatus")

sa/db-next/boulder_sa/20240801000000_OrderSchema.sql

@@ -14,7 +14,7 @@ CREATE TABLE `orders2` (
   `registrationID` bigint(20) UNSIGNED NOT NULL,
   `created` datetime NOT NULL,
   `expires` datetime NOT NULL,
-  `authorizations` json NOT NULL,
+  `authorizationIDs` json NOT NULL,
   `profile` varchar(255) NOT NULL,
   `beganProcessing` boolean NOT NULL,
   `error` mediumblob DEFAULT NULL,
@@ -40,7 +40,7 @@ CREATE TABLE `authorizations` (
   `challenges` tinyint(4) NOT NULL,
   `token` binary(32) NOT NULL,
   `status` tinyint(4) NOT NULL,
-  `validations` json DEFAULT NULL,
+  `validationIDs` json DEFAULT NULL,
   PRIMARY KEY (`id`),
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
  PARTITION BY RANGE(id)
@@ -52,7 +52,6 @@ CREATE TABLE `authorizations` (
 -- invalid), and an opaque blob of our audit record.
 CREATE TABLE `validations` (
   `id` bigint(20) UNSIGNED NOT NULL,
-  `registrationID` bigint(20) UNSIGNED NOT NULL,
   `challenge` tinyint(4) NOT NULL,
   `attemptedAt` datetime NOT NULL,
   `status` tinyint(4) NOT NULL,
@@ -66,7 +65,7 @@ CREATE TABLE `validations` (
 -- IDs. This allos us to not have expensive indices on the authorizations table.
 CREATE TABLE `authzReuse` (
   `accountID_identifier` VARCHAR(300) NOT NULL,
-  `authzID` VARCHAR(255) NOT NULL,
+  `authzID` bigint(20) UNSIGNED NOT NULL,
   `expires` DATETIME NOT NULL
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
  PARTITION BY RANGE(id)

sa/model.go

@@ -2,10 +2,12 @@ package sa
 
 import (
 	"context"
+	"crypto/rand"
 	"crypto/sha256"
 	"crypto/x509"
 	"database/sql"
 	"encoding/base64"
+	"encoding/binary"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -17,6 +19,7 @@ import (
 	"time"
 
 	"github.com/go-jose/go-jose/v4"
+	"github.com/jmhodges/clock"
 	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/letsencrypt/boulder/core"
@@ -59,6 +62,54 @@ func badJSONError(msg string, jsonData []byte, err error) error {
 	}
 }
 
+// newRandomID creates a 64-bit mostly-random number to be used as the
+// unique ID column in a table which no longer uses auto_increment IDs. It takes
+// the clock as an argument so that it can include the current "epoch" as the
+// first byte of the ID, for the sake of easily dropping old data.
+func newRandomID(clk clock.Clock) (int64, error) {
+	idBytes := make([]byte, 8) // 8 bytes is 64 bits
+
+	// Read random bits into the lower 7 bytes of the id.
+	_, err := rand.Read(idBytes[1:])
+	if err != nil {
+		return 0, fmt.Errorf("while generating unique database id: %w", err)
+	}
+
+	// Epochs are arbitrarily chosen to be 90 day chunks counting from the start
+	// of 2024. This gives us 127 * 90 = ~31 years worth of epochs before we have
+	// to worry about a rollover.
+	epoch := uint8(clk.Now().Sub(time.Date(2024, 01, 01, 00, 00, 00, 00, time.UTC)) / (90 * 24 * time.Hour))
+	if epoch&0x80 != 0 {
+		// If the first bit is a 1, either the current date is before the epoch
+		// start date, or we've gone too far into the future. Error out before we
+		// accidentally generate a negative ID.
+		return 0, fmt.Errorf("invalid epoch: %d", epoch)
+	}
+	idBytes[0] = epoch
+
+	id := binary.BigEndian.Uint64(idBytes)
+	return int64(id), nil
+}
+
+// looksLikeRandomID returns true if the input ID looks like it might belong to
+// the new schema which uses epoch-prefixed random IDs instead of auto-increment
+// columns. This is only necessary during the migration period when we are
+// reading from both the old and new schemas simultaneously.
+func looksLikeRandomID(id int64, clk clock.Clock) bool {
+	// Compute the current and previous epochs. If the input ID starts with one of
+	// those two epochs, it's one of ours. Otherwise, it came from somewhere
+	// unknown and we should ask the old schema about it just in case.
+	currEpoch := uint8(clk.Now().Sub(time.Date(2024, 01, 01, 00, 00, 00, 00, time.UTC)) / (90 * 24 * time.Hour))
+	prevEpoch := uint8(clk.Now().Add(-90*24*time.Hour).Sub(time.Date(2024, 01, 01, 00, 00, 00, 00, time.UTC)) / (90 * 24 * time.Hour))
+
+	buf := make([]byte, 8)
+	binary.BigEndian.PutUint64(buf, uint64(id))
+	if buf[0] == currEpoch || buf[0] == prevEpoch {
+		return true
+	}
+	return false
+}
+
 const regFields = "id, jwk, jwk_sha256, contact, agreement, initialIP, createdAt, LockCol, status"
 
 // ClearEmail removes the provided email address from one specified registration. If
@@ -1412,3 +1463,47 @@ type pausedModel struct {
 	PausedAt       time.Time  `db:"pausedAt"`
 	UnpausedAt     *time.Time `db:"unpausedAt"`
 }
+
+// orders2Model represents a row in the "orders2" table.
+type orders2Model struct {
+	ID                int64
+	RegistrationID    int64
+	Created           time.Time
+	Expires           time.Time
+	AuthorizationIDs  []int64 // Actually a JSON list of ints
+	Profile           string
+	BeganProcessing   bool
+	Error             []byte
+	CertificateSerial string
+}
+
+// authorizationsModel represents a row in the "authorizations" table.
+type authorizationsModel struct {
+	ID              int64
+	RegistrationID  int64
+	IdentifierType  uint8
+	IdentifierValue string
+	Created         time.Time
+	Expires         time.Time
+	Profile         string
+	Challenges      uint8
+	Token           []byte
+	Status          uint8
+	ValidationIDs   []int64 // Actually a JSON list of ints
+}
+
+// validationsModel represents a row in the "validations" table.
+type validationsModel struct {
+	ID          int64
+	Challenge   uint8
+	AttemptedAt time.Time
+	Status      uint8
+	Record      string
+}
+
+// authzReuseModel represents a row in the "authzReuse" table.
+type authzReuseModel struct {
+	ID      int64 `db:"accountID_identifier"`
+	AuthzID int64
+	Expires time.Time
+}

sa/model_test.go

@@ -8,6 +8,7 @@ import (
 	"crypto/x509/pkix"
 	"database/sql"
 	"encoding/base64"
+	"encoding/binary"
 	"fmt"
 	"math/big"
 	"net"
@@ -28,6 +29,59 @@ import (
 	"github.com/letsencrypt/boulder/test"
 )
 
+func TestNewRandomID(t *testing.T) {
+	t.Parallel()
+
+	testCases := []struct {
+		name         string
+		date         time.Time
+		expectPrefix uint8
+		expectError  string
+	}{
+		{
+			name:        "in the past",
+			date:        time.Date(2023, 01, 01, 00, 00, 00, 00, time.UTC),
+			expectError: "invalid epoch",
+		},
+		{
+			name:         "first epoch",
+			date:         time.Date(2024, 05, 01, 00, 00, 00, 00, time.UTC),
+			expectPrefix: 1,
+		},
+		{
+			name:         "last epoch",
+			date:         time.Date(2055, 07, 01, 00, 00, 00, 00, time.UTC),
+			expectPrefix: 127,
+		},
+		{
+			name:        "far future",
+			date:        time.Date(2056, 01, 01, 00, 00, 00, 00, time.UTC),
+			expectError: "invalid epoch",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			fc := clock.NewFake()
+			fc.Set(tc.date)
+			id, err := newRandomID(fc)
+
+			if tc.expectPrefix != 0 {
+				test.AssertNotError(t, err, "expected success")
+				buf := make([]byte, 8)
+				binary.BigEndian.PutUint64(buf, uint64(id))
+				test.AssertEquals(t, buf[0], tc.expectPrefix)
+			}
+
+			if tc.expectError != "" {
+				test.AssertError(t, err, "expected error")
+				test.AssertContains(t, err.Error(), tc.expectError)
+			}
+		})
+	}
+}
+
 func TestRegistrationModelToPb(t *testing.T) {
 	badCases := []struct {
 		name  string

sa/sa.go

@@ -458,6 +458,14 @@ func (ssa *SQLStorageAuthority) DeactivateAuthorization2(ctx context.Context, re
 // authorizations are created, but then their corresponding order is never
 // created, leading to "invisible" pending authorizations.
 func (ssa *SQLStorageAuthority) NewOrderAndAuthzs(ctx context.Context, req *sapb.NewOrderAndAuthzsRequest) (*corepb.Order, error) {
+	if !features.Get().WriteNewOrderSchema {
+		return ssa.deprecatedNewOrderAndAuthzs(ctx, req)
+	}
+
+	return nil, nil
+}
+
+func (ssa *SQLStorageAuthority) deprecatedNewOrderAndAuthzs(ctx context.Context, req *sapb.NewOrderAndAuthzsRequest) (*corepb.Order, error) {
 	if req.NewOrder == nil {
 		return nil, errIncompleteRequest
 	}

sa/saro.go

@@ -576,7 +576,83 @@ func (ssa *SQLStorageAuthorityRO) checkFQDNSetExists(ctx context.Context, select
 }
 
 // GetOrder is used to retrieve an already existing order object
+// TODO XXX TKTK Update this method
 func (ssa *SQLStorageAuthorityRO) GetOrder(ctx context.Context, req *sapb.OrderRequest) (*corepb.Order, error) {
+	if !features.Get().ReadNewOrderSchema {
+		return ssa.deprecatedGetOrder(ctx, req)
+	}
+
+	if req == nil || req.Id == 0 {
+		return nil, errIncompleteRequest
+	}
+
+	if !looksLikeRandomID(req.Id, ssa.clk) {
+		return ssa.deprecatedGetOrder(ctx, req)
+	}
+
+	output, err := db.WithTransaction(ctx, ssa.dbReadOnlyMap, func(tx db.Executor) (interface{}, error) {
+		oi, err := tx.Get(ctx, orders2Model{}, req.Id)
+		if err != nil {
+			if db.IsNoRows(err) {
+				return nil, berrors.NotFoundError("no order found for ID %d", req.Id)
+			}
+			return nil, err
+		}
+		om := oi.(orders2Model)
+
+		if om.Expires.Before(ssa.clk.Now()) {
+			return nil, berrors.NotFoundError("no order found for ID %d", req.Id)
+		}
+
+		avis := make([]authzValidity, len(om.AuthorizationIDs))
+		dnsNames := make([]string, len(om.AuthorizationIDs))
+		for i, authzId := range om.AuthorizationIDs {
+			ai, err := tx.Get(ctx, authorizationsModel{}, authzId)
+			if err != nil {
+				if db.IsNoRows(err) {
+					return nil, berrors.NotFoundError("no authorization found for ID %d", authzId)
+				}
+				return nil, err
+			}
+			am := ai.(authorizationsModel)
+
+			avis[i] = authzValidity{
+				IdentifierType:  am.IdentifierType,
+				IdentifierValue: am.IdentifierValue,
+				Status:          am.Status,
+				Expires:         am.Expires,
+			}
+			dnsNames[i] = am.IdentifierValue
+		}
+
+		order := corepb.Order{
+			Id:                     om.ID,
+			RegistrationID:         om.RegistrationID,
+			Expires:                timestamppb.New(om.Expires),
+			DnsNames:               dnsNames,
+			Error:                  om.Error,
+			V2Authorizations:       om.AuthorizationIDs,
+			CertificateSerial:      om.CertificateSerial,
+			Created:                timestamppb.New(om.Created),
+			CertificateProfileName: om.Profile,
+			BeganProcessing:        om.BeganProcessing,
+		}
+
+		status, err := statusForOrder(&order, avis, ssa.clk.Now())
+		order.Status = status
+
+		return &order, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	res := output.(*corepb.Order)
+	return res, nil
+}
+
+// deprecatedGetOrder retrieves an order from the old database schema.
+func (ssa *SQLStorageAuthorityRO) deprecatedGetOrder(ctx context.Context, req *sapb.OrderRequest) (*corepb.Order, error) {
 	if req == nil || req.Id == 0 {
 		return nil, errIncompleteRequest
 	}
@@ -735,6 +811,7 @@ func (ssa *SQLStorageAuthorityRO) GetOrderForNames(ctx context.Context, req *sap
 
 // GetAuthorization2 returns the authz2 style authorization identified by the provided ID or an error.
 // If no authorization is found matching the ID a berrors.NotFound type error is returned.
+// TODO XXX TKTK Update this method
 func (ssa *SQLStorageAuthorityRO) GetAuthorization2(ctx context.Context, req *sapb.AuthorizationID2) (*corepb.Authorization, error) {
 	if req.Id == 0 {
 		return nil, errIncompleteRequest

sa/type-converter.go

@@ -20,7 +20,7 @@ type BoulderTypeConverter struct{}
 // ToDb converts a Boulder object to one suitable for the DB representation.
 func (tc BoulderTypeConverter) ToDb(val interface{}) (interface{}, error) {
 	switch t := val.(type) {
-	case identifier.ACMEIdentifier, []core.Challenge, []string, [][]int:
+	case identifier.ACMEIdentifier, []core.Challenge, []string, [][]int, []int64:
 		jsonBytes, err := json.Marshal(t)
 		if err != nil {
 			return nil, err
@@ -56,7 +56,7 @@ func (tc BoulderTypeConverter) ToDb(val interface{}) (interface{}, error) {
 // FromDb converts a DB representation back into a Boulder object.
 func (tc BoulderTypeConverter) FromDb(target interface{}) (borp.CustomScanner, bool) {
 	switch target.(type) {
-	case *identifier.ACMEIdentifier, *[]core.Challenge, *[]string, *[][]int:
+	case *identifier.ACMEIdentifier, *[]core.Challenge, *[]string, *[][]int, *[]int64:
 		binder := func(holder, target interface{}) error {
 			s, ok := holder.(*string)
 			if !ok {