Store authzIDs directly in order table (#8460)

The `orders` table gets a new column `authzs`, containing a
protobuf-encoded list of authorization IDs as int64s. Storing this data
directly in the `orders` table will allow us to get rid of the
`orderToAuthz2` table, which is large (#8451).

There's a new feature flag, `StoreAuthzsInOrders`, which controls
whether the SA expects the new column to exist. The SA gracefully
handles the case where the new column is NULL, by querying the
`orderToAuthz2` table. Eventually all valid orders will have a non-NULL
`authzs` column and we can remove the fallback.

I removed `orderToModel` because it had only one call site, and that
call site only used a subset of fields. This avoids having to update
`orderToModel` to know about encoding authzs, only to have that code
actually go unused in practice.

Author: jsha

features/features.go

@@ -81,6 +81,11 @@ type Config struct {
 	// during certificate issuance. This flag must be set to true in the
 	// RA, VA, and WFE2 services for full functionality.
 	DNSAccount01Enabled bool
+
+	// StoreAuthzsInOrders causes the SA to write to the `authzs`
+	// column in NewOrder and read from it in GetOrder. It should be enabled
+	// after the migration to add that column has been run.
+	StoreAuthzsInOrders bool
 }
 
 var fMu = new(sync.RWMutex)

sa/database.go

@@ -254,6 +254,9 @@ func initTables(dbMap *borp.DbMap) {
 	if !features.Get().StoreARIReplacesInOrders {
 		tableMap.ColMap("Replaces").SetTransient(true)
 	}
+	if !features.Get().StoreAuthzsInOrders {
+		tableMap.ColMap("Authzs").SetTransient(true)
+	}
 
 	dbMap.AddTableWithName(orderToAuthzModel{}, "orderToAuthz").SetKeys(false, "OrderID", "AuthzID")
 	dbMap.AddTableWithName(orderFQDNSet{}, "orderFqdnSets").SetKeys(true, "ID")

sa/db-next/boulder_sa/20251024000000_TheAuthzsAreInTheOrder.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+-- SQL in section 'Up' is executed when this migration is applied
+
+ALTER TABLE `orders` ADD COLUMN `authzs` blob DEFAULT NULL;
+
+-- +migrate Down
+-- SQL section 'Down' is executed when this migration is rolled back
+
+ALTER TABLE `orders` DROP COLUMN `authzs`;

sa/model.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"google.golang.org/protobuf/proto"
 	"math"
 	"net/netip"
 	"net/url"
@@ -346,42 +347,14 @@ type orderModel struct {
 	BeganProcessing        bool
 	CertificateProfileName *string
 	Replaces               *string
+	Authzs                 []byte
 }
 
 type orderToAuthzModel struct {
 	OrderID int64
 	AuthzID int64
 }
 
-func orderToModel(order *corepb.Order) (*orderModel, error) {
-	// Make a local copy so we can take a reference to it below.
-	profile := order.CertificateProfileName
-	replaces := order.Replaces
-
-	om := &orderModel{
-		ID:                     order.Id,
-		RegistrationID:         order.RegistrationID,
-		Expires:                order.Expires.AsTime(),
-		Created:                order.Created.AsTime(),
-		BeganProcessing:        order.BeganProcessing,
-		CertificateSerial:      order.CertificateSerial,
-		CertificateProfileName: &profile,
-		Replaces:               &replaces,
-	}
-
-	if order.Error != nil {
-		errJSON, err := json.Marshal(order.Error)
-		if err != nil {
-			return nil, err
-		}
-		if len(errJSON) > mediumBlobSize {
-			return nil, fmt.Errorf("Error object is too large to store in the database")
-		}
-		om.Error = errJSON
-	}
-	return om, nil
-}
-
 func modelToOrder(om *orderModel) (*corepb.Order, error) {
 	profile := ""
 	if om.CertificateProfileName != nil {
@@ -391,6 +364,13 @@ func modelToOrder(om *orderModel) (*corepb.Order, error) {
 	if om.Replaces != nil {
 		replaces = *om.Replaces
 	}
+	if len(om.Authzs) > 0 {
+		var decodedAuthzs sapb.Authzs
+		err := proto.Unmarshal(om.Authzs, &decodedAuthzs)
+		if err != nil {
+			return nil, err
+		}
+	}
 	order := &corepb.Order{
 		Id:                     om.ID,
 		RegistrationID:         om.RegistrationID,

sa/model_test.go

@@ -223,26 +223,6 @@ func TestModelToOrderBadJSON(t *testing.T) {
 	test.AssertEquals(t, string(badJSONErr.json), string(badJSON))
 }
 
-func TestOrderModelThereAndBackAgain(t *testing.T) {
-	clk := clock.New()
-	now := clk.Now()
-	order := &corepb.Order{
-		Id:                     1,
-		RegistrationID:         2024,
-		Expires:                timestamppb.New(now.Add(24 * time.Hour)),
-		Created:                timestamppb.New(now),
-		Error:                  nil,
-		CertificateSerial:      "2",
-		BeganProcessing:        true,
-		CertificateProfileName: "phljny",
-	}
-	model, err := orderToModel(order)
-	test.AssertNotError(t, err, "orderToModelv2 should not have errored")
-	returnOrder, err := modelToOrder(model)
-	test.AssertNotError(t, err, "modelToOrderv2 should not have errored")
-	test.AssertDeepEquals(t, order, returnOrder)
-}
-
 // TestPopulateAttemptedFieldsBadJSON tests that populating a challenge from an
 // authz2 model with an invalid validation error or an invalid validation record
 // produces the expected bad JSON error.

sa/proto/sadb.pb.go

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+package sa;
+option go_package = "github.com/letsencrypt/boulder/sa/proto";
+
+// Used internally for storage in the DB, not for RPCs.
+message Authzs {
+  repeated int64 authzIDs = 1;
+}

sa/proto/sadb.proto

@@ -7,6 +7,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"google.golang.org/protobuf/proto"
 	"strings"
 	"time"
 
@@ -21,6 +22,7 @@ import (
 	corepb "github.com/letsencrypt/boulder/core/proto"
 	"github.com/letsencrypt/boulder/db"
 	berrors "github.com/letsencrypt/boulder/errors"
+	"github.com/letsencrypt/boulder/features"
 	bgrpc "github.com/letsencrypt/boulder/grpc"
 	"github.com/letsencrypt/boulder/identifier"
 	blog "github.com/letsencrypt/boulder/log"
@@ -486,24 +488,37 @@ func (ssa *SQLStorageAuthority) NewOrderAndAuthzs(ctx context.Context, req *sapb
 			newAuthzIDs = append(newAuthzIDs, am.ID)
 		}
 
+		// Combine the already-existing and newly-created authzs.
+		allAuthzIds := append(req.NewOrder.V2Authorizations, newAuthzIDs...)
+
 		// Second, insert the new order.
 		created := ssa.clk.Now()
+		var encodedAuthzs []byte
+		var err error
+		if features.Get().StoreAuthzsInOrders {
+			encodedAuthzs, err = proto.Marshal(&sapb.Authzs{
+				AuthzIDs: allAuthzIds,
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+
 		om := orderModel{
 			RegistrationID:         req.NewOrder.RegistrationID,
 			Expires:                req.NewOrder.Expires.AsTime(),
 			Created:                created,
 			CertificateProfileName: &req.NewOrder.CertificateProfileName,
 			Replaces:               &req.NewOrder.Replaces,
+			Authzs:                 encodedAuthzs,
 		}
-		err := tx.Insert(ctx, &om)
+		err = tx.Insert(ctx, &om)
 		if err != nil {
 			return nil, err
 		}
 		orderID := om.ID
 
 		// Third, insert all of the orderToAuthz relations.
-		// Have to combine the already-associated and newly-created authzs.
-		allAuthzIds := append(req.NewOrder.V2Authorizations, newAuthzIDs...)
 		inserter, err := db.NewMultiInserter("orderToAuthz2", []string{"orderID", "authzID"})
 		if err != nil {
 			return nil, err
@@ -621,21 +636,22 @@ func (ssa *SQLStorageAuthority) SetOrderError(ctx context.Context, req *sapb.Set
 	if req.Id == 0 || req.Error == nil {
 		return nil, errIncompleteRequest
 	}
-	_, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (any, error) {
-		om, err := orderToModel(&corepb.Order{
-			Id:    req.Id,
-			Error: req.Error,
-		})
-		if err != nil {
-			return nil, err
-		}
 
+	errJSON, err := json.Marshal(req.Error)
+	if err != nil {
+		return nil, err
+	}
+	if len(errJSON) > mediumBlobSize {
+		return nil, fmt.Errorf("error object is too large to store in the database")
+	}
+
+	_, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (any, error) {
 		result, err := tx.ExecContext(ctx, `
 		UPDATE orders
 		SET error = ?
 		WHERE id = ?`,
-			om.Error,
-			om.ID)
+			errJSON,
+			req.Id)
 		if err != nil {
 			return nil, berrors.InternalServerError("error updating order error field")
 		}