diff --git a/features/features.go b/features/features.go index 0c922f3eb7b..211eb6ae58e 100644 --- a/features/features.go +++ b/features/features.go @@ -86,6 +86,11 @@ type Config struct { // during certificate issuance. This flag must be set to true in the // RA, VA, and WFE2 services for full functionality. DNSAccount01Enabled bool + + // StoreAuthzsInOrders causes the SA to write to the `authzs` + // column in NewOrder and read from it in GetOrder. It should be enabled + // after the migration to add that column has been run. + StoreAuthzsInOrders bool } var fMu = new(sync.RWMutex) diff --git a/sa/database.go b/sa/database.go index 0d06fac5f90..e8923ebdb4e 100644 --- a/sa/database.go +++ b/sa/database.go @@ -276,6 +276,9 @@ func initTables(dbMap *borp.DbMap) { if !features.Get().StoreARIReplacesInOrders { tableMap.ColMap("Replaces").SetTransient(true) } + if !features.Get().StoreAuthzsInOrders { + tableMap.ColMap("Authzs").SetTransient(true) + } dbMap.AddTableWithName(orderToAuthzModel{}, "orderToAuthz").SetKeys(false, "OrderID", "AuthzID") dbMap.AddTableWithName(orderFQDNSet{}, "orderFqdnSets").SetKeys(true, "ID") diff --git a/sa/db-next/boulder_sa/20251024000000_TheAuthzsAreInTheOrder.sql b/sa/db-next/boulder_sa/20251024000000_TheAuthzsAreInTheOrder.sql new file mode 100644 index 00000000000..bcdb9625c63 --- /dev/null +++ b/sa/db-next/boulder_sa/20251024000000_TheAuthzsAreInTheOrder.sql @@ -0,0 +1,9 @@ +-- +migrate Up +-- SQL in section 'Up' is executed when this migration is applied + +ALTER TABLE `orders` ADD COLUMN `authzs` blob DEFAULT NULL; + +-- +migrate Down +-- SQL section 'Down' is executed when this migration is rolled back + +ALTER TABLE `orders` DROP COLUMN `authzs`; diff --git a/sa/model.go b/sa/model.go index 0384d01b382..f2807c4acf9 100644 --- a/sa/model.go +++ b/sa/model.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "google.golang.org/protobuf/proto" "math" "net/netip" "net/url" @@ -346,6 +347,7 @@ type orderModel struct { BeganProcessing bool CertificateProfileName *string Replaces *string + Authzs []byte } type orderToAuthzModel struct { @@ -353,35 +355,6 @@ type orderToAuthzModel struct { AuthzID int64 } -func orderToModel(order *corepb.Order) (*orderModel, error) { - // Make a local copy so we can take a reference to it below. - profile := order.CertificateProfileName - replaces := order.Replaces - - om := &orderModel{ - ID: order.Id, - RegistrationID: order.RegistrationID, - Expires: order.Expires.AsTime(), - Created: order.Created.AsTime(), - BeganProcessing: order.BeganProcessing, - CertificateSerial: order.CertificateSerial, - CertificateProfileName: &profile, - Replaces: &replaces, - } - - if order.Error != nil { - errJSON, err := json.Marshal(order.Error) - if err != nil { - return nil, err - } - if len(errJSON) > mediumBlobSize { - return nil, fmt.Errorf("Error object is too large to store in the database") - } - om.Error = errJSON - } - return om, nil -} - func modelToOrder(om *orderModel) (*corepb.Order, error) { profile := "" if om.CertificateProfileName != nil { @@ -391,6 +364,13 @@ func modelToOrder(om *orderModel) (*corepb.Order, error) { if om.Replaces != nil { replaces = *om.Replaces } + if len(om.Authzs) > 0 { + var decodedAuthzs sapb.Authzs + err := proto.Unmarshal(om.Authzs, &decodedAuthzs) + if err != nil { + return nil, err + } + } order := &corepb.Order{ Id: om.ID, RegistrationID: om.RegistrationID, diff --git a/sa/model_test.go b/sa/model_test.go index f5a1fe49abd..c14838643ae 100644 --- a/sa/model_test.go +++ b/sa/model_test.go @@ -223,26 +223,6 @@ func TestModelToOrderBadJSON(t *testing.T) { test.AssertEquals(t, string(badJSONErr.json), string(badJSON)) } -func TestOrderModelThereAndBackAgain(t *testing.T) { - clk := clock.New() - now := clk.Now() - order := &corepb.Order{ - Id: 1, - RegistrationID: 2024, - Expires: timestamppb.New(now.Add(24 * time.Hour)), - Created: timestamppb.New(now), - Error: nil, - CertificateSerial: "2", - BeganProcessing: true, - CertificateProfileName: "phljny", - } - model, err := orderToModel(order) - test.AssertNotError(t, err, "orderToModelv2 should not have errored") - returnOrder, err := modelToOrder(model) - test.AssertNotError(t, err, "modelToOrderv2 should not have errored") - test.AssertDeepEquals(t, order, returnOrder) -} - // TestPopulateAttemptedFieldsBadJSON tests that populating a challenge from an // authz2 model with an invalid validation error or an invalid validation record // produces the expected bad JSON error. diff --git a/sa/proto/sadb.pb.go b/sa/proto/sadb.pb.go new file mode 100644 index 00000000000..184a01df3bd --- /dev/null +++ b/sa/proto/sadb.pb.go @@ -0,0 +1,127 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.5 +// protoc v3.20.1 +// source: sadb.proto + +package proto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Used internally for storage in the DB, not for RPCs. +type Authzs struct { + state protoimpl.MessageState `protogen:"open.v1"` + AuthzIDs []int64 `protobuf:"varint,1,rep,packed,name=authzIDs,proto3" json:"authzIDs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Authzs) Reset() { + *x = Authzs{} + mi := &file_sadb_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Authzs) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Authzs) ProtoMessage() {} + +func (x *Authzs) ProtoReflect() protoreflect.Message { + mi := &file_sadb_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Authzs.ProtoReflect.Descriptor instead. +func (*Authzs) Descriptor() ([]byte, []int) { + return file_sadb_proto_rawDescGZIP(), []int{0} +} + +func (x *Authzs) GetAuthzIDs() []int64 { + if x != nil { + return x.AuthzIDs + } + return nil +} + +var File_sadb_proto protoreflect.FileDescriptor + +var file_sadb_proto_rawDesc = string([]byte{ + 0x0a, 0x0a, 0x73, 0x61, 0x64, 0x62, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x73, 0x61, + 0x22, 0x24, 0x0a, 0x06, 0x41, 0x75, 0x74, 0x68, 0x7a, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x61, 0x75, + 0x74, 0x68, 0x7a, 0x49, 0x44, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x03, 0x52, 0x08, 0x61, 0x75, + 0x74, 0x68, 0x7a, 0x49, 0x44, 0x73, 0x42, 0x29, 0x5a, 0x27, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6c, 0x65, 0x74, 0x73, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x2f, 0x62, 0x6f, 0x75, 0x6c, 0x64, 0x65, 0x72, 0x2f, 0x73, 0x61, 0x2f, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +}) + +var ( + file_sadb_proto_rawDescOnce sync.Once + file_sadb_proto_rawDescData []byte +) + +func file_sadb_proto_rawDescGZIP() []byte { + file_sadb_proto_rawDescOnce.Do(func() { + file_sadb_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_sadb_proto_rawDesc), len(file_sadb_proto_rawDesc))) + }) + return file_sadb_proto_rawDescData +} + +var file_sadb_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_sadb_proto_goTypes = []any{ + (*Authzs)(nil), // 0: sa.Authzs +} +var file_sadb_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_sadb_proto_init() } +func file_sadb_proto_init() { + if File_sadb_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_sadb_proto_rawDesc), len(file_sadb_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_sadb_proto_goTypes, + DependencyIndexes: file_sadb_proto_depIdxs, + MessageInfos: file_sadb_proto_msgTypes, + }.Build() + File_sadb_proto = out.File + file_sadb_proto_goTypes = nil + file_sadb_proto_depIdxs = nil +} diff --git a/sa/proto/sadb.proto b/sa/proto/sadb.proto new file mode 100644 index 00000000000..353993b14ee --- /dev/null +++ b/sa/proto/sadb.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package sa; +option go_package = "github.com/letsencrypt/boulder/sa/proto"; + +// Used internally for storage in the DB, not for RPCs. +message Authzs { + repeated int64 authzIDs = 1; +} diff --git a/sa/sa.go b/sa/sa.go index 0db3b60adb3..0f325d73b34 100644 --- a/sa/sa.go +++ b/sa/sa.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "google.golang.org/protobuf/proto" "strings" "time" @@ -20,6 +21,7 @@ import ( corepb "github.com/letsencrypt/boulder/core/proto" "github.com/letsencrypt/boulder/db" berrors "github.com/letsencrypt/boulder/errors" + "github.com/letsencrypt/boulder/features" bgrpc "github.com/letsencrypt/boulder/grpc" "github.com/letsencrypt/boulder/identifier" blog "github.com/letsencrypt/boulder/log" @@ -477,24 +479,37 @@ func (ssa *SQLStorageAuthority) NewOrderAndAuthzs(ctx context.Context, req *sapb newAuthzIDs = append(newAuthzIDs, am.ID) } + // Combine the already-existing and newly-created authzs. + allAuthzIds := append(req.NewOrder.V2Authorizations, newAuthzIDs...) + // Second, insert the new order. created := ssa.clk.Now() + var encodedAuthzs []byte + var err error + if features.Get().StoreAuthzsInOrders { + encodedAuthzs, err = proto.Marshal(&sapb.Authzs{ + AuthzIDs: allAuthzIds, + }) + if err != nil { + return nil, err + } + } + om := orderModel{ RegistrationID: req.NewOrder.RegistrationID, Expires: req.NewOrder.Expires.AsTime(), Created: created, CertificateProfileName: &req.NewOrder.CertificateProfileName, Replaces: &req.NewOrder.Replaces, + Authzs: encodedAuthzs, } - err := tx.Insert(ctx, &om) + err = tx.Insert(ctx, &om) if err != nil { return nil, err } orderID := om.ID // Third, insert all of the orderToAuthz relations. - // Have to combine the already-associated and newly-created authzs. - allAuthzIds := append(req.NewOrder.V2Authorizations, newAuthzIDs...) inserter, err := db.NewMultiInserter("orderToAuthz2", []string{"orderID", "authzID"}) if err != nil { return nil, err @@ -612,21 +627,22 @@ func (ssa *SQLStorageAuthority) SetOrderError(ctx context.Context, req *sapb.Set if req.Id == 0 || req.Error == nil { return nil, errIncompleteRequest } - _, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (any, error) { - om, err := orderToModel(&corepb.Order{ - Id: req.Id, - Error: req.Error, - }) - if err != nil { - return nil, err - } + errJSON, err := json.Marshal(req.Error) + if err != nil { + return nil, err + } + if len(errJSON) > mediumBlobSize { + return nil, fmt.Errorf("error object is too large to store in the database") + } + + _, overallError := db.WithTransaction(ctx, ssa.dbMap, func(tx db.Executor) (any, error) { result, err := tx.ExecContext(ctx, ` UPDATE orders SET error = ? WHERE id = ?`, - om.Error, - om.ID) + errJSON, + req.Id) if err != nil { return nil, berrors.InternalServerError("error updating order error field") } diff --git a/sa/saro.go b/sa/saro.go index 8440d71f274..8d4a51fbde8 100644 --- a/sa/saro.go +++ b/sa/saro.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/letsencrypt/boulder/features" "math" "regexp" "strings" @@ -376,11 +377,15 @@ func (ssa *SQLStorageAuthorityRO) GetOrder(ctx context.Context, req *sapb.OrderR return nil, berrors.NotFoundError("no order found for ID %d", req.Id) } - v2AuthzIDs, err := authzForOrder(ctx, tx, order.Id) - if err != nil { - return nil, err + // For orders created before feature flag StoreAuthzsInOrders, fetch the list of authz IDs + // from the orderToAuthz2 table. + if len(order.V2Authorizations) == 0 { + authzIDs, err := authzForOrder(ctx, tx, order.Id) + if err != nil { + return nil, err + } + order.V2Authorizations = authzIDs } - order.V2Authorizations = v2AuthzIDs // Get the partial Authorization objects for the order authzValidityInfo, err := getAuthorizationStatuses(ctx, tx, order.V2Authorizations) @@ -497,6 +502,38 @@ func (ssa *SQLStorageAuthorityRO) GetOrderForNames(ctx context.Context, req *sap return order, nil } +func (ssa *SQLStorageAuthorityRO) getAuthorizationsByID(ctx context.Context, ids []int64) (*sapb.Authorizations, error) { + selector, err := db.NewMappedSelector[authzModel](ssa.dbReadOnlyMap) + if err != nil { + return nil, fmt.Errorf("initializing db map: %w", err) + } + + clauses := fmt.Sprintf(`WHERE id IN (%s)`, db.QuestionMarks(len(ids))) + + var sliceOfAny []any + for _, id := range ids { + sliceOfAny = append(sliceOfAny, id) + } + rows, err := selector.QueryContext(ctx, clauses, sliceOfAny...) + if err != nil { + return nil, fmt.Errorf("reading db: %w", err) + } + + var ret []*corepb.Authorization + err = rows.ForEach(func(row *authzModel) error { + authz, err := modelToAuthzPB(*row) + if err != nil { + return err + } + ret = append(ret, authz) + return nil + }) + if err != nil { + return nil, fmt.Errorf("reading db: %w", err) + } + return &sapb.Authorizations{Authzs: ret}, nil +} + // GetAuthorization2 returns the authz2 style authorization identified by the provided ID or an error. // If no authorization is found matching the ID a berrors.NotFound type error is returned. func (ssa *SQLStorageAuthorityRO) GetAuthorization2(ctx context.Context, req *sapb.AuthorizationID2) (*corepb.Authorization, error) { @@ -656,6 +693,32 @@ func (ssa *SQLStorageAuthorityRO) GetValidOrderAuthorizations2(ctx context.Conte return nil, errIncompleteRequest } + if features.Get().StoreAuthzsInOrders { + om, err := ssa.dbReadOnlyMap.Get(ctx, &orderModel{}, req.Id) + if err != nil { + if db.IsNoRows(err) { + return nil, berrors.NotFoundError("no order found for ID %d", req.Id) + } + return nil, err + } + + order, err := modelToOrder(om.(*orderModel)) + if err != nil { + return nil, err + } + + // If the order had a list of authz IDs (from the `Authzs` column in the DB), query + // them and return. Otherwise, fall through to doing a JOIN query using orderToAuthz2 + // and authz2 tables. + if len(order.V2Authorizations) > 0 { + authzs, err := ssa.getAuthorizationsByID(ctx, order.V2Authorizations) + if err != nil { + return nil, err + } + return authzs, nil + } + } + // The authz2 and orderToAuthz2 tables both have a column named "id", so we // need to be explicit about which table's "id" column we want to select. qualifiedAuthzFields := strings.Split(authzFields, " ") diff --git a/test/config-next/sa.json b/test/config-next/sa.json index 1af58f20647..da4b33422c8 100644 --- a/test/config-next/sa.json +++ b/test/config-next/sa.json @@ -47,6 +47,7 @@ }, "healthCheckInterval": "4s", "features": { + "StoreAuthzsInOrders": true, "StoreARIReplacesInOrders": true } },