diff --git a/CHANGELOG.md b/CHANGELOG.md index 4618294c59b..19e89bd78dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ - fix(eth): properly return vm error in all gas estimation methods ([filecoin-project/lotus#13389](https://github.com/filecoin-project/lotus/pull/13389)) - chore: all actor cmd support --actor ([filecoin-project/lotus#13391](https://github.com/filecoin-project/lotus/pull/13391)) - feat(spcli): add a `deposit-margin-factor` option to `lotus-miner init` so the sent deposit still covers the on-chain requirement if it rises between lookup and execution +- feat(consensus): wire tipset gas reservations and reservation-aware mempool pre-pack to activate at network version 28 (UpgradeXxHeight), keeping receipts and gas accounting identical while preventing miner penalties from underfunded intra-tipset messages # Node and Miner v1.34.1 / 2025-09-15 diff --git a/chain/consensus/compute_state.go b/chain/consensus/compute_state.go index 66257ded083..f4729e7fc65 100644 --- a/chain/consensus/compute_state.go +++ b/chain/consensus/compute_state.go @@ -196,6 +196,10 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, } } + // Network version at the execution epoch, used for reservations activation + // and gating. + nv := sm.GetNetworkVersion(ctx, epoch) + vmEarlyDuration := partDone() earlyCronGas := cronGas cronGas = 0 @@ -206,6 +210,18 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, return cid.Undef, cid.Undef, xerrors.Errorf("making vm: %w", err) } + // Start a tipset reservation session around explicit messages. A deferred + // call ensures the session is closed on all paths, while the explicit call + // before cron keeps the session scope limited to explicit messages. + if err := startReservations(ctx, vmi, bms, nv); err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("starting tipset reservations: %w", err) + } + defer func() { + if err := endReservations(ctx, vmi, nv); err != nil { + log.Warnw("ending tipset reservations failed", "error", err) + } + }() + var ( receipts []*types.MessageReceipt storingEvents = sm.ChainStore().IsStoringEvents() @@ -260,6 +276,12 @@ func (t *TipSetExecutor) ApplyBlocks(ctx context.Context, } } + // End the reservation session before running cron so that reservations + // strictly cover explicit messages only. + if err := endReservations(ctx, vmi, nv); err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("ending tipset reservations: %w", err) + } + vmMsgDuration := partDone() partDone = metrics.Timer(ctx, metrics.VMApplyCron) diff --git a/chain/consensus/features.go b/chain/consensus/features.go new file mode 100644 index 00000000000..2661ff09e6f --- /dev/null +++ b/chain/consensus/features.go @@ -0,0 +1,53 @@ +package consensus + +import "os" + +// ReservationFeatureFlags holds feature toggles for tipset gas reservations. +// +// These flags are evaluated by consensus and the message pool when deciding +// whether to attempt tipset‑scope reservations pre‑activation, and how to +// interpret Begin/End reservation errors. +type ReservationFeatureFlags struct { + // MultiStageReservations enables tipset‑scope gas reservations + // pre‑activation. When false, ReservationsEnabled returns false for + // network versions before ReservationsActivationNetworkVersion and Lotus + // operates in legacy mode (no Begin/End calls). + // + // At or after activation, reservations are always enabled regardless of + // this flag. + MultiStageReservations bool + + // MultiStageReservationsStrict controls how pre‑activation reservation + // failures are handled when MultiStageReservations is true: + // + // - When false (non‑strict), non‑NotImplemented Begin/End errors such + // as ErrReservationsInsufficientFunds and ErrReservationsPlanTooLarge + // are treated as best‑effort: Lotus logs and falls back to legacy + // mode for that tipset. + // - When true (strict), those reservation failures invalidate the + // tipset pre‑activation. Node‑error classes (e.g. overflow or + // invariant violations) always surface as errors regardless of this + // flag. + MultiStageReservationsStrict bool +} + +// Feature exposes the current reservation feature flags. +// +// Defaults: +// - MultiStageReservations: enabled when LOTUS_ENABLE_TIPSET_RESERVATIONS=1. +// - MultiStageReservationsStrict: enabled when +// LOTUS_ENABLE_TIPSET_RESERVATIONS_STRICT=1. +// +// These defaults preserve the existing environment‑based gating while making +// the flags explicit and testable. +var Feature = ReservationFeatureFlags{ + MultiStageReservations: os.Getenv("LOTUS_ENABLE_TIPSET_RESERVATIONS") == "1", + MultiStageReservationsStrict: os.Getenv("LOTUS_ENABLE_TIPSET_RESERVATIONS_STRICT") == "1", +} + +// SetFeatures overrides the global reservation feature flags. This is intended +// for wiring from higher‑level configuration and for tests; callers should +// treat it as process‑wide and set it once during initialization. +func SetFeatures(flags ReservationFeatureFlags) { + Feature = flags +} diff --git a/chain/consensus/reservations.go b/chain/consensus/reservations.go new file mode 100644 index 00000000000..efbb61077e7 --- /dev/null +++ b/chain/consensus/reservations.go @@ -0,0 +1,154 @@ +package consensus + +import ( + "context" + "errors" + + "go.opencensus.io/stats" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/network" + cid "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lotus/metrics" +) + +var rlog = logging.Logger("reservations") + +// ReservationsEnabled returns true when tipset reservations should be attempted. +// Before the reservations activation network version, this helper consults the +// MultiStageReservations feature flag. At or after the activation network +// version, reservations are always enabled and become consensus-critical. +func ReservationsEnabled(nv network.Version) bool { + // After activation, reservations are required regardless of feature flags. + if nv >= vm.ReservationsActivationNetworkVersion() { + return true + } + + // Pre-activation: best-effort mode controlled by the feature flag. + return Feature.MultiStageReservations +} + +// buildReservationPlan aggregates per-sender gas reservations across the full +// tipset. The amount reserved for each message is gas_limit * gas_fee_cap, and +// messages are deduplicated by CID across all blocks in canonical order, +// matching processedMsgs handling in ApplyBlocks. +func buildReservationPlan(bms []FilecoinBlockMessages) map[address.Address]abi.TokenAmount { + plan := make(map[address.Address]abi.TokenAmount) + seen := make(map[cid.Cid]struct{}) + + for _, b := range bms { + // canonical order is preserved in the combined slices append below + for _, cm := range append(b.BlsMessages, b.SecpkMessages...) { + m := cm.VMMessage() + mcid := m.Cid() + if _, ok := seen[mcid]; ok { + continue + } + seen[mcid] = struct{}{} + // Only explicit messages are included in blocks; implicit messages are applied separately. + cost := types.BigMul(types.NewInt(uint64(m.GasLimit)), m.GasFeeCap) + if prev, ok := plan[m.From]; ok { + plan[m.From] = types.BigAdd(prev, cost) + } else { + plan[m.From] = cost + } + } + } + return plan +} + +// startReservations is a helper that starts a reservation session on the VM if enabled. +// If the computed plan is empty (no explicit messages), Begin is skipped entirely. +func startReservations(ctx context.Context, vmi vm.Interface, bms []FilecoinBlockMessages, nv network.Version) error { + if !ReservationsEnabled(nv) { + return nil + } + + plan := buildReservationPlan(bms) + if len(plan) == 0 { + rlog.Debugw("skipping tipset reservations for empty plan") + return nil + } + + total := abi.NewTokenAmount(0) + for _, amt := range plan { + total = big.Add(total, amt) + } + + stats.Record(ctx, + metrics.ReservationPlanSenders.M(int64(len(plan))), + metrics.ReservationPlanTotal.M(total.Int64()), + ) + + rlog.Infow("starting tipset reservations", "senders", len(plan), "total", total) + if err := vmi.StartTipsetReservations(ctx, plan); err != nil { + return handleReservationError("begin", err, nv) + } + return nil +} + +// endReservations ends the active reservation session if enabled. +func endReservations(ctx context.Context, vmi vm.Interface, nv network.Version) error { + if !ReservationsEnabled(nv) { + return nil + } + if err := vmi.EndTipsetReservations(ctx); err != nil { + return handleReservationError("end", err, nv) + } + return nil +} + +// handleReservationError interprets Begin/End reservation errors according to +// network version and feature flags, deciding whether to fall back to legacy +// mode (pre-activation, non-strict) or surface the error. +func handleReservationError(stage string, err error, nv network.Version) error { + if err == nil { + return nil + } + + // Post-activation: reservations are consensus-critical; all Begin/End + // errors surface to the caller. ErrReservationsNotImplemented becomes a + // node error (engine too old) under active rules. + if nv >= vm.ReservationsActivationNetworkVersion() { + return err + } + + // Pre-activation: ErrNotImplemented is always treated as a benign signal + // that the engine does not support reservations yet; fall back to legacy + // mode regardless of strictness. + if errors.Is(err, vm.ErrReservationsNotImplemented) { + rlog.Debugw("tipset reservations not implemented; continuing in legacy mode", + "stage", stage, "error", err) + return nil + } + + // Node-error classes: always surface as errors, even pre-activation. + if errors.Is(err, vm.ErrReservationsSessionOpen) || + errors.Is(err, vm.ErrReservationsSessionClosed) || + errors.Is(err, vm.ErrReservationsNonZeroRemainder) || + errors.Is(err, vm.ErrReservationsOverflow) || + errors.Is(err, vm.ErrReservationsInvariantViolation) { + return err + } + + // Reservation failures toggled by strict mode. When strict is disabled, + // treat these as best-effort pre-activation and fall back to legacy mode. + switch { + case errors.Is(err, vm.ErrReservationsInsufficientFunds), errors.Is(err, vm.ErrReservationsPlanTooLarge): + if Feature.MultiStageReservationsStrict { + return err + } + rlog.Debugw("tipset reservations failed pre-activation; continuing in legacy mode (non-strict)", + "stage", stage, "error", err) + return nil + default: + // Unknown errors pre-activation are treated as node errors. + return err + } +} diff --git a/chain/consensus/reservations_bench_test.go b/chain/consensus/reservations_bench_test.go new file mode 100644 index 00000000000..43e40055ab1 --- /dev/null +++ b/chain/consensus/reservations_bench_test.go @@ -0,0 +1,62 @@ +package consensus + +import ( + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" +) + +// BenchmarkBuildReservationPlan measures the cost of aggregating per-sender +// reservations across a large synthetic tipset. This provides an upper bound +// on the Stage-1 host-side overhead for tipset reservations. +func BenchmarkBuildReservationPlan(b *testing.B) { + addr1, err := address.NewIDAddress(100) + if err != nil { + b.Fatalf("creating addr1: %v", err) + } + addr2, err := address.NewIDAddress(200) + if err != nil { + b.Fatalf("creating addr2: %v", err) + } + + const numBlocks = 5 + const msgsPerBlock = 2000 // 10k messages total. + + bms := make([]FilecoinBlockMessages, numBlocks) + for i := range bms { + bls := make([]types.ChainMsg, 0, msgsPerBlock) + for j := 0; j < msgsPerBlock; j++ { + from := addr1 + if j%2 == 1 { + from = addr2 + } + msg := &types.Message{ + From: from, + To: addr2, + Nonce: uint64(j), + Value: abi.NewTokenAmount(0), + GasFeeCap: abi.NewTokenAmount(1), + GasLimit: 1_000_000, + } + bls = append(bls, msg) + } + bms[i] = FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: bls, + }, + WinCount: 1, + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + plan := buildReservationPlan(bms) + if len(plan) == 0 { + b.Fatalf("unexpected empty reservation plan") + } + } +} diff --git a/chain/consensus/reservations_test.go b/chain/consensus/reservations_test.go new file mode 100644 index 00000000000..e73f6fa5e32 --- /dev/null +++ b/chain/consensus/reservations_test.go @@ -0,0 +1,530 @@ +package consensus + +import ( + "context" + "errors" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/go-state-types/crypto" + "github.com/filecoin-project/go-state-types/network" + cid "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" +) + +type fakeReservationsVM struct { + startCalled bool + endCalled bool + + startPlan map[address.Address]abi.TokenAmount + startErr error + endErr error +} + +func (f *fakeReservationsVM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*vm.ApplyRet, error) { + return nil, nil +} + +func (f *fakeReservationsVM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*vm.ApplyRet, error) { + return nil, nil +} + +func (f *fakeReservationsVM) Flush(ctx context.Context) (cid.Cid, error) { + return cid.Undef, nil +} + +func (f *fakeReservationsVM) StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error { + f.startCalled = true + f.startPlan = plan + return f.startErr +} + +func (f *fakeReservationsVM) EndTipsetReservations(ctx context.Context) error { + f.endCalled = true + return f.endErr +} + +func withFeatures(t *testing.T, flags ReservationFeatureFlags) func() { + t.Helper() + orig := Feature + SetFeatures(flags) + return func() { + SetFeatures(orig) + } +} + +func TestReservationsEnabledFeatureFlags(t *testing.T) { + nvPre := network.Version(0) + nvPost := vm.ReservationsActivationNetworkVersion() + + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: false, + MultiStageReservationsStrict: false, + }) + defer restore() + + if ReservationsEnabled(nvPre) { + t.Fatalf("expected reservations to be disabled when feature flag is false pre-activation") + } + + // At or after activation, reservations are always enabled regardless of + // the feature flags. + if !ReservationsEnabled(nvPost) { + t.Fatalf("expected reservations to be enabled at or after activation network version") + } + + SetFeatures(ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + if !ReservationsEnabled(nvPre) { + t.Fatalf("expected reservations to be enabled when MultiStageReservations is true pre-activation") + } + + SetFeatures(ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + if !ReservationsEnabled(nvPre) { + t.Fatalf("expected reservations to remain enabled when strict mode is true pre-activation") + } +} + +func TestBuildReservationPlanDedupAcrossBlocks(t *testing.T) { + addr1, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr1: %v", err) + } + addr2, err := address.NewIDAddress(200) + if err != nil { + t.Fatalf("creating addr2: %v", err) + } + + feeCapA := abi.NewTokenAmount(2) + feeCapB := abi.NewTokenAmount(3) + feeCapC := abi.NewTokenAmount(4) + + const gasLimitA = int64(10) + const gasLimitB = int64(5) + const gasLimitC = int64(7) + + msgA := &types.Message{ + From: addr1, + To: addr2, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCapA, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimitA, + } + msgB := &types.Message{ + From: addr1, + To: addr2, + Nonce: 1, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCapB, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimitB, + } + msgC := &types.Message{ + From: addr2, + To: addr1, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCapC, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimitC, + } + + signedC := &types.SignedMessage{ + Message: *msgC, + Signature: crypto.Signature{Type: crypto.SigTypeSecp256k1, Data: []byte{0x01}}, + } + + b1 := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msgA}, + SecpkMessages: []types.ChainMsg{signedC}, + }, + } + b2 := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msgA, msgB}, + SecpkMessages: []types.ChainMsg{signedC}, + }, + } + + plan := buildReservationPlan([]FilecoinBlockMessages{b1, b2}) + + if len(plan) != 2 { + t.Fatalf("expected 2 senders in plan, got %d", len(plan)) + } + + costA := big.Mul(big.NewInt(gasLimitA), feeCapA) + costB := big.Mul(big.NewInt(gasLimitB), feeCapB) + costC := big.Mul(big.NewInt(gasLimitC), feeCapC) + + expected1 := big.Add(costA, costB) + expected2 := costC + + got1, ok := plan[addr1] + if !ok { + t.Fatalf("missing sender addr1 in plan") + } + if !got1.Equals(expected1) { + t.Fatalf("addr1 total mismatch: expected %s, got %s", expected1, got1) + } + + got2, ok := plan[addr2] + if !ok { + t.Fatalf("missing sender addr2 in plan") + } + if !got2.Equals(expected2) { + t.Fatalf("addr2 total mismatch: expected %s, got %s", expected2, got2) + } +} + +func TestStartReservationsEmptyPlanSkipsBegin(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + defer restore() + + vmStub := &fakeReservationsVM{} + if err := startReservations(context.Background(), vmStub, nil, network.Version(0)); err != nil { + t.Fatalf("startReservations returned error for empty plan: %v", err) + } + if vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations not to be called for empty plan") + } +} + +func TestStartReservationsErrorPropagation(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsInsufficientFunds, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, network.Version(0)) + if !errors.Is(err, vm.ErrReservationsInsufficientFunds) { + t.Fatalf("expected ErrReservationsInsufficientFunds from startReservations in strict mode, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestStartReservationsStrictPreActivationPlanTooLargeIsError(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsPlanTooLarge, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, network.Version(0)) + if !errors.Is(err, vm.ErrReservationsPlanTooLarge) { + t.Fatalf("expected ErrReservationsPlanTooLarge from startReservations in strict mode, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestStartReservationsNonStrictPreActivationFallsBackOnInsufficientFunds(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsInsufficientFunds, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, network.Version(0)) + if err != nil { + t.Fatalf("expected non-strict pre-activation startReservations to fall back on insufficient funds, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestStartReservationsNonStrictPreActivationFallsBackOnPlanTooLarge(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsPlanTooLarge, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, network.Version(0)) + if err != nil { + t.Fatalf("expected non-strict pre-activation startReservations to fall back on plan too large, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestStartReservationsPreActivationNotImplementedFallsBack(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsNotImplemented, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, network.Version(0)) + if err != nil { + t.Fatalf("expected pre-activation NotImplemented to fall back to legacy mode, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestStartReservationsPostActivationNotImplementedIsError(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + defer restore() + + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating addr: %v", err) + } + + feeCap := abi.NewTokenAmount(1) + const gasLimit = int64(10) + + msg := &types.Message{ + From: addr, + To: addr, + Nonce: 0, + Value: abi.NewTokenAmount(0), + GasFeeCap: feeCap, + GasPremium: abi.NewTokenAmount(0), + GasLimit: gasLimit, + } + + b := FilecoinBlockMessages{ + BlockMessages: store.BlockMessages{ + BlsMessages: []types.ChainMsg{msg}, + }, + } + + vmStub := &fakeReservationsVM{ + startErr: vm.ErrReservationsNotImplemented, + } + + err = startReservations(context.Background(), vmStub, []FilecoinBlockMessages{b}, vm.ReservationsActivationNetworkVersion()) + if !errors.Is(err, vm.ErrReservationsNotImplemented) { + t.Fatalf("expected ErrReservationsNotImplemented post-activation, got %v", err) + } + if !vmStub.startCalled { + t.Fatalf("expected StartTipsetReservations to be called") + } +} + +func TestEndReservationsErrorPropagation(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + defer restore() + + vmStub := &fakeReservationsVM{ + endErr: vm.ErrReservationsNonZeroRemainder, + } + + err := endReservations(context.Background(), vmStub, network.Version(0)) + if !errors.Is(err, vm.ErrReservationsNonZeroRemainder) { + t.Fatalf("expected ErrReservationsNonZeroRemainder from endReservations in strict mode, got %v", err) + } + if !vmStub.endCalled { + t.Fatalf("expected EndTipsetReservations to be called") + } +} + +func TestEndReservationsNonStrictPreActivationFallsBackOnPlanTooLarge(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: false, + }) + defer restore() + + vmStub := &fakeReservationsVM{ + endErr: vm.ErrReservationsPlanTooLarge, + } + + err := endReservations(context.Background(), vmStub, network.Version(0)) + if err != nil { + t.Fatalf("expected non-strict pre-activation endReservations to fall back on plan too large, got %v", err) + } + if !vmStub.endCalled { + t.Fatalf("expected EndTipsetReservations to be called") + } +} + +func TestEndReservationsStrictPreActivationPlanTooLargeIsError(t *testing.T) { + restore := withFeatures(t, ReservationFeatureFlags{ + MultiStageReservations: true, + MultiStageReservationsStrict: true, + }) + defer restore() + + vmStub := &fakeReservationsVM{ + endErr: vm.ErrReservationsPlanTooLarge, + } + + err := endReservations(context.Background(), vmStub, network.Version(0)) + if !errors.Is(err, vm.ErrReservationsPlanTooLarge) { + t.Fatalf("expected ErrReservationsPlanTooLarge from endReservations in strict mode, got %v", err) + } + if !vmStub.endCalled { + t.Fatalf("expected EndTipsetReservations to be called") + } +} diff --git a/chain/messagepool/config.go b/chain/messagepool/config.go index 72b7d9567b3..07d23e688d0 100644 --- a/chain/messagepool/config.go +++ b/chain/messagepool/config.go @@ -98,5 +98,8 @@ func DefaultConfig() *types.MpoolConfig { ReplaceByFeeRatio: ReplaceByFeePercentageDefault, PruneCooldown: PruneCooldownDefault, GasLimitOverestimation: GasLimitOverestimation, + // Reservation-aware pre-pack simulation is opt-in and advisory. It is + // additionally gated by the global reservations feature flag. + EnableReservationPrePack: false, } } diff --git a/chain/messagepool/selection.go b/chain/messagepool/selection.go index f3af9019d81..fd823b62763 100644 --- a/chain/messagepool/selection.go +++ b/chain/messagepool/selection.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/go-state-types/crypto" "github.com/filecoin-project/lotus/build/buildconstants" + "github.com/filecoin-project/lotus/chain/consensus" "github.com/filecoin-project/lotus/chain/messagepool/gasguess" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" @@ -89,6 +90,124 @@ type selectedMessages struct { gasLimit int64 secpLimit int blsLimit int + + // mp references the parent message pool; it is used for reservation-aware + // pre-pack simulation when enabled. + mp *MessagePool + + // reservationEnabled toggles reservation-aware pre-pack simulation for this + // selection pass. When true, reservedBySender and balanceBySender track + // per-sender Σ(gas_limit * gas_fee_cap) and the sender's on-chain balance + // at the selection base state. + reservationEnabled bool + reservationCtx context.Context + reservationTipset *types.TipSet + reservedBySender map[address.Address]types.BigInt + balanceBySender map[address.Address]types.BigInt +} + +func (mp *MessagePool) reservationsPrePackEnabled() bool { + cfg := mp.getConfig() + if !cfg.EnableReservationPrePack { + return false + } + // Pre-pack simulation shares the same activation/gating switch as tipset + // reservations. Before activation, it is additionally gated by the mempool + // config flag. After activation, it follows consensus reservations + // activation and is always enabled when building blocks. + // + // We use the network version at the next epoch (the height at which the + // selected messages will be applied). + mp.curTsLk.RLock() + defer mp.curTsLk.RUnlock() + + if mp.curTs == nil { + return false + } + + nextEpoch := mp.curTs.Height() + 1 + nv, err := mp.getNtwkVersion(nextEpoch) + if err != nil { + log.Warnw("mpool reservation pre-pack: failed to get network version; disabling reservation heuristics", + "height", nextEpoch, "error", err) + return false + } + + return consensus.ReservationsEnabled(nv) +} + +func (sm *selectedMessages) initReservations(ctx context.Context, ts *types.TipSet, mp *MessagePool) { + sm.mp = mp + if !mp.reservationsPrePackEnabled() { + return + } + + sm.reservationEnabled = true + sm.reservationCtx = ctx + sm.reservationTipset = ts + sm.reservedBySender = make(map[address.Address]types.BigInt) + sm.balanceBySender = make(map[address.Address]types.BigInt) +} + +func (sm *selectedMessages) reserveForMessages(sender address.Address, msgs []*types.SignedMessage) bool { + if !sm.reservationEnabled || len(msgs) == 0 { + return true + } + + balance, ok := sm.balanceBySender[sender] + if !ok { + // If we don't already have a cached balance, attempt to load it from + // chain state. If we cannot, disable reservation heuristics to avoid + // partial behaviour. + if sm.mp == nil || sm.reservationCtx == nil || sm.reservationTipset == nil { + sm.reservationEnabled = false + sm.reservedBySender = nil + sm.balanceBySender = nil + return true + } + + var err error + balance, err = sm.mp.getStateBalance(sm.reservationCtx, sender, sm.reservationTipset) + if err != nil { + log.Warnw("mpool reservation pre-pack: failed to load sender balance; disabling reservation heuristics", + "from", sender, "error", err) + sm.reservationEnabled = false + sm.reservedBySender = nil + sm.balanceBySender = nil + return true + } + sm.balanceBySender[sender] = balance + } + + existing := types.NewInt(0) + if prev, ok := sm.reservedBySender[sender]; ok { + existing = prev + } + + // Compute additional Σ(cap * limit) for this sender. + additional := types.NewInt(0) + for _, m := range msgs { + // All messages in a chain share the same sender, but we are defensive. + if m.Message.From != sender { + continue + } + gasLimit := types.NewInt(uint64(m.Message.GasLimit)) + cost := types.BigMul(m.Message.GasFeeCap, gasLimit) + additional = types.BigAdd(additional, cost) + } + + if types.BigCmp(additional, types.NewInt(0)) == 0 { + return true + } + + nextTotal := types.BigAdd(existing, additional) + if types.BigCmp(nextTotal, balance) > 0 { + // Over-committed for this sender at the selection base state. + return false + } + + sm.reservedBySender[sender] = nextTotal + return true } // returns false if chain can't be added due to block constraints @@ -99,6 +218,16 @@ func (sm *selectedMessages) tryToAdd(mc *msgChain) bool { return false } + if sm.reservationEnabled && len(mc.msgs) > 0 { + sender := mc.msgs[0].Message.From + if !sm.reserveForMessages(sender, mc.msgs) { + // Over-committed for this sender; invalidate this chain (and any + // dependents) and treat it as not addable in this pass. + mc.Invalidate() + return false + } + } + if mc.sigType == crypto.SigTypeBLS { if sm.blsLimit < l { return false @@ -170,6 +299,22 @@ func (sm *selectedMessages) tryToAddWithDeps(mc *msgChain, mp *MessagePool, base return false } + if sm.reservationEnabled && len(mc.msgs) > 0 { + sender := mc.msgs[0].Message.From + var toReserve []*types.SignedMessage + for i := len(chainDeps) - 1; i >= 0; i-- { + toReserve = append(toReserve, chainDeps[i].msgs...) + } + toReserve = append(toReserve, mc.msgs...) + + if !sm.reserveForMessages(sender, toReserve) { + // Over-committed for this sender; invalidate the chain (and its + // dependents) so subsequent passes skip them. + mc.Invalidate() + return false + } + } + // the chain fits! include it together with all dependencies for i := len(chainDeps) - 1; i >= 0; i-- { curChain := chainDeps[i] @@ -603,7 +748,9 @@ func (mp *MessagePool) selectPriorityMessages(ctx context.Context, pending map[a gasLimit: buildconstants.BlockGasLimit, blsLimit: cbg.MaxLength, secpLimit: cbg.MaxLength, + mp: mp, } + result.initReservations(ctx, ts, mp) minGas := int64(gasguess.MinGas) // 1. Get priority actor chains diff --git a/chain/messagepool/selection_test.go b/chain/messagepool/selection_test.go index fa13a0f8f54..f3ec06f6f41 100644 --- a/chain/messagepool/selection_test.go +++ b/chain/messagepool/selection_test.go @@ -1498,6 +1498,130 @@ func TestGasReward(t *testing.T) { } } +func TestReservationPrePackPrunesOverCommittedChain(t *testing.T) { + // Construct a selectedMessages instance with reservation heuristics enabled + // and a single chain whose Σ(cap×limit) exceeds the configured sender + // balance. The chain should be invalidated and not added. + + var ( + addr, _ = address.NewIDAddress(100) + ) + + sm := &selectedMessages{ + msgs: nil, + gasLimit: buildconstants.BlockGasLimit, + blsLimit: cbg.MaxLength, + secpLimit: cbg.MaxLength, + reservationEnabled: true, + reservedBySender: make(map[address.Address]types.BigInt), + balanceBySender: make(map[address.Address]types.BigInt), + } + + // Sender balance is 50 atto. + sm.balanceBySender[addr] = types.NewInt(50) + + m1 := &types.SignedMessage{ + Message: types.Message{ + From: addr, + GasLimit: 1, + GasFeeCap: types.NewInt(10), + }, + } + m2 := &types.SignedMessage{ + Message: types.Message{ + From: addr, + GasLimit: 1, + GasFeeCap: types.NewInt(100), + }, + } + + mc := &msgChain{ + msgs: []*types.SignedMessage{m1, m2}, + gasLimit: m1.Message.GasLimit + m2.Message.GasLimit, + valid: true, + sigType: crypto.SigTypeSecp256k1, + } + + ok := sm.tryToAdd(mc) + if ok { + t.Fatalf("expected tryToAdd to fail due to reservation pre-pack, but it succeeded") + } + if mc.valid { + t.Fatalf("expected chain to be invalidated when over-committed") + } + if len(sm.msgs) != 0 { + t.Fatalf("expected no messages to be added, got %d", len(sm.msgs)) + } +} + +func TestReservationPrePackAccumulatesCapTimesLimit(t *testing.T) { + // Verify that reservation-aware pre-pack simulation uses Σ(cap×limit) per + // sender when accumulating reserved totals. + + addr, _ := address.NewIDAddress(200) + + sm := &selectedMessages{ + msgs: nil, + gasLimit: buildconstants.BlockGasLimit, + blsLimit: cbg.MaxLength, + secpLimit: cbg.MaxLength, + reservationEnabled: true, + reservedBySender: make(map[address.Address]types.BigInt), + balanceBySender: make(map[address.Address]types.BigInt), + } + + // Large balance so neither chain is pruned. + sm.balanceBySender[addr] = types.NewInt(1_000_000) + + m1 := &types.SignedMessage{ + Message: types.Message{ + From: addr, + GasLimit: 5, + GasFeeCap: types.NewInt(10), + }, + } + // cost1 = 5 * 10 = 50 + mc1 := &msgChain{ + msgs: []*types.SignedMessage{m1}, + gasLimit: m1.Message.GasLimit, + valid: true, + sigType: crypto.SigTypeSecp256k1, + } + + if !sm.tryToAdd(mc1) { + t.Fatalf("expected first chain to be accepted") + } + + expected := types.NewInt(50) + if got := sm.reservedBySender[addr]; !got.Equals(expected) { + t.Fatalf("expected reserved total %s, got %s", expected, got) + } + + m2 := &types.SignedMessage{ + Message: types.Message{ + From: addr, + GasLimit: 3, + GasFeeCap: types.NewInt(20), + }, + } + // cost2 = 3 * 20 = 60; cumulative = 110 + mc2 := &msgChain{ + msgs: []*types.SignedMessage{m2}, + gasLimit: m2.Message.GasLimit, + valid: true, + sigType: crypto.SigTypeSecp256k1, + } + + if !sm.tryToAdd(mc2) { + t.Fatalf("expected second chain to be accepted") + } + + expected = types.NewInt(110) + if got := sm.reservedBySender[addr]; !got.Equals(expected) { + t.Fatalf("expected reserved total %s after second chain, got %s", expected, got) + } +} + func TestRealWorldSelection(t *testing.T) { // load test-messages.json.gz and rewrite the messages so that diff --git a/chain/types/mpool.go b/chain/types/mpool.go index 497d5c590e1..5d5ca2e0258 100644 --- a/chain/types/mpool.go +++ b/chain/types/mpool.go @@ -13,6 +13,13 @@ type MpoolConfig struct { ReplaceByFeeRatio Percent PruneCooldown time.Duration GasLimitOverestimation float64 + + // EnableReservationPrePack enables reservation-aware pre-pack simulation in + // the mpool selector. When combined with the tipset reservation feature + // gating, this provides an advisory check that per-sender Σ(gas_limit * + // gas_fee_cap) across the selected block does not exceed the sender's + // on-chain balance at the selection base state. + EnableReservationPrePack bool } func (mc *MpoolConfig) Clone() *MpoolConfig { diff --git a/chain/vm/execution.go b/chain/vm/execution.go index 4fb626f4390..a6875aa4498 100644 --- a/chain/vm/execution.go +++ b/chain/vm/execution.go @@ -10,6 +10,8 @@ import ( "go.opencensus.io/stats" "go.opencensus.io/tag" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/metrics" ) @@ -58,6 +60,21 @@ func (e *vmExecutor) Flush(ctx context.Context) (cid.Cid, error) { return e.vmi.Flush(ctx) } +// StartTipsetReservations forwards the call to the underlying VM implementation, +// guarding execution lanes similarly to message application. +func (e *vmExecutor) StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error { + token := execution.getToken(ctx, e.lane) + defer token.Done() + return e.vmi.StartTipsetReservations(ctx, plan) +} + +// EndTipsetReservations forwards the call to the underlying VM implementation. +func (e *vmExecutor) EndTipsetReservations(ctx context.Context) error { + token := execution.getToken(ctx, e.lane) + defer token.Done() + return e.vmi.EndTipsetReservations(ctx) +} + type executionToken struct { lane ExecutionLane reserved int diff --git a/chain/vm/fvm.go b/chain/vm/fvm.go index 77103d31d8d..3691030ac06 100644 --- a/chain/vm/fvm.go +++ b/chain/vm/fvm.go @@ -3,6 +3,7 @@ package vm import ( "bytes" "context" + "errors" "fmt" "io" "math" @@ -22,6 +23,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/big" "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/go-state-types/manifest" "github.com/filecoin-project/go-state-types/network" @@ -44,6 +46,35 @@ import ( var _ Interface = (*FVM)(nil) var _ ffi_cgo.Externs = (*FvmExtern)(nil) +var ( + ErrReservationsNotImplemented = ffi.ErrReservationsNotImplemented + ErrReservationsInsufficientFunds = ffi.ErrReservationsInsufficientFunds + ErrReservationsSessionOpen = ffi.ErrReservationsSessionOpen + ErrReservationsSessionClosed = ffi.ErrReservationsSessionClosed + ErrReservationsNonZeroRemainder = ffi.ErrReservationsNonZeroRemainder + ErrReservationsPlanTooLarge = ffi.ErrReservationsPlanTooLarge + ErrReservationsOverflow = ffi.ErrReservationsOverflow + ErrReservationsInvariantViolation = ffi.ErrReservationsInvariantViolation +) + +func reservationStatusToError(code int32) error { + return ffi.ReservationStatusToError(code) +} + +// reservationsActivationNetworkVersion is the network version at which tipset +// reservations become consensus-critical. Before this version, reservations +// are best-effort and Begin/End calls may fall back to legacy mode when the +// engine does not implement reservations. At or after this version, a node +// must run an engine that implements reservations; ErrReservationsNotImplemented +// is treated as a node error. +var reservationsActivationNetworkVersion = network.Version28 + +// ReservationsActivationNetworkVersion returns the network version at which +// tipset reservations become consensus-critical. +func ReservationsActivationNetworkVersion() network.Version { + return reservationsActivationNetworkVersion +} + type FvmExtern struct { rand.Rand blockstore.Blockstore @@ -250,6 +281,11 @@ type FVM struct { // returnEvents specifies whether to parse and return events when applying messages. returnEvents bool + + // reservationsActive tracks whether a reservation session is currently open + // on the underlying FVM instance. This ensures we only call EndReservations + // when a session was successfully started. + reservationsActive bool } func defaultFVMOpts(ctx context.Context, opts *VMOpts) (*ffi.FVMOpts, error) { @@ -413,6 +449,14 @@ func NewDebugFVM(ctx context.Context, opts *VMOpts) (*FVM, error) { return ret, nil } +func (vm *FVM) BeginReservations(plan []byte) error { + return vm.fvm.BeginReservations(plan) +} + +func (vm *FVM) EndReservations() error { + return vm.fvm.EndReservations() +} + func (vm *FVM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, error) { start := build.Clock.Now() defer atomic.AddUint64(&StatApplied, 1) @@ -478,6 +522,104 @@ func (vm *FVM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet return applyRet, nil } +func (vm *FVM) StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error { + // Empty plans are a no-op and must not enter reservation mode. + if len(plan) == 0 { + return nil + } + + planCBOR, err := encodeReservationPlanCBOR(plan) + if err != nil { + return xerrors.Errorf("encoding reservation plan: %w", err) + } + + if err := vm.BeginReservations(planCBOR); err != nil { + // Pre-activation behavior: if the engine does not implement + // reservations, fall back to legacy mode while leaving + // reservationsActive=false so EndTipsetReservations is a no-op. + if errors.Is(err, ErrReservationsNotImplemented) && vm.nv < reservationsActivationNetworkVersion { + log.Debugw("FVM reservations not implemented; continuing in legacy mode") + vm.reservationsActive = false + return nil + } + return err + } + + vm.reservationsActive = true + return nil +} + +func (vm *FVM) EndTipsetReservations(ctx context.Context) error { + // If no reservation session was started (empty plan or engine did not + // implement reservations), this is a no-op. + if !vm.reservationsActive { + return nil + } + + if err := vm.EndReservations(); err != nil { + // If Begin succeeded, ErrReservationsNotImplemented is not expected. + // Before activation, treat it as a legacy-mode fallback; at or after + // activation, surface it as a node error. + if errors.Is(err, ErrReservationsNotImplemented) && vm.nv < reservationsActivationNetworkVersion { + log.Debugw("FVM reservations end not implemented; continuing in legacy mode") + vm.reservationsActive = false + return nil + } + return err + } + + vm.reservationsActive = false + return nil +} + +func encodeReservationPlanCBOR(plan map[address.Address]abi.TokenAmount) ([]byte, error) { + entries := make([][][]byte, 0, len(plan)) + + for addr, amount := range plan { + amountBytes, err := amount.Bytes() + if err != nil { + return nil, xerrors.Errorf("serializing reservation amount: %w", err) + } + + entries = append(entries, [][]byte{addr.Bytes(), amountBytes}) + } + + return cbor.DumpObject(entries) +} + +func decodeReservationPlanCBOR(data []byte) (map[address.Address]abi.TokenAmount, error) { + var entries [][][]byte + if err := cbor.DecodeInto(data, &entries); err != nil { + return nil, xerrors.Errorf("decoding reservation plan: %w", err) + } + + result := make(map[address.Address]abi.TokenAmount, len(entries)) + + for _, entry := range entries { + if len(entry) != 2 { + return nil, xerrors.Errorf("invalid reservation entry length %d", len(entry)) + } + + addr, err := address.NewFromBytes(entry[0]) + if err != nil { + return nil, xerrors.Errorf("invalid reservation address: %w", err) + } + + if _, exists := result[addr]; exists { + return nil, xerrors.Errorf("duplicate sender %s in reservation plan", addr) + } + + amt, err := big.FromBytes(entry[1]) + if err != nil { + return nil, xerrors.Errorf("invalid reservation amount encoding: %w", err) + } + + result[addr] = abi.TokenAmount(amt) + } + + return result, nil +} + func (vm *FVM) ApplyImplicitMessage(ctx context.Context, cmsg *types.Message) (*ApplyRet, error) { start := build.Clock.Now() defer atomic.AddUint64(&StatApplied, 1) @@ -608,6 +750,14 @@ func (vm *dualExecutionFVM) Flush(ctx context.Context) (cid.Cid, error) { return vm.main.Flush(ctx) } +func (vm *dualExecutionFVM) StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error { + return vm.main.StartTipsetReservations(ctx, plan) +} + +func (vm *dualExecutionFVM) EndTipsetReservations(ctx context.Context) error { + return vm.main.EndTipsetReservations(ctx) +} + // Passing this as a pointer of structs has proven to be an enormous PiTA; hence this code. type xRedirect struct{ from, to cid.Cid } type xMapping struct{ redirects []xRedirect } diff --git a/chain/vm/fvm_reservations_test.go b/chain/vm/fvm_reservations_test.go new file mode 100644 index 00000000000..56d06eeae1a --- /dev/null +++ b/chain/vm/fvm_reservations_test.go @@ -0,0 +1,140 @@ +package vm + +import ( + "strings" + "testing" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + cbor "github.com/ipfs/go-ipld-cbor" +) + +func TestReservationStatusToErrorMapping(t *testing.T) { + testCases := []struct { + code int32 + err error + }{ + {code: 0, err: nil}, + {code: 1, err: ErrReservationsNotImplemented}, + {code: 2, err: ErrReservationsInsufficientFunds}, + {code: 3, err: ErrReservationsSessionOpen}, + {code: 4, err: ErrReservationsSessionClosed}, + {code: 5, err: ErrReservationsNonZeroRemainder}, + {code: 6, err: ErrReservationsPlanTooLarge}, + {code: 7, err: ErrReservationsOverflow}, + {code: 8, err: ErrReservationsInvariantViolation}, + } + + for _, tc := range testCases { + got := reservationStatusToError(tc.code) + if tc.err == nil { + if got != nil { + t.Fatalf("code %d: expected nil error, got %v", tc.code, got) + } + continue + } + + if got != tc.err { + t.Fatalf("code %d: expected error %v, got %v", tc.code, tc.err, got) + } + } + + unknown := reservationStatusToError(99) + if unknown == nil { + t.Fatalf("expected non-nil error for unknown status code") + } +} + +func TestReservationPlanCBORRoundTrip(t *testing.T) { + addr1, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating address 1: %v", err) + } + addr2, err := address.NewIDAddress(200) + if err != nil { + t.Fatalf("creating address 2: %v", err) + } + + plan := map[address.Address]abi.TokenAmount{ + addr1: abi.NewTokenAmount(123), + addr2: abi.NewTokenAmount(456), + } + + encoded, err := encodeReservationPlanCBOR(plan) + if err != nil { + t.Fatalf("encodeReservationPlanCBOR failed: %v", err) + } + + decoded, err := decodeReservationPlanCBOR(encoded) + if err != nil { + t.Fatalf("decodeReservationPlanCBOR failed: %v", err) + } + + if len(decoded) != len(plan) { + t.Fatalf("decoded plan length %d != original %d", len(decoded), len(plan)) + } + + for addr, amt := range plan { + got, ok := decoded[addr] + if !ok { + t.Fatalf("missing entry for address %s", addr) + } + + if amt.String() != got.String() { + t.Fatalf("amount mismatch for %s: expected %s, got %s", addr, amt.String(), got.String()) + } + } +} + +func TestReservationPlanCBORDecodeDuplicateSender(t *testing.T) { + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating address: %v", err) + } + + amount := abi.NewTokenAmount(123) + amountBytes, err := amount.Bytes() + if err != nil { + t.Fatalf("serializing amount: %v", err) + } + + entry := [][]byte{addr.Bytes(), amountBytes} + entries := [][][]byte{entry, entry} + + encoded, err := cbor.DumpObject(entries) + if err != nil { + t.Fatalf("DumpObject failed: %v", err) + } + + _, err = decodeReservationPlanCBOR(encoded) + if err == nil { + t.Fatalf("expected error when decoding duplicate sender in reservation plan") + } + if got, want := err.Error(), "duplicate sender"; !strings.Contains(got, want) { + t.Fatalf("expected error to contain %q, got %q", want, got) + } +} + +func TestReservationPlanCBORDecodeInvalidEntryLength(t *testing.T) { + addr, err := address.NewIDAddress(100) + if err != nil { + t.Fatalf("creating address: %v", err) + } + + // Entry with only the address bytes should be rejected. + entry := [][]byte{addr.Bytes()} + entries := [][][]byte{entry} + + encoded, err := cbor.DumpObject(entries) + if err != nil { + t.Fatalf("DumpObject failed: %v", err) + } + + _, err = decodeReservationPlanCBOR(encoded) + if err == nil { + t.Fatalf("expected error when decoding entry with invalid length") + } + if got, want := err.Error(), "invalid reservation entry length"; !strings.Contains(got, want) { + t.Fatalf("expected error to contain %q, got %q", want, got) + } +} diff --git a/chain/vm/vm.go b/chain/vm/vm.go index c108f8d1a4e..0899dbcde01 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -389,6 +389,18 @@ func (vm *LegacyVM) send(ctx context.Context, msg *types.Message, parent *Runtim return ret, err, rt } +// StartTipsetReservations is a no-op for the legacy VM; reservations are only +// supported in FVM mode. This method is provided to satisfy the Interface and +// enable scaffolding of the reservation flow in consensus. +func (vm *LegacyVM) StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error { + return nil +} + +// EndTipsetReservations is a no-op for the legacy VM. +func (vm *LegacyVM) EndTipsetReservations(ctx context.Context) error { + return nil +} + func checkMessage(msg *types.Message) error { if msg.GasLimit == 0 { return xerrors.Errorf("message has no gas limit set") diff --git a/chain/vm/vmi.go b/chain/vm/vmi.go index 042621ca2d4..d28657152cd 100644 --- a/chain/vm/vmi.go +++ b/chain/vm/vmi.go @@ -7,6 +7,8 @@ import ( cid "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" @@ -35,6 +37,13 @@ type Interface interface { ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) // Flush all buffered objects into the state store provided to the VM at construction. Flush(ctx context.Context) (cid.Cid, error) + // StartTipsetReservations begins a tipset-scoped reservation session. + // The plan maps sender addresses to the total amount to reserve for gas + // across the full tipset. Implementations may ignore this call if the + // feature is disabled or unsupported. + StartTipsetReservations(ctx context.Context, plan map[address.Address]abi.TokenAmount) error + // EndTipsetReservations ends the active reservation session, if any. + EndTipsetReservations(ctx context.Context) error } // WARNING: You will not affect your node's execution by misusing this feature, but you will confuse yourself thoroughly! diff --git a/documentation/en/tipset-reservations.md b/documentation/en/tipset-reservations.md new file mode 100644 index 00000000000..63a7ac8178a --- /dev/null +++ b/documentation/en/tipset-reservations.md @@ -0,0 +1,90 @@ +# Tipset Gas Reservations (Multi‑Stage Execution) + +This document summarizes activation, operational behaviour, and observability for tipset‑scope gas reservations as implemented in this branch. + +## Activation and Gating + +- **Network version:** reservations become consensus‑critical at **network.Version28** (the “Xx” network upgrade). The activation epoch is the upgrade height for Version28 on each network (see `build/buildconstants/params_*.go` for the concrete `UpgradeXxHeight` per network). +- **Pre‑activation behaviour (nv < 28):** + - Tipset reservations are controlled by two Lotus feature flags (`Feature.MultiStageReservations` and `Feature.MultiStageReservationsStrict`), with environment variables providing defaults. + - `Feature.MultiStageReservations` (default: false; enabled when `LOTUS_ENABLE_TIPSET_RESERVATIONS=1`) toggles whether Lotus attempts tipset‑scope reservations and enables reservation‑aware pre‑pack in the mempool when configured. + - `Feature.MultiStageReservationsStrict` (default: false; can be enabled via `LOTUS_ENABLE_TIPSET_RESERVATIONS_STRICT=1` or config) controls whether non‑NotImplemented Begin/End reservation failures invalidate tipsets pre‑activation. + - When `Feature.MultiStageReservations` is false, Lotus operates in legacy mode (no `BeginReservations` / `EndReservations` calls). + - When `Feature.MultiStageReservations` is true and `Feature.MultiStageReservationsStrict` is **false** (non‑strict): + - Lotus builds a per‑sender reservation plan for each applied tipset (`Σ(gas_limit * gas_fee_cap)` per sender) and calls `BeginReservations` / `EndReservations` on the FVM if the engine implements them. + - Any Begin/End reservation failure (including `ErrReservationsInsufficientFunds` and `ErrReservationsPlanTooLarge`) is treated as best‑effort: Lotus logs and falls back to legacy mode for that tipset; consensus validity is unchanged. + - When `Feature.MultiStageReservations` is true and `Feature.MultiStageReservationsStrict` is **true** (strict): + - Begin/End reservation failures such as `ErrReservationsInsufficientFunds` and `ErrReservationsPlanTooLarge` cause the tipset to be considered invalid even before nv28, matching post‑activation behaviour. + - If the engine returns `ErrReservationsNotImplemented` from `BeginReservations` or `EndReservations`, Lotus logs a debug message and continues in legacy mode (no reservations); consensus validity is unchanged. + - In all pre‑activation modes, node‑error classes (`ErrReservationsSessionOpen`, `ErrReservationsSessionClosed`, `ErrReservationsNonZeroRemainder`, `ErrReservationsOverflow`, `ErrReservationsInvariantViolation`) surface as node errors and are **not** downgraded to best‑effort. +- **Post‑activation behaviour (nv ≥ 28):** + - `ReservationsEnabled` is always true; Lotus will always build a reservation plan and call `BeginReservations` / `EndReservations` when applying tipsets with explicit messages. + - If the engine returns `ErrReservationsNotImplemented` after activation, Lotus treats this as a **node error** (the node cannot validate under active rules); operators must upgrade the engine. + - Reservation failures from the engine at Begin/End (e.g., insufficient funds at `BeginReservations`, plan too large) cause the tipset to be considered invalid under consensus; node‑error classes remain node errors. + +## Feature Flags and Defaults + +- **Consensus and reservations (package `chain/consensus`):** + - `Feature.MultiStageReservations`: + - Default: `false`, unless `LOTUS_ENABLE_TIPSET_RESERVATIONS=1` is set in the Lotus process environment at startup. + - Pre‑activation: when `true`, Lotus attempts tipset‑scope reservations and enables reservation‑aware pre‑pack in the mempool when `EnableReservationPrePack` is set. + - Post‑activation: ignored by consensus; reservations are always enabled for validation (`ReservationsEnabled` is always true for nv ≥ 28). + - `Feature.MultiStageReservationsStrict`: + - Default: `false` (non‑strict). + - Pre‑activation: when `true`, non‑NotImplemented reservation failures at Begin/End (`ErrReservationsInsufficientFunds`, `ErrReservationsPlanTooLarge`) invalidate tipsets instead of falling back to legacy mode. + - Post‑activation: redundant; reservations are already strict by consensus rules. +- **Mempool pre‑pack simulation:** + - Controlled by `EnableReservationPrePack` in the mempool config (`types.MpoolConfig`, surfaced via `MpoolConfig.EnableReservationPrePack`): + - Default: `false` (opt‑in and advisory). + - When `true`, and when tipset reservations are enabled for the current network version (`ReservationsEnabled` is true at the selection height), the packer performs a reservation‑aware simulation using `Σ(gas_limit * gas_fee_cap)` per sender at the selection base state. + - Chains whose additional reservation would exceed the sender’s on‑chain balance at the selection base state are skipped. + - This pre‑pack simulation is advisory only and does not affect consensus; it simply avoids constructing blocks that would later fail reservation checks at `BeginReservations`. + +## Detecting Reservation Failures + +### Logs + +- The consensus reservations helper logs under the `reservations` subsystem: + - `starting tipset reservations` with fields: + - `senders`: number of unique senders in the plan. + - `total`: sum of all per‑sender reservations in attoFIL. + - `skipping tipset reservations for empty plan` when no explicit messages are present. +- FVM‑level reservation errors are surfaced via the `vm` logs and the corresponding Go errors in `chain/vm/fvm.go`: + - `ErrReservationsInsufficientFunds` + - `ErrReservationsSessionOpen` + - `ErrReservationsSessionClosed` + - `ErrReservationsNonZeroRemainder` + - `ErrReservationsPlanTooLarge` + - `ErrReservationsOverflow` + - `ErrReservationsInvariantViolation` +- Pre‑activation, `ErrReservationsNotImplemented` is logged at debug level and causes Lotus to continue in legacy mode for that tipset. Post‑activation, it is treated as a node error. + +### Metrics + +- Two OpenCensus metrics capture plan‑level reservation behaviour: + - `vm/reservations_plan_senders` (`metrics.ReservationPlanSenders`): + - Number of unique senders in the reservation plan for each applied tipset. + - `vm/reservations_plan_total_atto` (`metrics.ReservationPlanTotal`): + - Total reserved maximum gas cost across all senders in attoFIL for each tipset. +- Recommended operator checks: + - Alert on sudden drops to zero of both metrics after nv28 on a healthy chain (may indicate reservations are not being attempted when they should be). + - Track long‑term distribution of `reservations_plan_total_atto` to identify tipsets with unusually large reserved gas cost. + +## Upgrade Expectations + +- **Before nv28 (activation upgrade):** + - Nodes may: + - Run without reservations (default behaviour). + - Opt‑in to reservations and reservation‑aware pre‑pack on dev/test networks by: + - Enabling `Feature.MultiStageReservations` (e.g., via `LOTUS_ENABLE_TIPSET_RESERVATIONS=1`). + - Optionally enabling `Feature.MultiStageReservationsStrict` to exercise strict invalidation behaviour before activation. + - Enabling `EnableReservationPrePack` in the mempool config for advisory pre‑pack simulation. + - The engine may or may not implement reservations; pre‑activation, `ErrReservationsNotImplemented` is treated as a benign indication that the node is running an older engine and triggers a legacy fallback for the affected tipset. +- **At and after nv28:** + - Reservations are part of consensus: + - All mainnet nodes must run an engine version that implements the reservation FFI (`BeginReservations` / `EndReservations`). + - Tipsets that fail reservation checks are invalid under the new rules. + - Operators should: + - Upgrade Lotus and the underlying engine (filecoin‑ffi / ref‑fvm) ahead of the nv28 upgrade epoch (`UpgradeXxHeight` per network). + - Monitor `reservations` logs and reservation metrics during and after the upgrade window. + - Receipts, `GasOutputs`, and on‑chain accounting remain identical to legacy behaviour; the reservation ledger is internal to the engine and does not change actor code or on‑chain state. diff --git a/extern/filecoin-ffi b/extern/filecoin-ffi index 3d5f2311117..b503fd16c06 160000 --- a/extern/filecoin-ffi +++ b/extern/filecoin-ffi @@ -1 +1 @@ -Subproject commit 3d5f23111173a8f8449b2aefd8fc7c42acc01362 +Subproject commit b503fd16c069cc0dd16469457d99415fb8833cd4 diff --git a/metrics/metrics.go b/metrics/metrics.go index 940e547a45c..acab44cfe1d 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -134,6 +134,8 @@ var ( VMApplied = stats.Int64("vm/applied", "Counter for messages (including internal messages) processed by the VM", stats.UnitDimensionless) VMExecutionWaiting = stats.Int64("vm/execution_waiting", "Counter for VM executions waiting to be assigned to a lane", stats.UnitDimensionless) VMExecutionRunning = stats.Int64("vm/execution_running", "Counter for running VM executions", stats.UnitDimensionless) + ReservationPlanSenders = stats.Int64("vm/reservations_plan_senders", "Number of unique senders in the reservation plan for a tipset", stats.UnitDimensionless) + ReservationPlanTotal = stats.Int64("vm/reservations_plan_total_atto", "Total reserved maximum gas cost across senders in attoFIL for a tipset", stats.UnitDimensionless) // message fetching MessageFetchRequested = stats.Int64("message/fetch_requested", "Number of messages requested for fetch", stats.UnitDimensionless) @@ -461,6 +463,16 @@ var ( Aggregation: view.Sum(), TagKeys: []tag.Key{ExecutionLane, Network}, } + ReservationPlanSendersView = &view.View{ + Measure: ReservationPlanSenders, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{Network}, + } + ReservationPlanTotalView = &view.View{ + Measure: ReservationPlanTotal, + Aggregation: view.LastValue(), + TagKeys: []tag.Key{Network}, + } // message fetching MessageFetchRequestedView = &view.View{ @@ -860,6 +872,8 @@ var ChainNodeViews = append([]*view.View{ VMAppliedView, VMExecutionWaitingView, VMExecutionRunningView, + ReservationPlanSendersView, + ReservationPlanTotalView, MessageFetchRequestedView, MessageFetchLocalView, MessageFetchNetworkView,