diff --git a/builtin/v16/migration/miner.go b/builtin/v16/migration/miner.go new file mode 100644 index 00000000..87fe7260 --- /dev/null +++ b/builtin/v16/migration/miner.go @@ -0,0 +1,106 @@ +package migration + +import ( + "context" + "fmt" + "sync/atomic" + + miner15 "github.com/filecoin-project/go-state-types/builtin/v15/miner" + miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner" + "github.com/filecoin-project/go-state-types/migration" + "github.com/ipfs/go-cid" + cbor "github.com/ipfs/go-ipld-cbor" + "golang.org/x/xerrors" +) + +type minerMigrator struct { + OutCodeCID cid.Cid + EmptyPreCommittedSectorsHamtCid cid.Cid + EmptyPrecommitCleanUpAmtCid cid.Cid +} + +func newMinerMigrator( + _ context.Context, + _ cbor.IpldStore, + outCode, + emptyPreCommittedSectorsHamtCid, + emptyPrecommitCleanUpAmtCid cid.Cid, +) (*minerMigrator, error) { + return &minerMigrator{ + OutCodeCID: outCode, + EmptyPreCommittedSectorsHamtCid: emptyPreCommittedSectorsHamtCid, + EmptyPrecommitCleanUpAmtCid: emptyPrecommitCleanUpAmtCid, + }, nil +} + +// TOOD: remove these, they're just for debugging +var minerCount, withEmptyPrecommitCleanUpAmtCidCount, withEmptyPreCommittedSectorsHamtCidCount, migratedCount uint64 + +func (m *minerMigrator) MigrateState(ctx context.Context, store cbor.IpldStore, in migration.ActorMigrationInput) (result *migration.ActorMigrationResult, err error) { + var inState miner15.State + if err := store.Get(ctx, in.Head, &inState); err != nil { + return nil, xerrors.Errorf("failed to load miner state for %s: %w", in.Address, err) + } + + newHead := in.Head + + var epccuacc, epcshcc uint64 + var preCommittedSectors, preCommittedSectorsCleanUp *cid.Cid + if !m.EmptyPrecommitCleanUpAmtCid.Equals(inState.PreCommittedSectorsCleanUp) { + preCommittedSectorsCleanUp = &inState.PreCommittedSectorsCleanUp + } else { + epccuacc = atomic.AddUint64(&withEmptyPrecommitCleanUpAmtCidCount, 1) + } + if !m.EmptyPreCommittedSectorsHamtCid.Equals(inState.PreCommittedSectors) { + preCommittedSectors = &inState.PreCommittedSectors + } else { + epcshcc = atomic.AddUint64(&withEmptyPreCommittedSectorsHamtCidCount, 1) + } + + // we only need to update the state if one of these need to be nullable, otherwise the existing + // state, with a valid CID, will decode correctly even though they're now pointer fields + if preCommittedSectors == nil || preCommittedSectorsCleanUp == nil { + outState := miner16.State{ + Info: inState.Info, + PreCommitDeposits: inState.PreCommitDeposits, + LockedFunds: inState.LockedFunds, + VestingFunds: inState.VestingFunds, + FeeDebt: inState.FeeDebt, + InitialPledge: inState.InitialPledge, + PreCommittedSectors: preCommittedSectors, + PreCommittedSectorsCleanUp: preCommittedSectorsCleanUp, + AllocatedSectors: inState.AllocatedSectors, + Sectors: inState.Sectors, + ProvingPeriodStart: inState.ProvingPeriodStart, + CurrentDeadline: inState.CurrentDeadline, + Deadlines: inState.Deadlines, + EarlyTerminations: inState.EarlyTerminations, + DeadlineCronActive: inState.DeadlineCronActive, + } + + if newHead, err = store.Put(ctx, &outState); err != nil { + return nil, xerrors.Errorf("failed to put new state: %w", err) + } + + atomic.AddUint64(&migratedCount, 1) + } + + if nc := atomic.AddUint64(&minerCount, 1); nc%10000 == 0 { + fmt.Printf("Checked %d miners, %d with empty PreCommittedSectors, %d with empty PreCommittedSectorsCleanUp, %d migrated\n", nc, epcshcc, epccuacc, atomic.LoadUint64(&migratedCount)) + } + + return &migration.ActorMigrationResult{ + NewCodeCID: m.MigratedCodeCID(), + NewHead: newHead, + }, nil +} + +func (m *minerMigrator) MigratedCodeCID() cid.Cid { + return m.OutCodeCID +} + +func (m *minerMigrator) Deferred() bool { + return false +} + +var _ migration.ActorMigration = (*minerMigrator)(nil) diff --git a/builtin/v16/migration/top.go b/builtin/v16/migration/top.go index 930cc10d..217efb40 100644 --- a/builtin/v16/migration/top.go +++ b/builtin/v16/migration/top.go @@ -3,9 +3,10 @@ package migration import ( "context" - adt16 "github.com/filecoin-project/go-state-types/builtin/v16/util/adt" + "github.com/filecoin-project/go-state-types/builtin/v16/util/adt" system15 "github.com/filecoin-project/go-state-types/builtin/v13/system" + miner16 "github.com/filecoin-project/go-state-types/builtin/v16/miner" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin" @@ -24,7 +25,7 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("invalid migration config with %d workers", cfg.MaxWorkers) } - adtStore := adt16.WrapStore(ctx, store) + adtStore := adt.WrapStore(ctx, store) // Load input and output state trees actorsIn, err := builtin.LoadTree(adtStore, actorsRootIn) @@ -67,14 +68,22 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID // Set of prior version code CIDs for actors to defer during iteration, for explicit migration afterwards. deferredCodeIDs := make(map[cid.Cid]struct{}) + miner15Cid := cid.Undef + for _, oldEntry := range oldManifestData.Entries { newCodeCID, ok := newManifest.Get(oldEntry.Name) if !ok { return cid.Undef, xerrors.Errorf("code cid for %s actor not found in new manifest", oldEntry.Name) } + if oldEntry.Name == manifest.MinerKey { + miner15Cid = oldEntry.Code + } migrations[oldEntry.Code] = migration.CachedMigration(cache, migration.CodeMigrator{OutCodeCID: newCodeCID}) } + if miner15Cid == cid.Undef { + return cid.Undef, xerrors.Errorf("could not find miner actor in old manifest") + } // migrations that migrate both code and state, override entries in `migrations` // The System Actor @@ -90,6 +99,35 @@ func MigrateStateTree(ctx context.Context, store cbor.IpldStore, newManifestCID return cid.Undef, xerrors.Errorf("incomplete migration specification with %d code CIDs, need %d", len(migrations)+len(deferredCodeIDs), len(oldManifestData.Entries)) } + var emptyPreCommittedSectorsHamtCid cid.Cid + if hamt, err := adt.MakeEmptyMap(adtStore, builtin.DefaultHamtBitwidth); err != nil { + return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err) + } else { + if emptyPreCommittedSectorsHamtCid, err = hamt.Root(); err != nil { + return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err) + } + } + var emptyPrecommitCleanUpAmtCid cid.Cid + if amt, err := adt.MakeEmptyArray(adtStore, miner16.PrecommitCleanUpAmtBitwidth); err != nil { + return cid.Undef, xerrors.Errorf("failed to create empty precommit clean up amount array: %w", err) + } else { + if emptyPrecommitCleanUpAmtCid, err = amt.Root(); err != nil { + return cid.Undef, xerrors.Errorf("failed to get root of empty precommit clean up amount array: %w", err) + } + } + + miner16Cid, ok := newManifest.Get(manifest.MinerKey) + if !ok { + return cid.Undef, xerrors.Errorf("code cid for miner actor not found in new manifest") + } + + minerMig, err := newMinerMigrator(ctx, store, miner16Cid, emptyPreCommittedSectorsHamtCid, emptyPrecommitCleanUpAmtCid) + if err != nil { + return cid.Undef, xerrors.Errorf("failed to create miner migrator: %w", err) + } + + migrations[miner15Cid] = migration.CachedMigration(cache, minerMig) + actorsOut, err := migration.RunMigration(ctx, cfg, cache, store, log, actorsIn, migrations) if err != nil { return cid.Undef, xerrors.Errorf("failed to run migration: %w", err) diff --git a/builtin/v16/miner/cbor_gen.go b/builtin/v16/miner/cbor_gen.go index 14a2dfac..a2a2f914 100644 --- a/builtin/v16/miner/cbor_gen.go +++ b/builtin/v16/miner/cbor_gen.go @@ -70,14 +70,26 @@ func (t *State) MarshalCBOR(w io.Writer) error { // t.PreCommittedSectors (cid.Cid) (struct) - if err := cbg.WriteCid(cw, t.PreCommittedSectors); err != nil { - return xerrors.Errorf("failed to write cid field t.PreCommittedSectors: %w", err) + if t.PreCommittedSectors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PreCommittedSectors); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommittedSectors: %w", err) + } } // t.PreCommittedSectorsCleanUp (cid.Cid) (struct) - if err := cbg.WriteCid(cw, t.PreCommittedSectorsCleanUp); err != nil { - return xerrors.Errorf("failed to write cid field t.PreCommittedSectorsCleanUp: %w", err) + if t.PreCommittedSectorsCleanUp == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PreCommittedSectorsCleanUp); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommittedSectorsCleanUp: %w", err) + } } // t.AllocatedSectors (cid.Cid) (struct) @@ -214,24 +226,44 @@ func (t *State) UnmarshalCBOR(r io.Reader) (err error) { { - c, err := cbg.ReadCid(cr) + b, err := cr.ReadByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.PreCommittedSectors: %w", err) + return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } - t.PreCommittedSectors = c + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommittedSectors: %w", err) + } + + t.PreCommittedSectors = &c + } } // t.PreCommittedSectorsCleanUp (cid.Cid) (struct) { - c, err := cbg.ReadCid(cr) + b, err := cr.ReadByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.PreCommittedSectorsCleanUp: %w", err) + return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommittedSectorsCleanUp: %w", err) + } - t.PreCommittedSectorsCleanUp = c + t.PreCommittedSectorsCleanUp = &c + } } // t.AllocatedSectors (cid.Cid) (struct) diff --git a/builtin/v16/miner/invariants.go b/builtin/v16/miner/invariants.go index 0d7521a0..935c2d61 100644 --- a/builtin/v16/miner/invariants.go +++ b/builtin/v16/miner/invariants.go @@ -778,54 +778,58 @@ func CheckPreCommits(st *State, store adt.Store, allocatedSectorsMap map[uint64] // invert pre-commit clean up queue into a lookup by sector number cleanUpEpochs := make(map[uint64]abi.ChainEpoch) - if cleanUpQ, err := util.LoadBitfieldQueue(store, st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil { - acc.Addf("error loading pre-commit clean up queue: %v", err) - } else { - err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error { - quantized := quant.QuantizeUp(epoch) - acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch) - if err = bf.ForEach(func(secNum uint64) error { - cleanUpEpochs[secNum] = epoch + if st.PreCommittedSectorsCleanUp != nil { + if cleanUpQ, err := util.LoadBitfieldQueue(store, *st.PreCommittedSectorsCleanUp, st.QuantSpecEveryDeadline(), PrecommitCleanUpAmtBitwidth); err != nil { + acc.Addf("error loading pre-commit clean up queue: %v", err) + } else { + err = cleanUpQ.ForEach(func(epoch abi.ChainEpoch, bf bitfield.BitField) error { + quantized := quant.QuantizeUp(epoch) + acc.Require(quantized == epoch, "precommit expiration %d is not quantized", epoch) + if err = bf.ForEach(func(secNum uint64) error { + cleanUpEpochs[secNum] = epoch + return nil + }); err != nil { + acc.Addf("error iteration pre-commit expiration bitfield: %v", err) + } return nil - }); err != nil { - acc.Addf("error iteration pre-commit expiration bitfield: %v", err) - } - return nil - }) - acc.RequireNoError(err, "error iterating pre-commit clean up queue") + }) + acc.RequireNoError(err, "error iterating pre-commit clean up queue") + } } precommitTotal := big.Zero() - if precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil { - acc.Addf("error loading precommitted sectors: %v", err) - } else { - var precommit SectorPreCommitOnChainInfo - err = precommitted.ForEach(&precommit, func(key string) error { - secNum, err := abi.ParseUIntKey(key) - if err != nil { - acc.Addf("error parsing pre-commit key as uint: %v", err) - return nil - } - - allocated := false - if allocatedSectorsMap != nil { - allocated = allocatedSectorsMap[secNum] - } else { - allocated, err = allocatedSectorsBf.IsSet(secNum) + if st.PreCommittedSectors != nil { + if precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth); err != nil { + acc.Addf("error loading precommitted sectors: %v", err) + } else { + var precommit SectorPreCommitOnChainInfo + err = precommitted.ForEach(&precommit, func(key string) error { + secNum, err := abi.ParseUIntKey(key) if err != nil { - acc.Addf("error checking allocated sectors: %v", err) + acc.Addf("error parsing pre-commit key as uint: %v", err) return nil } - } - acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum) - _, found := cleanUpEpochs[secNum] - acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch) + allocated := false + if allocatedSectorsMap != nil { + allocated = allocatedSectorsMap[secNum] + } else { + allocated, err = allocatedSectorsBf.IsSet(secNum) + if err != nil { + acc.Addf("error checking allocated sectors: %v", err) + return nil + } + } + acc.Require(allocated, "pre-committed sector number has not been allocated %d", secNum) - precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit) - return nil - }) - acc.RequireNoError(err, "error iterating pre-committed sectors") + _, found := cleanUpEpochs[secNum] + acc.Require(found, "no clean up epoch for pre-commit at %d", precommit.PreCommitEpoch) + + precommitTotal = big.Add(precommitTotal, precommit.PreCommitDeposit) + return nil + }) + acc.RequireNoError(err, "error iterating pre-committed sectors") + } } acc.Require(st.PreCommitDeposits.Equals(precommitTotal), diff --git a/builtin/v16/miner/miner_state.go b/builtin/v16/miner/miner_state.go index f156686e..f8550f60 100644 --- a/builtin/v16/miner/miner_state.go +++ b/builtin/v16/miner/miner_state.go @@ -35,10 +35,10 @@ type State struct { InitialPledge abi.TokenAmount // Sum of initial pledge requirements of all active sectors // Sectors that have been pre-committed but not yet proven. - PreCommittedSectors cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo + PreCommittedSectors *cid.Cid // Map, HAMT[SectorNumber]SectorPreCommitOnChainInfo // PreCommittedSectorsCleanUp maintains the state required to cleanup expired PreCommittedSectors. - PreCommittedSectorsCleanUp cid.Cid // BitFieldQueue (AMT[Epoch]*BitField) + PreCommittedSectorsCleanUp *cid.Cid // BitFieldQueue (AMT[Epoch]*BitField) // Allocated sector IDs. Sector IDs can never be reused once allocated. AllocatedSectors cid.Cid // BitField @@ -217,7 +217,11 @@ func (st *State) QuantSpecForDeadline(dlIdx uint64) builtin.QuantSpec { } func (st *State) GetPrecommittedSector(store adt.Store, sectorNo abi.SectorNumber) (*SectorPreCommitOnChainInfo, bool, error) { - precommitted, err := adt.AsMap(store, st.PreCommittedSectors, builtin.DefaultHamtBitwidth) + if st.PreCommittedSectors == nil { + return nil, false, nil + } + + precommitted, err := adt.AsMap(store, *st.PreCommittedSectors, builtin.DefaultHamtBitwidth) if err != nil { return nil, false, err }