Skip to content

Commit 593dad6

Browse files
frristplacer14
frrist
and
placer14
authored
feat: add method for finding oldest state and computing state (#1038)
* feat: add method for finding oldest state and computing state * chore: Some UX around the `chain state-*` commands (#1042) * chore: Refactor state list commands into one; Output JSON * chore: Add docs and prefixes * chore: fmt the go Co-authored-by: Mike Greenberg <[email protected]> * fixup: remove unused flag and progbar setting Co-authored-by: Mike Greenberg <[email protected]> Co-authored-by: Mike Greenberg <[email protected]>
1 parent 2c06b9b commit 593dad6

File tree

4 files changed

+319
-14
lines changed

4 files changed

+319
-14
lines changed

‎commands/chain.go‎

Lines changed: 202 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/ipfs/go-cid"
1818
"github.com/jedib0t/go-pretty/v6/table"
1919
"github.com/urfave/cli/v2"
20+
"gopkg.in/cheggaaa/pb.v1"
2021

2122
"github.com/filecoin-project/lily/chain/actors"
2223
init_ "github.com/filecoin-project/lily/chain/actors/builtin/init"
@@ -42,6 +43,9 @@ var ChainCmd = &cli.Command{
4243
ChainListCmd,
4344
ChainSetHeadCmd,
4445
ChainActorCodesCmd,
46+
ChainStateInspect,
47+
ChainStateCompute,
48+
ChainStateComputeRange,
4549
},
4650
}
4751

@@ -74,24 +78,188 @@ var ChainActorCodesCmd = &cli.Command{
7478
},
7579
}
7680

77-
func printSortedActorVersions(av map[actors.Version]cid.Cid) error {
78-
t := table.NewWriter()
79-
t.AppendHeader(table.Row{"Version", "Name", "Family", "Code"})
80-
var versions []int
81-
for v := range av {
82-
versions = append(versions, int(v))
81+
func marshalReport(reports []*lily.StateReport, verbose bool) ([]byte, error) {
82+
type stateHeights struct {
83+
Newest int64 `json:"newest"`
84+
Oldest int64 `json:"oldest"`
8385
}
84-
sort.Ints(versions)
85-
for _, v := range versions {
86-
name, family, err := util.ActorNameAndFamilyFromCode(av[actors.Version(v)])
86+
type summarizedHeights struct {
87+
Messages stateHeights `json:"messages"`
88+
StateRoots stateHeights `json:"stateroots"`
89+
}
90+
type hasState struct {
91+
Messages bool `json:"messages"`
92+
Receipts bool `json:"receipts"`
93+
StateRoot bool `json:"stateroot"`
94+
}
95+
type stateReport struct {
96+
Summary summarizedHeights `json:"summary"`
97+
Detail map[int64]hasState `json:"details,omitempty"`
98+
}
99+
100+
var (
101+
details = make(map[int64]hasState)
102+
headSet bool
103+
head = reports[0]
104+
oldestMessage = &lily.StateReport{}
105+
oldestStateRoot = &lily.StateReport{}
106+
)
107+
108+
for _, r := range reports {
109+
if verbose {
110+
details[r.Height] = hasState{
111+
Messages: r.HasMessages,
112+
Receipts: r.HasReceipts,
113+
StateRoot: r.HasState,
114+
}
115+
}
116+
if !headSet && (r.HasState && r.HasMessages && r.HasReceipts) {
117+
head = r
118+
headSet = true
119+
}
120+
if r.HasState {
121+
oldestStateRoot = r
122+
}
123+
if r.HasMessages {
124+
oldestMessage = r
125+
}
126+
}
127+
128+
compiledReport := stateReport{
129+
Detail: details,
130+
Summary: summarizedHeights{
131+
Messages: stateHeights{Newest: head.Height, Oldest: oldestMessage.Height},
132+
StateRoots: stateHeights{Newest: head.Height, Oldest: oldestStateRoot.Height},
133+
},
134+
}
135+
136+
reportOut, err := json.Marshal(compiledReport)
137+
if err != nil {
138+
return nil, err
139+
}
140+
141+
return reportOut, nil
142+
}
143+
144+
var ChainStateInspect = &cli.Command{
145+
Name: "state-inspect",
146+
Usage: "Returns details about each epoch's state in the local datastore",
147+
Flags: []cli.Flag{
148+
&cli.Uint64Flag{
149+
Name: "limit",
150+
Aliases: []string{"l"},
151+
Value: 100,
152+
Usage: "Limit traversal of statetree when searching for oldest state by `N` heights starting from most recent",
153+
},
154+
&cli.BoolFlag{
155+
Name: "verbose",
156+
Aliases: []string{"v"},
157+
Usage: "Include detailed information about the completeness of state for all traversed height(s) starting from most recent",
158+
},
159+
},
160+
Action: func(cctx *cli.Context) error {
161+
ctx := lotuscli.ReqContext(cctx)
162+
lapi, closer, err := GetAPI(ctx)
87163
if err != nil {
88164
return err
89165
}
90-
t.AppendRow(table.Row{v, name, family, av[actors.Version(v)]})
91-
t.AppendSeparator()
92-
}
93-
fmt.Println(t.Render())
94-
return nil
166+
defer closer()
167+
168+
report, err := lapi.FindOldestState(ctx, cctx.Int64("limit"))
169+
if err != nil {
170+
return err
171+
}
172+
sort.Slice(report, func(i, j int) bool {
173+
return report[i].Height > report[j].Height
174+
})
175+
176+
out, err := marshalReport(report, cctx.Bool("verbose"))
177+
if err != nil {
178+
return err
179+
}
180+
fmt.Println(string(out))
181+
return nil
182+
},
183+
}
184+
185+
var ChainStateComputeRange = &cli.Command{
186+
Name: "state-compute",
187+
Usage: "Generates the state at epoch `N`",
188+
Flags: []cli.Flag{
189+
&cli.Uint64Flag{
190+
Name: "epoch",
191+
Aliases: []string{"e"},
192+
Required: true,
193+
},
194+
},
195+
Action: func(cctx *cli.Context) error {
196+
ctx := lotuscli.ReqContext(cctx)
197+
lapi, closer, err := GetAPI(ctx)
198+
if err != nil {
199+
return err
200+
}
201+
defer closer()
202+
203+
head, err := lapi.ChainHead(ctx)
204+
if err != nil {
205+
return err
206+
}
207+
ts, err := lapi.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(cctx.Uint64("epoch")), head.Key())
208+
if err != nil {
209+
return err
210+
}
211+
212+
_, err = lapi.StateCompute(ctx, ts.Key())
213+
return err
214+
215+
},
216+
}
217+
218+
var ChainStateCompute = &cli.Command{
219+
Name: "state-compute-range",
220+
Usage: "Generates the state from epoch `FROM` to epoch `TO`",
221+
Flags: []cli.Flag{
222+
&cli.Uint64Flag{
223+
Name: "from",
224+
Required: true,
225+
},
226+
&cli.Uint64Flag{
227+
Name: "to",
228+
Required: true,
229+
},
230+
},
231+
Action: func(cctx *cli.Context) error {
232+
ctx := lotuscli.ReqContext(cctx)
233+
lapi, closer, err := GetAPI(ctx)
234+
if err != nil {
235+
return err
236+
}
237+
defer closer()
238+
239+
head, err := lapi.ChainHead(ctx)
240+
if err != nil {
241+
return err
242+
}
243+
bar := pb.StartNew(int(cctx.Uint64("to") - cctx.Uint64("from")))
244+
bar.ShowTimeLeft = true
245+
bar.ShowPercent = true
246+
bar.Units = pb.U_NO
247+
for i := cctx.Int64("from"); i <= cctx.Int64("to"); i++ {
248+
ts, err := lapi.ChainGetTipSetByHeight(ctx, abi.ChainEpoch(i), head.Key())
249+
if err != nil {
250+
return err
251+
}
252+
253+
_, err = lapi.StateCompute(ctx, ts.Key())
254+
if err != nil {
255+
return err
256+
}
257+
bar.Add(1)
258+
}
259+
bar.Finish()
260+
return nil
261+
262+
},
95263
}
96264

97265
var ChainHeadCmd = &cli.Command{
@@ -547,3 +715,23 @@ func parseTipSet(ctx context.Context, api lily.LilyAPI, vals []string) (*types.T
547715

548716
return types.NewTipSet(headers)
549717
}
718+
719+
func printSortedActorVersions(av map[actors.Version]cid.Cid) error {
720+
t := table.NewWriter()
721+
t.AppendHeader(table.Row{"Version", "Name", "Family", "Code"})
722+
var versions []int
723+
for v := range av {
724+
versions = append(versions, int(v))
725+
}
726+
sort.Ints(versions)
727+
for _, v := range versions {
728+
name, family, err := util.ActorNameAndFamilyFromCode(av[actors.Version(v)])
729+
if err != nil {
730+
return err
731+
}
732+
t.AppendRow(table.Row{v, name, family, av[actors.Version(v)]})
733+
t.AppendSeparator()
734+
}
735+
fmt.Println(t.Render())
736+
return nil
737+
}

‎lens/lily/api.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ type LilyAPI interface {
7171
NetPeerInfo(context.Context, peer.ID) (*api.ExtendedPeerInfo, error)
7272

7373
StartTipSetWorker(ctx context.Context, cfg *LilyTipSetWorkerConfig) (*schedule.JobSubmitResult, error)
74+
75+
FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error)
76+
StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error)
7477
}
7578
type LilyJobConfig struct {
7679
// Name is the name of the job.

‎lens/lily/impl.go‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -624,6 +624,109 @@ func (m *LilyNodeAPI) LilySurvey(_ context.Context, cfg *LilySurveyConfig) (*sch
624624
return res, nil
625625
}
626626

627+
type StateReport struct {
628+
Height int64
629+
TipSet *types.TipSet
630+
HasMessages bool
631+
HasReceipts bool
632+
HasState bool
633+
}
634+
635+
func (m *LilyNodeAPI) StateCompute(ctx context.Context, key types.TipSetKey) (interface{}, error) {
636+
ts, err := m.ChainAPI.ChainGetTipSet(ctx, key)
637+
if err != nil {
638+
return nil, err
639+
}
640+
_, _, err = m.StateManager.TipSetState(ctx, ts)
641+
return nil, err
642+
}
643+
644+
func (m *LilyNodeAPI) FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error) {
645+
head, err := m.ChainHead(ctx)
646+
if err != nil {
647+
return nil, err
648+
}
649+
650+
var out []*StateReport
651+
var oldestEpochLimit = head.Height() - abi.ChainEpoch(limit)
652+
653+
for i := int64(0); true; i++ {
654+
maybeBaseTS, err := m.ChainGetTipSetByHeight(ctx, head.Height()-abi.ChainEpoch(i), head.Key())
655+
if err != nil {
656+
return nil, err
657+
}
658+
659+
maybeFullTS := TryLoadFullTipSet(ctx, m, maybeBaseTS)
660+
out = append(out, &StateReport{
661+
Height: int64(maybeBaseTS.Height()),
662+
TipSet: maybeBaseTS,
663+
HasMessages: maybeFullTS.HasMessages,
664+
HasReceipts: maybeFullTS.HasReceipts,
665+
HasState: maybeFullTS.HasState,
666+
})
667+
if (head.Height() - abi.ChainEpoch(i)) <= oldestEpochLimit {
668+
break
669+
}
670+
}
671+
return out, nil
672+
}
673+
674+
type FullBlock struct {
675+
Header *types.BlockHeader
676+
BlsMessages []*types.Message
677+
SecpMessages []*types.SignedMessage
678+
}
679+
680+
type FullTipSet struct {
681+
Blocks []*FullBlock
682+
TipSet *types.TipSet
683+
684+
HasMessages bool
685+
HasState bool
686+
HasReceipts bool
687+
}
688+
689+
func TryLoadFullTipSet(ctx context.Context, m *LilyNodeAPI, ts *types.TipSet) *FullTipSet {
690+
var (
691+
out []*FullBlock
692+
err error
693+
hasState = true
694+
hasMessages = true
695+
hasReceipts = true
696+
)
697+
698+
for _, b := range ts.Blocks() {
699+
fb := &FullBlock{Header: b}
700+
701+
fb.BlsMessages, fb.SecpMessages, err = m.ChainAPI.Chain.MessagesForBlock(ctx, b)
702+
if err != nil {
703+
log.Debugw("failed to load messages", "height", b.Height)
704+
hasMessages = false
705+
}
706+
out = append(out, fb)
707+
}
708+
709+
_, err = adt.AsArray(m.ChainAPI.Chain.ActorStore(ctx), ts.Blocks()[0].ParentMessageReceipts)
710+
if err != nil {
711+
log.Debugw("failed to load receipts", "height", ts.Blocks()[0].Height)
712+
hasReceipts = false
713+
}
714+
715+
_, err = m.StateManager.ParentState(ts)
716+
if err != nil {
717+
log.Debugw("failed to load statetree", "height", ts.Blocks()[0].Height)
718+
hasState = false
719+
}
720+
721+
return &FullTipSet{
722+
Blocks: out,
723+
TipSet: ts,
724+
HasMessages: hasMessages,
725+
HasState: hasState,
726+
HasReceipts: hasReceipts,
727+
}
728+
}
729+
627730
// used for debugging querries, call ORM.AddHook and this will print all queries.
628731
type LogQueryHook struct{}
629732

‎lens/lily/struct.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,20 @@ type LilyAPIStruct struct {
7777
NetPeerInfo func(context.Context, peer.ID) (*api.ExtendedPeerInfo, error) `perm:"read"`
7878

7979
StartTipSetWorker func(ctx context.Context, cfg *LilyTipSetWorkerConfig) (*schedule.JobSubmitResult, error) `perm:"read"`
80+
81+
FindOldestState func(ctx context.Context, limit int64) ([]*StateReport, error) `perm:"read"`
82+
StateCompute func(ctx context.Context, tsk types.TipSetKey) (interface{}, error) `perm:"read"`
8083
}
8184
}
8285

86+
func (s *LilyAPIStruct) StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error) {
87+
return s.Internal.StateCompute(ctx, tsk)
88+
}
89+
90+
func (s *LilyAPIStruct) FindOldestState(ctx context.Context, limit int64) ([]*StateReport, error) {
91+
return s.Internal.FindOldestState(ctx, limit)
92+
}
93+
8394
func (s *LilyAPIStruct) ChainGetTipSetAfterHeight(ctx context.Context, epoch abi.ChainEpoch, key types.TipSetKey) (*types.TipSet, error) {
8495
return s.Internal.ChainGetTipSetAfterHeight(ctx, epoch, key)
8596
}

0 commit comments

Comments
 (0)