From 43c66d8215d5e4195ab1f5d13b1b3db03847721b Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 12:19:17 -0800 Subject: [PATCH 01/48] refactor: add InfoCid to miner interface --- chain/actors/adt/diff/generic.go | 2 ++ chain/actors/builtin/miner/state.go.template | 4 ++++ chain/actors/builtin/miner/v0.go | 4 ++++ chain/actors/builtin/miner/v2.go | 4 ++++ chain/actors/builtin/miner/v3.go | 4 ++++ chain/actors/builtin/miner/v4.go | 4 ++++ chain/actors/builtin/miner/v5.go | 4 ++++ chain/actors/builtin/miner/v6.go | 4 ++++ chain/actors/builtin/miner/v7.go | 4 ++++ chain/actors/builtin/miner/v8.go | 4 ++++ chain/actors/builtin/miner/v9.go | 4 ++++ 11 files changed, 42 insertions(+) diff --git a/chain/actors/adt/diff/generic.go b/chain/actors/adt/diff/generic.go index b3fa72926..4ae4aa054 100644 --- a/chain/actors/adt/diff/generic.go +++ b/chain/actors/adt/diff/generic.go @@ -28,6 +28,7 @@ type ArrayDiffer interface { // - All values that exist in curArr nnd not in prevArr are passed to ArrayDiffer.Add() // - All values that exist in preArr and in curArr are passed to ArrayDiffer.Modify() // - It is the responsibility of ArrayDiffer.Modify() to determine if the values it was passed have been modified. +// // If `preArr` and `curArr` are both backed by /v3/AMTs with the same bitwidth use the more efficient Amt method. func CompareArray(preArr, curArr adt.Array, out ArrayDiffer) error { notNew := make(map[int64]struct{}, curArr.Length()) @@ -84,6 +85,7 @@ type MapDiffer interface { // - All values that exist in curMap nnd not in prevArr are passed to MapDiffer.Add() // - All values that exist in preMap and in curMap are passed to MapDiffer.Modify() // - It is the responsibility of ArrayDiffer.Modify() to determine if the values it was passed have been modified. +// // If `preMap` and `curMap` are both backed by /v3/HAMTs with the same bitwidth and hash function use the more efficient Hamt method. func CompareMap(preMap, curMap adt.Map, out MapDiffer) error { notNew := make(map[string]struct{}) diff --git a/chain/actors/builtin/miner/state.go.template b/chain/actors/builtin/miner/state.go.template index 7d8c4b967..48278794c 100644 --- a/chain/actors/builtin/miner/state.go.template +++ b/chain/actors/builtin/miner/state.go.template @@ -705,3 +705,7 @@ func (s *state{{.v}}) Code() cid.Cid { return code } + +func (s *state{{.v}}) InfoCid() cid.Cid { + return s.State.Info +} \ No newline at end of file diff --git a/chain/actors/builtin/miner/v0.go b/chain/actors/builtin/miner/v0.go index eca0f28d1..6d28bb2b8 100644 --- a/chain/actors/builtin/miner/v0.go +++ b/chain/actors/builtin/miner/v0.go @@ -611,3 +611,7 @@ func (s *state0) Code() cid.Cid { return code } + +func (s *state0) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v2.go b/chain/actors/builtin/miner/v2.go index 2fd9f6102..d4ac71ee7 100644 --- a/chain/actors/builtin/miner/v2.go +++ b/chain/actors/builtin/miner/v2.go @@ -641,3 +641,7 @@ func (s *state2) Code() cid.Cid { return code } + +func (s *state2) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v3.go b/chain/actors/builtin/miner/v3.go index b07ab6487..867f0ff4f 100644 --- a/chain/actors/builtin/miner/v3.go +++ b/chain/actors/builtin/miner/v3.go @@ -642,3 +642,7 @@ func (s *state3) Code() cid.Cid { return code } + +func (s *state3) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v4.go b/chain/actors/builtin/miner/v4.go index 5a60a2dc0..0982bac3b 100644 --- a/chain/actors/builtin/miner/v4.go +++ b/chain/actors/builtin/miner/v4.go @@ -642,3 +642,7 @@ func (s *state4) Code() cid.Cid { return code } + +func (s *state4) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v5.go b/chain/actors/builtin/miner/v5.go index 15e7c53ed..646724c18 100644 --- a/chain/actors/builtin/miner/v5.go +++ b/chain/actors/builtin/miner/v5.go @@ -642,3 +642,7 @@ func (s *state5) Code() cid.Cid { return code } + +func (s *state5) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v6.go b/chain/actors/builtin/miner/v6.go index ec4d81dfe..94a0fa171 100644 --- a/chain/actors/builtin/miner/v6.go +++ b/chain/actors/builtin/miner/v6.go @@ -642,3 +642,7 @@ func (s *state6) Code() cid.Cid { return code } + +func (s *state6) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v7.go b/chain/actors/builtin/miner/v7.go index ea5f39316..95b483e67 100644 --- a/chain/actors/builtin/miner/v7.go +++ b/chain/actors/builtin/miner/v7.go @@ -643,3 +643,7 @@ func (s *state7) Code() cid.Cid { return code } + +func (s *state7) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v8.go b/chain/actors/builtin/miner/v8.go index b33d7fa09..81c1feaa3 100644 --- a/chain/actors/builtin/miner/v8.go +++ b/chain/actors/builtin/miner/v8.go @@ -642,3 +642,7 @@ func (s *state8) Code() cid.Cid { return code } + +func (s *state8) InfoCid() cid.Cid { + return s.State.Info +} diff --git a/chain/actors/builtin/miner/v9.go b/chain/actors/builtin/miner/v9.go index 26f930e48..79aaaa7ce 100644 --- a/chain/actors/builtin/miner/v9.go +++ b/chain/actors/builtin/miner/v9.go @@ -606,3 +606,7 @@ func (s *state9) Code() cid.Cid { return code } + +func (s *state9) InfoCid() cid.Cid { + return s.State.Info +} From 1dca7c79cb112f27dbfb3cce16cf6a1697e86880 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 12:19:47 -0800 Subject: [PATCH 02/48] chore: fix code comment --- storage/sql.go | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/storage/sql.go b/storage/sql.go index 5aa1602cd..b314e1557 100644 --- a/storage/sql.go +++ b/storage/sql.go @@ -437,24 +437,27 @@ func (s *TxStorage) PersistModel(ctx context.Context, m interface{}) error { return nil } -// GenerateUpsertString accepts a lily model and returns two string containing SQL that may be used +// GenerateUpsertStrings accepts a lily model and returns two string containing SQL that may be used // to upsert the model. The first string is the conflict statement and the second is the insert. // // Example given the below model: // -// type SomeModel struct { -// Height int64 `pg:",pk,notnull,use_zero"` -// MinerID string `pg:",pk,notnull"` -// StateRoot string `pg:",pk,notnull"` -// OwnerID string `pg:",notnull"` -// WorkerID string `pg:",notnull"` -// } +// type SomeModel struct { +// Height int64 `pg:",pk,notnull,use_zero"` +// MinerID string `pg:",pk,notnull"` +// StateRoot string `pg:",pk,notnull"` +// OwnerID string `pg:",notnull"` +// WorkerID string `pg:",notnull"` +// } // // The strings returned are: // conflict string: +// // "(cid, height, state_root) DO UPDATE" +// // update string: -// "owner_id" = EXCLUDED.owner_id, "worker_id" = EXCLUDED.worker_id +// +// "owner_id" = EXCLUDED.owner_id, "worker_id" = EXCLUDED.worker_id func GenerateUpsertStrings(model interface{}) (string, string) { var cf []string var ucf []string From 877e3cd454c92d32dbcc6190b80534eef0b8011c Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 12:21:39 -0800 Subject: [PATCH 03/48] feat: add deferred miner precommit & sector diff --- chain/actors/builtin/miner/diff_deferred.go | 91 +++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 chain/actors/builtin/miner/diff_deferred.go diff --git a/chain/actors/builtin/miner/diff_deferred.go b/chain/actors/builtin/miner/diff_deferred.go new file mode 100644 index 000000000..8befef7db --- /dev/null +++ b/chain/actors/builtin/miner/diff_deferred.go @@ -0,0 +1,91 @@ +package miner + +import ( + "context" + + "github.com/filecoin-project/go-amt-ipld/v4" + "github.com/filecoin-project/go-hamt-ipld/v3" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +type Changes struct { + Added []*typegen.Deferred + Modified []*ChangeDiff + Removed []*typegen.Deferred +} + +type ChangeDiff struct { + Previous *typegen.Deferred + Current *typegen.Deferred +} + +func DiffPreCommitsDeferred(ctx context.Context, store adt.Store, parent, child State) (*Changes, error) { + parentMap, err := parent.PrecommitsMap() + if err != nil { + return nil, err + } + childMap, err := child.PrecommitsMap() + if err != nil { + return nil, err + } + changes, err := diff.Hamt(ctx, parentMap, childMap, store, store, hamt.UseHashFunction(parent.PrecommitsMapHashFunction()), hamt.UseTreeBitWidth(parent.PrecommitsMapBitWidth())) + if err != nil { + return nil, err + } + out := &Changes{ + Added: make([]*typegen.Deferred, 0, len(changes)), + Modified: make([]*ChangeDiff, 0, len(changes)), + Removed: make([]*typegen.Deferred, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case hamt.Add: + out.Added = append(out.Added, change.After) + case hamt.Modify: + out.Modified = append(out.Modified, &ChangeDiff{ + Previous: change.Before, + Current: change.After, + }) + case hamt.Remove: + out.Removed = append(out.Removed, change.Before) + } + } + return out, nil +} + +func DiffSectorsDeferred(ctx context.Context, store adt.Store, parent, child State) (*Changes, error) { + parentArray, err := parent.SectorsArray() + if err != nil { + return nil, err + } + childArray, err := child.SectorsArray() + if err != nil { + return nil, err + } + changes, err := diff.Amt(ctx, parentArray, childArray, store, store, amt.UseTreeBitWidth(uint(parent.SectorsAmtBitwidth()))) + if err != nil { + return nil, err + } + out := &Changes{ + Added: make([]*typegen.Deferred, 0, len(changes)), + Modified: make([]*ChangeDiff, 0, len(changes)), + Removed: make([]*typegen.Deferred, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case amt.Add: + out.Added = append(out.Added, change.After) + case amt.Modify: + out.Modified = append(out.Modified, &ChangeDiff{ + Previous: change.Before, + Current: change.After, + }) + case amt.Remove: + out.Removed = append(out.Removed, change.Before) + } + } + return out, nil +} From 68946362ae479dce61cdd53164d8865bd1abdebc Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 12:49:52 -0800 Subject: [PATCH 04/48] fixup! refactor: add InfoCid to miner interface --- chain/actors/builtin/miner/actor.go.template | 1 + chain/actors/builtin/miner/miner.go | 1 + 2 files changed, 2 insertions(+) diff --git a/chain/actors/builtin/miner/actor.go.template b/chain/actors/builtin/miner/actor.go.template index 5d7c84bb6..4ade99414 100644 --- a/chain/actors/builtin/miner/actor.go.template +++ b/chain/actors/builtin/miner/actor.go.template @@ -105,6 +105,7 @@ type State interface { DeadlinesChanged(State) (bool, error) Info() (MinerInfo, error) + InfoCid() cid.Cid MinerInfoChanged(State) (bool, error) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index c71ce8a93..c07876bf9 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -154,6 +154,7 @@ type State interface { DeadlinesChanged(State) (bool, error) Info() (MinerInfo, error) + InfoCid() cid.Cid MinerInfoChanged(State) (bool, error) DeadlineInfo(epoch abi.ChainEpoch) (*dline.Info, error) From df8b54ecf520fb96c19c86249d793990bda87351 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 17:01:27 -0800 Subject: [PATCH 05/48] MVP: complete miner extraction MVP --- chain/actors/builtin/miner/diff_deferred.go | 2 + chain/datasource/datasource.go | 4 + go.mod | 6 +- go.sum | 13 +- lens/lily/impl.go | 46 +++-- pkg/core/types.go | 25 +++ pkg/extract/actors/minerdiff/debt.go | 65 +++++++ pkg/extract/actors/minerdiff/funds.go | 90 +++++++++ pkg/extract/actors/minerdiff/infos.go | 79 ++++++++ pkg/extract/actors/minerdiff/precommits.go | 101 ++++++++++ pkg/extract/actors/minerdiff/sector_status.go | 179 ++++++++++++++++++ pkg/extract/actors/minerdiff/sectors.go | 102 ++++++++++ pkg/extract/actors/minerdiff/state.go | 47 +++++ pkg/extract/actors/types.go | 30 +++ pkg/extract/procesor/process.go | 79 ++++++++ pkg/extract/statetree/diff.go | 152 +++++++++++++++ tasks/api.go | 3 + 17 files changed, 1002 insertions(+), 21 deletions(-) create mode 100644 pkg/core/types.go create mode 100644 pkg/extract/actors/minerdiff/debt.go create mode 100644 pkg/extract/actors/minerdiff/funds.go create mode 100644 pkg/extract/actors/minerdiff/infos.go create mode 100644 pkg/extract/actors/minerdiff/precommits.go create mode 100644 pkg/extract/actors/minerdiff/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/sectors.go create mode 100644 pkg/extract/actors/minerdiff/state.go create mode 100644 pkg/extract/actors/types.go create mode 100644 pkg/extract/procesor/process.go create mode 100644 pkg/extract/statetree/diff.go diff --git a/chain/actors/builtin/miner/diff_deferred.go b/chain/actors/builtin/miner/diff_deferred.go index 8befef7db..139499709 100644 --- a/chain/actors/builtin/miner/diff_deferred.go +++ b/chain/actors/builtin/miner/diff_deferred.go @@ -31,6 +31,7 @@ func DiffPreCommitsDeferred(ctx context.Context, store adt.Store, parent, child if err != nil { return nil, err } + // TODO handle case where bitwidths or hash functions differ. changes, err := diff.Hamt(ctx, parentMap, childMap, store, store, hamt.UseHashFunction(parent.PrecommitsMapHashFunction()), hamt.UseTreeBitWidth(parent.PrecommitsMapBitWidth())) if err != nil { return nil, err @@ -65,6 +66,7 @@ func DiffSectorsDeferred(ctx context.Context, store adt.Store, parent, child Sta if err != nil { return nil, err } + // TODO handle case where bitwidths differ. changes, err := diff.Amt(ctx, parentArray, childArray, store, store, amt.UseTreeBitWidth(uint(parent.SectorsAmtBitwidth()))) if err != nil { return nil, err diff --git a/chain/datasource/datasource.go b/chain/datasource/datasource.go index cb2a27a46..45960746c 100644 --- a/chain/datasource/datasource.go +++ b/chain/datasource/datasource.go @@ -282,6 +282,10 @@ func (t *DataSource) ShouldBurnFn(ctx context.Context, ts *types.TipSet) (lens.S return t.node.BurnFundsFn(ctx, ts) } +func (t *DataSource) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) { + return t.node.ChainReadObj(ctx, obj) +} + func ComputeGasOutputs(ctx context.Context, block *types.BlockHeader, message *types.Message, receipt *types.MessageReceipt, shouldBurnFn lens.ShouldBurnFn) (vm.GasOutputs, error) { burn, err := shouldBurnFn(ctx, message, receipt.ExitCode) if err != nil { diff --git a/go.mod b/go.mod index 15e185fe3..8aa62969c 100644 --- a/go.mod +++ b/go.mod @@ -27,7 +27,7 @@ require ( github.com/go-pg/pg/v10 v10.10.6 github.com/hashicorp/golang-lru v0.5.4 github.com/ipfs/go-block-format v0.0.3 - github.com/ipfs/go-cid v0.2.0 + github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-fs-lock v0.0.7 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipld-cbor v0.0.6 @@ -210,7 +210,7 @@ require ( github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect github.com/ipld/go-car/v2 v2.4.1 // indirect github.com/ipld/go-codec-dagpb v1.3.2 // indirect - github.com/ipld/go-ipld-prime v0.17.0 // indirect + github.com/ipld/go-ipld-prime v0.19.0 // indirect github.com/ipld/go-ipld-selector-text-lite v0.0.1 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -269,7 +269,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.5.0 // indirect + github.com/multiformats/go-multicodec v0.6.0 // indirect github.com/multiformats/go-multistream v0.3.3 // indirect github.com/nikkolasg/hexjson v0.0.0-20181101101858-78e39397e00c // indirect github.com/nkovacs/streamquote v1.0.0 // indirect diff --git a/go.sum b/go.sum index 3c467edeb..5cebb0093 100644 --- a/go.sum +++ b/go.sum @@ -514,7 +514,7 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -655,8 +655,9 @@ github.com/ipfs/go-cid v0.0.6-0.20200501230655-7c82f3b81c00/go.mod h1:plgt+Y5MnO github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.2.0 h1:01JTiihFq9en9Vz0lc0VDWvZe/uBonGpzo4THP0vcQ0= github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= +github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= @@ -831,8 +832,8 @@ github.com/ipld/go-ipld-prime v0.10.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= -github.com/ipld/go-ipld-prime v0.17.0 h1:+U2peiA3aQsE7mrXjD2nYZaZrCcakoz2Wge8K42Ld8g= -github.com/ipld/go-ipld-prime v0.17.0/go.mod h1:aYcKm5TIvGfY8P3QBKz/2gKcLxzJ1zDaD+o0bOowhgs= +github.com/ipld/go-ipld-prime v0.19.0 h1:5axC7rJmPc17Emw6TelxGwnzALk0PdupZ2oj2roDj04= +github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= @@ -1343,8 +1344,8 @@ github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1 github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.5.0 h1:EgU6cBe/D7WRwQb1KmnBvU7lrcFGMggZVTPtOW9dDHs= -github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= +github.com/multiformats/go-multicodec v0.6.0 h1:KhH2kSuCARyuJraYMFxrNO3DqIaYhOdS039kbhgVwpE= +github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 00ebe1eab..56c1286f7 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -41,6 +41,7 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" + "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -133,37 +134,58 @@ func (m *LilyNodeAPI) StartTipSetWorker(_ context.Context, cfg *LilyTipSetWorker } func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { - md := storage.Metadata{ - JobName: cfg.JobConfig.Name, - } + /* + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } + + */ // the context's passed to these methods live for the duration of the clients request, so make a new one. ctx := context.Background() // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. - strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) - if err != nil { - return nil, err - } + /* + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return nil, err + } + + */ taskAPI, err := datasource.NewDataSource(m) if err != nil { return nil, err } - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) if err != nil { return nil, err } - ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) + executedTs, err := m.ChainGetTipSet(ctx, currentTs.Parents()) if err != nil { return nil, err } - success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) + _, err = procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + /* + + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } + + ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return nil, err + } + + success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) + + */ - return success, err + return true, err } func (m *LilyNodeAPI) LilyIndexNotify(_ context.Context, cfg *LilyIndexNotifyConfig) (interface{}, error) { diff --git a/pkg/core/types.go b/pkg/core/types.go new file mode 100644 index 000000000..d33ceaad3 --- /dev/null +++ b/pkg/core/types.go @@ -0,0 +1,25 @@ +package core + +// ChangeType denotes type of state change +type ChangeType int + +const ( + ChangeTypeUnknown ChangeType = iota + ChangeTypeAdd + ChangeTypeRemove + ChangeTypeModify +) + +func (c ChangeType) String() string { + switch c { + case ChangeTypeUnknown: + return "unknown" + case ChangeTypeAdd: + return "add" + case ChangeTypeRemove: + return "remove" + case ChangeTypeModify: + return "modify" + } + panic("unreachable") +} diff --git a/pkg/extract/actors/minerdiff/debt.go b/pkg/extract/actors/minerdiff/debt.go new file mode 100644 index 000000000..7855c5ed5 --- /dev/null +++ b/pkg/extract/actors/minerdiff/debt.go @@ -0,0 +1,65 @@ +package minerdiff + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*DebtChange)(nil) + +type DebtChange struct { + FeeDebt abi.TokenAmount +} + +const KindMinerDebt = "miner_debt" + +func (d *DebtChange) Kind() actors.ActorStateKind { + return KindMinerDebt +} + +var _ actors.ActorDiffer = (*Debt)(nil) + +type Debt struct{} + +func (Debt) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DebtDiff(ctx, api, act) +} + +func DebtDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + // was removed, no change + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + currentMiner, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + currentDebt, err := currentMiner.FeeDebt() + if err != nil { + return nil, err + } + // added, all debt (assumed to be zero) is new + if act.Type == core.ChangeTypeAdd { + return &DebtChange{FeeDebt: currentDebt}, nil + } + // actor state was modified, check if debt differ. + + executedMiner, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + executedDebt, err := executedMiner.FeeDebt() + if err != nil { + return nil, err + } + // no change if equal. + if executedDebt.Equals(currentDebt) { + return nil, nil + } + return &DebtChange{FeeDebt: currentDebt}, nil +} diff --git a/pkg/extract/actors/minerdiff/funds.go b/pkg/extract/actors/minerdiff/funds.go new file mode 100644 index 000000000..81dca46bb --- /dev/null +++ b/pkg/extract/actors/minerdiff/funds.go @@ -0,0 +1,90 @@ +package minerdiff + +import ( + "context" + _ "embed" + + "github.com/filecoin-project/go-state-types/abi" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*FundsChange)(nil) + +type FundsChange struct { + VestingFunds abi.TokenAmount + InitialPledgeRequirement abi.TokenAmount + PreCommitDeposit abi.TokenAmount +} + +const KindMinerFunds = "miner_funds" + +func (f *FundsChange) Kind() actors.ActorStateKind { + return KindMinerFunds +} + +var _ actors.ActorDiffer = (*Funds)(nil) + +type Funds struct{} + +func (Funds) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return FundsDiff(ctx, api, act) +} + +func FundsDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + // was removed, no change + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + currentMiner, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + currentFunds, err := currentMiner.LockedFunds() + if err != nil { + return nil, err + } + // added, all funds are new. + if act.Type == core.ChangeTypeAdd { + return &FundsChange{ + VestingFunds: currentFunds.VestingFunds, + InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, + PreCommitDeposit: currentFunds.PreCommitDeposits, + }, nil + } + + executedMiner, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + executedFunds, err := executedMiner.LockedFunds() + if err != nil { + return nil, err + } + // no change if equal + if LockedFundsEqual(currentFunds, executedFunds) { + return nil, nil + } + // funds differ + return &FundsChange{ + VestingFunds: currentFunds.VestingFunds, + InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, + PreCommitDeposit: currentFunds.PreCommitDeposits, + }, nil +} + +func LockedFundsEqual(cur, pre miner.LockedFunds) bool { + if !cur.VestingFunds.Equals(pre.VestingFunds) { + return true + } + if !cur.PreCommitDeposits.Equals(pre.PreCommitDeposits) { + return true + } + if !cur.InitialPledgeRequirement.Equals(pre.InitialPledgeRequirement) { + return true + } + return false +} diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/infos.go new file mode 100644 index 000000000..3fa4ef804 --- /dev/null +++ b/pkg/extract/actors/minerdiff/infos.go @@ -0,0 +1,79 @@ +package minerdiff + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*InfoChange)(nil) + +type InfoChange struct { + Info typegen.Deferred +} + +const KindMinerInfo = "miner_info" + +func (i *InfoChange) Kind() actors.ActorStateKind { + return KindMinerInfo +} + +var _ actors.ActorDiffer = (*Info)(nil) + +type Info struct{} + +func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return InfoDiff(ctx, api, act) +} + +type DiffInfoAPI interface { + Store() adt.Store + ChainReadObj(ctx context.Context, c cid.Cid) ([]byte, error) + MinerLoad(store adt.Store, act *types.Actor) (miner.State, error) + Actor(ctx context.Context, addr address.Address, tsk types.TipSetKey) (*types.Actor, error) +} + +// separate method for testing purposes + +func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*InfoChange, error) { + // was removed, no new info + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentMiner, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + infoBytes, err := api.ChainReadObj(ctx, currentMiner.InfoCid()) + if err != nil { + return nil, err + } + // was added, info is new + if act.Type == core.ChangeTypeAdd { + return &InfoChange{ + Info: typegen.Deferred{Raw: infoBytes}, + }, nil + } + + executedMiner, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + // wasn't modified + if executedMiner.InfoCid().Equals(currentMiner.InfoCid()) { + return nil, nil + } + return &InfoChange{ + Info: typegen.Deferred{Raw: infoBytes}, + }, nil +} diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/precommits.go new file mode 100644 index 000000000..337fc217b --- /dev/null +++ b/pkg/extract/actors/minerdiff/precommits.go @@ -0,0 +1,101 @@ +package minerdiff + +import ( + "context" + + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) + +type PreCommitChange struct { + PreCommit typegen.Deferred + Type core.ChangeType +} + +type PreCommitChangeList []*PreCommitChange + +const KindMinerPreCommit = "miner_precommit" + +func (p PreCommitChangeList) Kind() actors.ActorStateKind { + return KindMinerPreCommit +} + +type PreCommit struct{} + +func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DiffPreCommits(ctx, api, act) +} + +func DiffPreCommits(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + // the actor was removed, nothing has changes in the current state. + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentMiner, err := miner.Load(api.Store(), act.Current) + if err != nil { + return nil, err + } + // the actor was added, everything is new in the current state. + if act.Type == core.ChangeTypeAdd { + var out PreCommitChangeList + pm, err := currentMiner.PrecommitsMap() + if err != nil { + return nil, err + } + var v typegen.Deferred + if err := pm.ForEach(&v, func(key string) error { + out = append(out, &PreCommitChange{ + PreCommit: v, + Type: core.ChangeTypeAdd, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + // actor was modified load executed state. + executedMiner, err := miner.Load(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + preCommitChanges, err := miner.DiffPreCommitsDeferred(ctx, api.Store(), executedMiner, currentMiner) + if err != nil { + return nil, err + } + + idx := 0 + out := make(PreCommitChangeList, len(preCommitChanges.Added)+len(preCommitChanges.Removed)+len(preCommitChanges.Modified)) + for _, change := range preCommitChanges.Added { + out[idx] = &PreCommitChange{ + PreCommit: *change, + Type: core.ChangeTypeAdd, + } + idx++ + } + for _, change := range preCommitChanges.Removed { + out[idx] = &PreCommitChange{ + PreCommit: *change, + Type: core.ChangeTypeRemove, + } + idx++ + } + // NB: PreCommits cannot be modified, but check anyway. + for _, change := range preCommitChanges.Modified { + out[idx] = &PreCommitChange{ + PreCommit: *change.Current, + Type: core.ChangeTypeModify, + } + idx++ + } + return out, nil +} diff --git a/pkg/extract/actors/minerdiff/sector_status.go b/pkg/extract/actors/minerdiff/sector_status.go new file mode 100644 index 000000000..b51b29a99 --- /dev/null +++ b/pkg/extract/actors/minerdiff/sector_status.go @@ -0,0 +1,179 @@ +package minerdiff + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-bitfield" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*SectorStatusChange)(nil) + +type SectorStatusChange struct { + // Removed sectors this epoch + Removed bitfield.BitField + // Recovering sectors this epoch + Recovering bitfield.BitField + // Faulted sectors this epoch + Faulted bitfield.BitField + // Recovered sectors this epoch + Recovered bitfield.BitField +} + +const KindMinerSectorStatus = "miner_sector_status" + +func (s SectorStatusChange) Kind() actors.ActorStateKind { + return KindMinerSectorStatus +} + +type SectorStatus struct{} + +func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + child, err := api.MinerLoad(api.Store(), act.Current) + if err != nil { + return nil, err + } + parent, err := api.MinerLoad(api.Store(), act.Executed) + if err != nil { + return nil, err + } + return DiffMinerSectorStates(ctx, child, parent) +} + +// DiffMinerSectorStates loads the SectorStates for the current and parent miner states in parallel from `extState`. +// Then compares current and parent SectorStates to produce a SectorStateEvents structure containing all sectors that are +// removed, recovering, faulted, and recovered for the state transition from parent miner state to current miner state. +func DiffMinerSectorStates(ctx context.Context, child, parent miner.State) (*SectorStatusChange, error) { + var ( + previous, current *SectorStates + err error + ) + + // load previous and current miner sector states in parallel + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + previous, err = LoadSectorState(grpCtx, parent) + if err != nil { + return fmt.Errorf("loading previous sector states %w", err) + } + return nil + }) + grp.Go(func() error { + current, err = LoadSectorState(grpCtx, child) + if err != nil { + return fmt.Errorf("loading current sector states %w", err) + } + return nil + }) + // if either load operation fails abort + if err := grp.Wait(); err != nil { + return nil, err + } + + // previous live sector minus current live sectors are sectors removed this epoch. + removed, err := bitfield.SubtractBitField(previous.Live, current.Live) + if err != nil { + return nil, fmt.Errorf("comparing previous live sectors to current live sectors %w", err) + } + + // current recovering sectors minus previous recovering sectors are sectors recovering this epoch. + recovering, err := bitfield.SubtractBitField(current.Recovering, previous.Recovering) + if err != nil { + return nil, fmt.Errorf("comparing current recovering sectors to previous recovering sectors %w", err) + } + + // current faulty sectors minus previous faulty sectors are sectors faulted this epoch. + faulted, err := bitfield.SubtractBitField(current.Faulty, previous.Faulty) + if err != nil { + return nil, fmt.Errorf("comparing current faulty sectors to previous faulty sectors %w", err) + } + + // previous faulty sectors matching (intersect) active sectors are sectors recovered this epoch. + recovered, err := bitfield.IntersectBitField(previous.Faulty, current.Active) + if err != nil { + return nil, fmt.Errorf("comparing previous faulty sectors to current active sectors %w", err) + } + + return &SectorStatusChange{ + Removed: removed, + Recovering: recovering, + Faulted: faulted, + Recovered: recovered, + }, nil + +} + +// SectorStates contains a set of bitfields for active, live, fault, and recovering sectors. +type SectorStates struct { + // Active sectors are those that are neither terminated nor faulty nor unproven, i.e. actively contributing power. + Active bitfield.BitField + // Live sectors are those that are not terminated (but may be faulty). + Live bitfield.BitField + // Faulty contains a subset of sectors detected/declared faulty and not yet recovered (excl. from PoSt). + Faulty bitfield.BitField + // Recovering contains a subset of faulty sectors expected to recover on next PoSt. + Recovering bitfield.BitField +} + +// LoadSectorState loads all sectors from a miners partitions and returns a SectorStates structure containing individual +// bitfields for all active, live, faulty and recovering sector. +func LoadSectorState(ctx context.Context, state miner.State) (*SectorStates, error) { + activeSectors := []bitfield.BitField{} + liveSectors := []bitfield.BitField{} + faultySectors := []bitfield.BitField{} + recoveringSectors := []bitfield.BitField{} + + // iterate the sector states + if err := state.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { + return dl.ForEachPartition(func(_ uint64, part miner.Partition) error { + active, err := part.ActiveSectors() + if err != nil { + return err + } + activeSectors = append(activeSectors, active) + + live, err := part.LiveSectors() + if err != nil { + return err + } + liveSectors = append(liveSectors, live) + + faulty, err := part.FaultySectors() + if err != nil { + return err + } + faultySectors = append(faultySectors, faulty) + + recovering, err := part.RecoveringSectors() + if err != nil { + return err + } + recoveringSectors = append(recoveringSectors, recovering) + + return nil + }) + }); err != nil { + return nil, err + } + var err error + sectorStates := &SectorStates{} + if sectorStates.Active, err = bitfield.MultiMerge(activeSectors...); err != nil { + return nil, err + } + if sectorStates.Live, err = bitfield.MultiMerge(liveSectors...); err != nil { + return nil, err + } + if sectorStates.Faulty, err = bitfield.MultiMerge(faultySectors...); err != nil { + return nil, err + } + if sectorStates.Recovering, err = bitfield.MultiMerge(recoveringSectors...); err != nil { + return nil, err + } + + return sectorStates, nil +} diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/sectors.go new file mode 100644 index 000000000..15164bd96 --- /dev/null +++ b/pkg/extract/actors/minerdiff/sectors.go @@ -0,0 +1,102 @@ +package minerdiff + +import ( + "context" + + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var _ actors.ActorStateChange = (*SectorChangeList)(nil) + +type SectorChange struct { + Sector typegen.Deferred + Type core.ChangeType +} + +type SectorChangeList []*SectorChange + +const KindMinerSector = "miner_sector" + +func (s SectorChangeList) Kind() actors.ActorStateKind { + return KindMinerSector +} + +type Sectors struct{} + +func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DiffSectors(ctx, api, act) +} + +func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + // the actor was removed, nothing has changes in the current state. + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentMiner, err := miner.Load(api.Store(), act.Current) + if err != nil { + return nil, err + } + + // the actor was added, everything is new in the current state. + if act.Type == core.ChangeTypeAdd { + sa, err := currentMiner.SectorsArray() + if err != nil { + return nil, err + } + out := make(SectorChangeList, int(sa.Length())) + var v typegen.Deferred + if err := sa.ForEach(&v, func(idx int64) error { + out[idx] = &SectorChange{ + Sector: v, + Type: core.ChangeTypeAdd, + } + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + // the actor was modified, diff against executed state. + executedMiner, err := miner.Load(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + sectorChanges, err := miner.DiffSectorsDeferred(ctx, api.Store(), executedMiner, currentMiner) + if err != nil { + return nil, err + } + + idx := 0 + out := make(SectorChangeList, len(sectorChanges.Added)+len(sectorChanges.Removed)+len(sectorChanges.Modified)) + for _, change := range sectorChanges.Added { + out[idx] = &SectorChange{ + Sector: *change, + Type: core.ChangeTypeAdd, + } + idx++ + } + for _, change := range sectorChanges.Removed { + out[idx] = &SectorChange{ + Sector: *change, + Type: core.ChangeTypeRemove, + } + idx++ + } + for _, change := range sectorChanges.Modified { + out[idx] = &SectorChange{ + Sector: *change.Current, + Type: core.ChangeTypeModify, + } + idx++ + } + return out, nil + +} diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go new file mode 100644 index 000000000..ee6546ec8 --- /dev/null +++ b/pkg/extract/actors/minerdiff/state.go @@ -0,0 +1,47 @@ +package minerdiff + +import ( + "context" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiff struct { + InfoChange *InfoChange + FundsChange *FundsChange + DebtChange *DebtChange + SectorStatusChanges *SectorStatusChange + PreCommitChanges PreCommitChangeList + SectorChanges SectorChangeList +} + +type ActorStateKind string + +func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { + var stateDiff = new(StateDiff) + for _, f := range diffFns { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindMinerInfo: + stateDiff.InfoChange = stateChange.(*InfoChange) + case KindMinerSector: + stateDiff.SectorChanges = stateChange.(SectorChangeList) + case KindMinerPreCommit: + stateDiff.PreCommitChanges = stateChange.(PreCommitChangeList) + case KindMinerFunds: + stateDiff.FundsChange = stateChange.(*FundsChange) + case KindMinerDebt: + stateDiff.DebtChange = stateChange.(*DebtChange) + case KindMinerSectorStatus: + stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) + } + } + return stateDiff, nil +} diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go new file mode 100644 index 000000000..68ae2f3c4 --- /dev/null +++ b/pkg/extract/actors/types.go @@ -0,0 +1,30 @@ +package actors + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/tasks" +) + +type ActorDiffer interface { + Diff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorStateChange, error) +} + +type ActorStateKind string + +type ActorStateChange interface { + Kind() ActorStateKind +} + +type ActorChange struct { + Address address.Address + Executed *types.Actor + Current *types.Actor + Type core.ChangeType +} + +type ActorChanges []*ActorChange diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go new file mode 100644 index 000000000..c7bb0fb4a --- /dev/null +++ b/pkg/extract/procesor/process.go @@ -0,0 +1,79 @@ +package procesor + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/actors/builtin/market" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/chain/actors/builtin/power" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/extract/statetree" + "github.com/filecoin-project/lily/tasks" +) + +var ( + MinerCodes = cid.NewSet() + PowerCodes = cid.NewSet() + MarketCodes = cid.NewSet() +) + +func init() { + for _, c := range miner.AllCodes() { + MinerCodes.Add(c) + } + for _, c := range power.AllCodes() { + PowerCodes.Add(c) + } + for _, c := range market.AllCodes() { + MarketCodes.Add(c) + } +} + +type ActorStateChanges struct { + Current *types.TipSet + Executed *types.TipSet + Actors map[address.Address]statetree.ActorDiff + MinerActors map[address.Address]*minerdiff.StateDiff +} + +func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ActorStateChanges, error) { + actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) + if err != nil { + return nil, err + } + asc := &ActorStateChanges{ + Current: current, + Executed: executed, + Actors: actorChanges, + MinerActors: make(map[address.Address]*minerdiff.StateDiff, len(actorChanges)), // there are at most actorChanges entries + } + + for addr, change := range actorChanges { + if MinerCodes.Has(change.Current.Code) { + minerChanges, err := minerdiff.State(ctx, api, &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, + }, + minerdiff.Debt{}, + minerdiff.Funds{}, + minerdiff.Info{}, + minerdiff.PreCommit{}, + minerdiff.Sectors{}, + minerdiff.SectorStatus{}, + ) + if err != nil { + return nil, err + } + asc.MinerActors[addr] = minerChanges + } + } + return asc, nil + +} diff --git a/pkg/extract/statetree/diff.go b/pkg/extract/statetree/diff.go new file mode 100644 index 000000000..79c297e31 --- /dev/null +++ b/pkg/extract/statetree/diff.go @@ -0,0 +1,152 @@ +package statetree + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-hamt-ipld/v3" + "github.com/filecoin-project/lotus/chain/types" + states0 "github.com/filecoin-project/specs-actors/actors/states" + states2 "github.com/filecoin-project/specs-actors/v2/actors/states" + states3 "github.com/filecoin-project/specs-actors/v3/actors/states" + states4 "github.com/filecoin-project/specs-actors/v4/actors/states" + states5 "github.com/filecoin-project/specs-actors/v5/actors/states" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" + "github.com/filecoin-project/lily/pkg/core" +) + +type ActorDiff struct { + Executed *types.Actor + Current *types.Actor + ChangeType core.ChangeType +} + +func ActorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet) (map[address.Address]ActorDiff, error) { + // we have this special method here to get the HAMT node root required by the faster diffing logic. I hate this. + oldRoot, oldVersion, err := getStateTreeHamtRootCIDAndVersion(ctx, store, executed.ParentState()) + if err != nil { + return nil, err + } + newRoot, newVersion, err := getStateTreeHamtRootCIDAndVersion(ctx, store, current.ParentState()) + if err != nil { + return nil, err + } + + if oldVersion != newVersion { + panic("here") + } + + changes, err := diff.Hamt(ctx, oldRoot, newRoot, store, store, hamt.UseTreeBitWidth(5), hamt.UseHashFunction(func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + })) + if err != nil { + return nil, err + } + buf := bytes.NewReader(nil) + out := map[address.Address]ActorDiff{} + for _, change := range changes { + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, fmt.Errorf("address in state tree was not valid: %w", err) + } + ch := ActorDiff{ + Executed: new(types.Actor), + Current: new(types.Actor), + ChangeType: core.ChangeTypeUnknown, + } + switch change.Type { + case hamt.Add: + ch.ChangeType = core.ChangeTypeAdd + buf.Reset(change.After.Raw) + err = ch.Current.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + case hamt.Remove: + ch.ChangeType = core.ChangeTypeRemove + buf.Reset(change.Before.Raw) + err = ch.Executed.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + case hamt.Modify: + ch.ChangeType = core.ChangeTypeModify + buf.Reset(change.Before.Raw) + err = ch.Executed.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + + buf.Reset(change.After.Raw) + err = ch.Current.UnmarshalCBOR(buf) + buf.Reset(nil) + if err != nil { + return nil, err + } + } + out[addr] = ch + } + return out, nil +} + +func getStateTreeHamtRootCIDAndVersion(ctx context.Context, store adt.Store, c cid.Cid) (adt.Map, types.StateTreeVersion, error) { + var root types.StateRoot + // Try loading as a new-style state-tree (version/actors tuple). + if err := store.Get(ctx, c, &root); err != nil { + // We failed to decode as the new version, must be an old version. + root.Actors = c + root.Version = types.StateTreeVersion0 + } + + switch root.Version { + case types.StateTreeVersion0: + var tree *states0.Tree + tree, err := states0.LoadTree(store, root.Actors) + if err != nil { + return nil, 0, err + } + return tree.Map, root.Version, nil + case types.StateTreeVersion1: + var tree *states2.Tree + tree, err := states2.LoadTree(store, root.Actors) + if err != nil { + return nil, 0, err + } + return tree.Map, root.Version, nil + case types.StateTreeVersion2: + var tree *states3.Tree + tree, err := states3.LoadTree(store, root.Actors) + if err != nil { + return nil, 0, err + } + return tree.Map, root.Version, nil + case types.StateTreeVersion3: + var tree *states4.Tree + tree, err := states4.LoadTree(store, root.Actors) + if err != nil { + return nil, 0, err + } + return tree.Map, root.Version, nil + case types.StateTreeVersion4: + var tree *states5.Tree + tree, err := states5.LoadTree(store, root.Actors) + if err != nil { + return nil, 0, err + } + return tree.Map, root.Version, nil + default: + return nil, 0, fmt.Errorf("unsupported state tree version: %d", root.Version) + } +} diff --git a/tasks/api.go b/tasks/api.go index a0f78c956..9700a7a81 100644 --- a/tasks/api.go +++ b/tasks/api.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/api" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -66,4 +67,6 @@ type DataSource interface { MinerLoad(store adt.Store, act *types.Actor) (miner.State, error) ShouldBurnFn(ctx context.Context, ts *types.TipSet) (lens.ShouldBurnFn, error) + + ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) } From a5464c49e6e9c5bb6df710688d982d394cf8649b Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 6 Dec 2022 19:22:54 -0800 Subject: [PATCH 06/48] feat: persist miner state as IPLD --- lens/lily/impl.go | 10 +- pkg/core/types.go | 2 +- pkg/extract/actors/minerdiff/cbor_gen.go | 704 ++++++++++++++++++ pkg/extract/actors/minerdiff/debt.go | 7 +- pkg/extract/actors/minerdiff/funds.go | 9 +- pkg/extract/actors/minerdiff/infos.go | 11 +- pkg/extract/actors/minerdiff/precommits.go | 12 +- pkg/extract/actors/minerdiff/sector_status.go | 8 +- pkg/extract/actors/minerdiff/sectors.go | 12 +- pkg/extract/actors/minerdiff/state.go | 2 +- pkg/gen/main.go | 32 + pkg/transform/cbor/cbor_gen.go | 352 +++++++++ pkg/transform/cbor/router.go | 153 ++++ pkg/util/cid.go | 34 + 14 files changed, 1320 insertions(+), 28 deletions(-) create mode 100644 pkg/extract/actors/minerdiff/cbor_gen.go create mode 100644 pkg/gen/main.go create mode 100644 pkg/transform/cbor/cbor_gen.go create mode 100644 pkg/transform/cbor/router.go create mode 100644 pkg/util/cid.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 56c1286f7..7d44dc51f 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -10,6 +10,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" network2 "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/state" @@ -20,6 +21,7 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/net" "github.com/filecoin-project/specs-actors/actors/util/adt" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/go-pg/pg/v10" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" @@ -42,6 +44,7 @@ import ( "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -167,7 +170,12 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf return nil, err } - _, err = procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + bs := blockstore.NewMemorySync() + store := adt2.WrapBlockStore(ctx, bs) + if _, err := cbor.ProcessActors(ctx, store, changes); err != nil { + return false, err + } /* // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. diff --git a/pkg/core/types.go b/pkg/core/types.go index d33ceaad3..ca16b4bb3 100644 --- a/pkg/core/types.go +++ b/pkg/core/types.go @@ -1,7 +1,7 @@ package core // ChangeType denotes type of state change -type ChangeType int +type ChangeType uint8 const ( ChangeTypeUnknown ChangeType = iota diff --git a/pkg/extract/actors/minerdiff/cbor_gen.go b/pkg/extract/actors/minerdiff/cbor_gen.go new file mode 100644 index 000000000..acf8bcb3d --- /dev/null +++ b/pkg/extract/actors/minerdiff/cbor_gen.go @@ -0,0 +1,704 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package minerdiff + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *SectorStatusChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Removed (bitfield.BitField) (struct) + if len("removed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"removed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("removed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("removed")); err != nil { + return err + } + + if err := t.Removed.MarshalCBOR(cw); err != nil { + return err + } + + // t.Recovering (bitfield.BitField) (struct) + if len("recovering") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"recovering\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("recovering"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("recovering")); err != nil { + return err + } + + if err := t.Recovering.MarshalCBOR(cw); err != nil { + return err + } + + // t.Faulted (bitfield.BitField) (struct) + if len("faulted") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"faulted\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("faulted"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("faulted")); err != nil { + return err + } + + if err := t.Faulted.MarshalCBOR(cw); err != nil { + return err + } + + // t.Recovered (bitfield.BitField) (struct) + if len("recovered") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"recovered\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("recovered"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("recovered")); err != nil { + return err + } + + if err := t.Recovered.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *SectorStatusChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorStatusChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorStatusChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Removed (bitfield.BitField) (struct) + case "removed": + + { + + if err := t.Removed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Removed: %w", err) + } + + } + // t.Recovering (bitfield.BitField) (struct) + case "recovering": + + { + + if err := t.Recovering.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Recovering: %w", err) + } + + } + // t.Faulted (bitfield.BitField) (struct) + case "faulted": + + { + + if err := t.Faulted.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Faulted: %w", err) + } + + } + // t.Recovered (bitfield.BitField) (struct) + case "recovered": + + { + + if err := t.Recovered.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Recovered: %w", err) + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *PreCommitChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.PreCommit (typegen.Deferred) (struct) + if len("pre_commit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commit")); err != nil { + return err + } + + if err := t.PreCommit.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *PreCommitChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = PreCommitChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("PreCommitChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.PreCommit (typegen.Deferred) (struct) + case "pre_commit": + + { + + if err := t.PreCommit.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SectorChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Sector (typegen.Deferred) (struct) + if len("sector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector")); err != nil { + return err + } + + if err := t.Sector.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *SectorChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = SectorChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SectorChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Sector (typegen.Deferred) (struct) + case "sector": + + { + + if err := t.Sector.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *FundsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.VestingFunds (big.Int) (struct) + if len("vesting_funds") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vesting_funds\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vesting_funds"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vesting_funds")); err != nil { + return err + } + + if err := t.VestingFunds.MarshalCBOR(cw); err != nil { + return err + } + + // t.InitialPledgeRequirement (big.Int) (struct) + if len("initial_pledge_requirement") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"initial_pledge_requirement\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("initial_pledge_requirement"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("initial_pledge_requirement")); err != nil { + return err + } + + if err := t.InitialPledgeRequirement.MarshalCBOR(cw); err != nil { + return err + } + + // t.PreCommitDeposit (big.Int) (struct) + if len("pre_commit_deposit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commit_deposit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commit_deposit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commit_deposit")); err != nil { + return err + } + + if err := t.PreCommitDeposit.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *FundsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = FundsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FundsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.VestingFunds (big.Int) (struct) + case "vesting_funds": + + { + + if err := t.VestingFunds.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.VestingFunds: %w", err) + } + + } + // t.InitialPledgeRequirement (big.Int) (struct) + case "initial_pledge_requirement": + + { + + if err := t.InitialPledgeRequirement.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.InitialPledgeRequirement: %w", err) + } + + } + // t.PreCommitDeposit (big.Int) (struct) + case "pre_commit_deposit": + + { + + if err := t.PreCommitDeposit.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.PreCommitDeposit: %w", err) + } + + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *DebtChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.FeeDebt (big.Int) (struct) + if len("fee_debt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fee_debt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fee_debt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fee_debt")); err != nil { + return err + } + + if err := t.FeeDebt.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *DebtChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = DebtChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DebtChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.FeeDebt (big.Int) (struct) + case "fee_debt": + + { + + if err := t.FeeDebt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FeeDebt: %w", err) + } + + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/minerdiff/debt.go b/pkg/extract/actors/minerdiff/debt.go index 7855c5ed5..12eb4ae2e 100644 --- a/pkg/extract/actors/minerdiff/debt.go +++ b/pkg/extract/actors/minerdiff/debt.go @@ -13,7 +13,8 @@ import ( var _ actors.ActorStateChange = (*DebtChange)(nil) type DebtChange struct { - FeeDebt abi.TokenAmount + FeeDebt abi.TokenAmount `cborgen:"fee_debt"` + Change core.ChangeType `cborgen:"change"` } const KindMinerDebt = "miner_debt" @@ -45,7 +46,7 @@ func DebtDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange } // added, all debt (assumed to be zero) is new if act.Type == core.ChangeTypeAdd { - return &DebtChange{FeeDebt: currentDebt}, nil + return &DebtChange{FeeDebt: currentDebt, Change: core.ChangeTypeAdd}, nil } // actor state was modified, check if debt differ. @@ -61,5 +62,5 @@ func DebtDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange if executedDebt.Equals(currentDebt) { return nil, nil } - return &DebtChange{FeeDebt: currentDebt}, nil + return &DebtChange{FeeDebt: currentDebt, Change: core.ChangeTypeModify}, nil } diff --git a/pkg/extract/actors/minerdiff/funds.go b/pkg/extract/actors/minerdiff/funds.go index 81dca46bb..e8385e21d 100644 --- a/pkg/extract/actors/minerdiff/funds.go +++ b/pkg/extract/actors/minerdiff/funds.go @@ -15,9 +15,10 @@ import ( var _ actors.ActorStateChange = (*FundsChange)(nil) type FundsChange struct { - VestingFunds abi.TokenAmount - InitialPledgeRequirement abi.TokenAmount - PreCommitDeposit abi.TokenAmount + VestingFunds abi.TokenAmount `cborgen:"vesting_funds"` + InitialPledgeRequirement abi.TokenAmount `cborgen:"initial_pledge_requirement"` + PreCommitDeposit abi.TokenAmount `cborgen:"pre_commit_deposit"` + Change core.ChangeType `cborgen:"change"` } const KindMinerFunds = "miner_funds" @@ -53,6 +54,7 @@ func FundsDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChang VestingFunds: currentFunds.VestingFunds, InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, PreCommitDeposit: currentFunds.PreCommitDeposits, + Change: core.ChangeTypeAdd, }, nil } @@ -73,6 +75,7 @@ func FundsDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChang VestingFunds: currentFunds.VestingFunds, InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, PreCommitDeposit: currentFunds.PreCommitDeposits, + Change: core.ChangeTypeModify, }, nil } diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/infos.go index 3fa4ef804..92ec56aa4 100644 --- a/pkg/extract/actors/minerdiff/infos.go +++ b/pkg/extract/actors/minerdiff/infos.go @@ -4,6 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" @@ -18,7 +19,9 @@ import ( var _ actors.ActorStateChange = (*InfoChange)(nil) type InfoChange struct { - Info typegen.Deferred + cbor.Er + Info typegen.Deferred `cborgen:"info"` + Change core.ChangeType `cborgen:"change"` } const KindMinerInfo = "miner_info" @@ -61,7 +64,8 @@ func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*I // was added, info is new if act.Type == core.ChangeTypeAdd { return &InfoChange{ - Info: typegen.Deferred{Raw: infoBytes}, + Info: typegen.Deferred{Raw: infoBytes}, + Change: core.ChangeTypeAdd, }, nil } @@ -74,6 +78,7 @@ func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*I return nil, nil } return &InfoChange{ - Info: typegen.Deferred{Raw: infoBytes}, + Info: typegen.Deferred{Raw: infoBytes}, + Change: core.ChangeTypeModify, }, nil } diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/precommits.go index 337fc217b..7b64d7cd0 100644 --- a/pkg/extract/actors/minerdiff/precommits.go +++ b/pkg/extract/actors/minerdiff/precommits.go @@ -14,8 +14,8 @@ import ( var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) type PreCommitChange struct { - PreCommit typegen.Deferred - Type core.ChangeType + PreCommit typegen.Deferred `cborgen:"pre_commit"` + Change core.ChangeType `cborgen:"change"` } type PreCommitChangeList []*PreCommitChange @@ -53,7 +53,7 @@ func DiffPreCommits(ctx context.Context, api tasks.DataSource, act *actors.Actor if err := pm.ForEach(&v, func(key string) error { out = append(out, &PreCommitChange{ PreCommit: v, - Type: core.ChangeTypeAdd, + Change: core.ChangeTypeAdd, }) return nil }); err != nil { @@ -78,14 +78,14 @@ func DiffPreCommits(ctx context.Context, api tasks.DataSource, act *actors.Actor for _, change := range preCommitChanges.Added { out[idx] = &PreCommitChange{ PreCommit: *change, - Type: core.ChangeTypeAdd, + Change: core.ChangeTypeAdd, } idx++ } for _, change := range preCommitChanges.Removed { out[idx] = &PreCommitChange{ PreCommit: *change, - Type: core.ChangeTypeRemove, + Change: core.ChangeTypeRemove, } idx++ } @@ -93,7 +93,7 @@ func DiffPreCommits(ctx context.Context, api tasks.DataSource, act *actors.Actor for _, change := range preCommitChanges.Modified { out[idx] = &PreCommitChange{ PreCommit: *change.Current, - Type: core.ChangeTypeModify, + Change: core.ChangeTypeModify, } idx++ } diff --git a/pkg/extract/actors/minerdiff/sector_status.go b/pkg/extract/actors/minerdiff/sector_status.go index b51b29a99..228a03d6b 100644 --- a/pkg/extract/actors/minerdiff/sector_status.go +++ b/pkg/extract/actors/minerdiff/sector_status.go @@ -16,13 +16,13 @@ var _ actors.ActorStateChange = (*SectorStatusChange)(nil) type SectorStatusChange struct { // Removed sectors this epoch - Removed bitfield.BitField + Removed bitfield.BitField `cborgen:"removed"` // Recovering sectors this epoch - Recovering bitfield.BitField + Recovering bitfield.BitField `cborgen:"recovering"` // Faulted sectors this epoch - Faulted bitfield.BitField + Faulted bitfield.BitField `cborgen:"faulted"` // Recovered sectors this epoch - Recovered bitfield.BitField + Recovered bitfield.BitField `cborgen:"recovered"` } const KindMinerSectorStatus = "miner_sector_status" diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/sectors.go index 15164bd96..52a166153 100644 --- a/pkg/extract/actors/minerdiff/sectors.go +++ b/pkg/extract/actors/minerdiff/sectors.go @@ -14,8 +14,8 @@ import ( var _ actors.ActorStateChange = (*SectorChangeList)(nil) type SectorChange struct { - Sector typegen.Deferred - Type core.ChangeType + Sector typegen.Deferred `cborgen:"sector"` + Change core.ChangeType `cborgen:"change"` } type SectorChangeList []*SectorChange @@ -54,7 +54,7 @@ func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorCha if err := sa.ForEach(&v, func(idx int64) error { out[idx] = &SectorChange{ Sector: v, - Type: core.ChangeTypeAdd, + Change: core.ChangeTypeAdd, } return nil }); err != nil { @@ -79,21 +79,21 @@ func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorCha for _, change := range sectorChanges.Added { out[idx] = &SectorChange{ Sector: *change, - Type: core.ChangeTypeAdd, + Change: core.ChangeTypeAdd, } idx++ } for _, change := range sectorChanges.Removed { out[idx] = &SectorChange{ Sector: *change, - Type: core.ChangeTypeRemove, + Change: core.ChangeTypeRemove, } idx++ } for _, change := range sectorChanges.Modified { out[idx] = &SectorChange{ Sector: *change.Current, - Type: core.ChangeTypeModify, + Change: core.ChangeTypeModify, } idx++ } diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go index ee6546ec8..289aa76d7 100644 --- a/pkg/extract/actors/minerdiff/state.go +++ b/pkg/extract/actors/minerdiff/state.go @@ -11,9 +11,9 @@ type StateDiff struct { InfoChange *InfoChange FundsChange *FundsChange DebtChange *DebtChange - SectorStatusChanges *SectorStatusChange PreCommitChanges PreCommitChangeList SectorChanges SectorChangeList + SectorStatusChanges *SectorStatusChange } type ActorStateKind string diff --git a/pkg/gen/main.go b/pkg/gen/main.go new file mode 100644 index 000000000..b54a6627e --- /dev/null +++ b/pkg/gen/main.go @@ -0,0 +1,32 @@ +package main + +import ( + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/transform/cbor" +) + +const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" +const minerDiffPkg = "minerdiff" + +const actorTransformPath = "pkg/transform/cbor/cbor_gen.go" +const actorTransformPkg = "cbor" + +func main() { + if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, + minerdiff.SectorStatusChange{}, + minerdiff.PreCommitChange{}, + minerdiff.SectorChange{}, + minerdiff.FundsChange{}, + minerdiff.DebtChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, + cbor.MinerStateChange{}, + ); err != nil { + panic(err) + } +} diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go new file mode 100644 index 000000000..127bdf069 --- /dev/null +++ b/pkg/transform/cbor/cbor_gen.go @@ -0,0 +1,352 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package cbor + +import ( + "fmt" + "io" + "math" + "sort" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *MinerStateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{167}); err != nil { + return err + } + + // t.Miner (address.Address) (struct) + if len("miner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("miner")); err != nil { + return err + } + + if err := t.Miner.MarshalCBOR(cw); err != nil { + return err + } + + // t.Funds (minerdiff.FundsChange) (struct) + if len("funds") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"funds\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("funds"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("funds")); err != nil { + return err + } + + if err := t.Funds.MarshalCBOR(cw); err != nil { + return err + } + + // t.Debt (minerdiff.DebtChange) (struct) + if len("debt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"debt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("debt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("debt")); err != nil { + return err + } + + if err := t.Debt.MarshalCBOR(cw); err != nil { + return err + } + + // t.SectorStatus (minerdiff.SectorStatusChange) (struct) + if len("sector_status") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_status\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_status")); err != nil { + return err + } + + if err := t.SectorStatus.MarshalCBOR(cw); err != nil { + return err + } + + // t.Info (cid.Cid) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if t.Info == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + } + + // t.PreCommits (cid.Cid) (struct) + if len("pre_commits") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commits\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commits")); err != nil { + return err + } + + if t.PreCommits == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PreCommits); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) + } + } + + // t.Sectors (cid.Cid) (struct) + if len("sectors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sectors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sectors")); err != nil { + return err + } + + if t.Sectors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Sectors); err != nil { + return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) + } + } + + return nil +} + +func (t *MinerStateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = MinerStateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("MinerStateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Miner (address.Address) (struct) + case "miner": + + { + + if err := t.Miner.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Miner: %w", err) + } + + } + // t.Funds (minerdiff.FundsChange) (struct) + case "funds": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Funds = new(minerdiff.FundsChange) + if err := t.Funds.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Funds pointer: %w", err) + } + } + + } + // t.Debt (minerdiff.DebtChange) (struct) + case "debt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Debt = new(minerdiff.DebtChange) + if err := t.Debt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Debt pointer: %w", err) + } + } + + } + // t.SectorStatus (minerdiff.SectorStatusChange) (struct) + case "sector_status": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.SectorStatus = new(minerdiff.SectorStatusChange) + if err := t.SectorStatus.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.SectorStatus pointer: %w", err) + } + } + + } + // t.Info (cid.Cid) (struct) + case "info": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = &c + } + + } + // t.PreCommits (cid.Cid) (struct) + case "pre_commits": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) + } + + t.PreCommits = &c + } + + } + // t.Sectors (cid.Cid) (struct) + case "sectors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) + } + + t.Sectors = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go new file mode 100644 index 000000000..7041fd99f --- /dev/null +++ b/pkg/transform/cbor/router.go @@ -0,0 +1,153 @@ +package cbor + +import ( + "context" + "fmt" + "sort" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/util" +) + +type ActorIPLDContainer struct { + // HAMT of actor addresses to their changes + // HAMT of miner address to some big structure... +} + +type MinerStateChange struct { + // Miner is the address of the miner that changes. + Miner address.Address `cborgen:"miner"` + // Funds is the funds that changed for this miner or empty. + Funds *minerdiff.FundsChange `cborgen:"funds"` + // Debt is the debt that changed for this miner or empty. + Debt *minerdiff.DebtChange `cborgen:"debt"` + // SectorStatus is the sectors whose status changed for this miner or empty. + SectorStatus *minerdiff.SectorStatusChange `cborgen:"sector_status"` + // Info is the cid of the miner change info that changed for this miner or empty. + Info *cid.Cid `cborgen:"info"` + // PreCommits is an AMT of the pre commits that changed for this miner or empty. + PreCommits *cid.Cid `cborgen:"pre_commits"` + // Sectors is an AMT of the sectors that changed for this miner or empty. + Sectors *cid.Cid `cborgen:"sectors"` +} + +func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.ActorStateChanges) (interface{}, error) { + m, err := adt.MakeEmptyMap(store, 5 /*TODO*/) + if err != nil { + return nil, err + } + if err := HandleMinerChanges(ctx, store, m, changes.MinerActors); err != nil { + return nil, err + } + return nil, nil +} + +func HandleMinerChanges(ctx context.Context, store adt.Store, actorHamt *adt.Map, miners map[address.Address]*minerdiff.StateDiff) error { + for addr, change := range miners { + msc, err := HandleMinerChange(ctx, store, addr, change) + if err != nil { + return err + } + if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { + return err + } + fmt.Printf("%+v\n", msc) + } + return nil +} + +func HandleMinerChange(ctx context.Context, store adt.Store, miner address.Address, change *minerdiff.StateDiff) (*MinerStateChange, error) { + out := &MinerStateChange{ + Miner: miner, + } + if change.FundsChange != nil { + out.Funds = change.FundsChange + } + if change.DebtChange != nil { + out.Debt = change.DebtChange + } + if change.SectorStatusChanges != nil { + out.SectorStatus = change.SectorStatusChanges + } + if change.InfoChange != nil { + c, err := MinerInfoHandler(ctx, store, change.InfoChange) + if err != nil { + return nil, err + } + out.Info = &c + } + if change.SectorChanges != nil && len(change.SectorChanges) > 0 { + c, err := MinerSectorHandler(ctx, store, change.SectorChanges) + if err != nil { + return nil, err + } + out.Sectors = &c + } + if change.PreCommitChanges != nil && len(change.PreCommitChanges) > 0 { + c, err := MinerPreCommitHandler(ctx, store, change.PreCommitChanges) + if err != nil { + return nil, err + } + out.PreCommits = &c + } + return out, nil +} + +func MinerPreCommitHandler(ctx context.Context, store adt.Store, list minerdiff.PreCommitChangeList) (cid.Cid, error) { + // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. + sort.Slice(list, func(i, j int) bool { + return util.MustCidOf(&list[i].PreCommit).KeyString() < util.MustCidOf(&list[j].PreCommit).KeyString() + }) + arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for _, l := range list { + if err := arr.AppendContinuous(l); err != nil { + return cid.Undef, err + } + } + return arr.Root() +} + +func MinerSectorHandler(ctx context.Context, store adt.Store, list minerdiff.SectorChangeList) (cid.Cid, error) { + // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. + sort.Slice(list, func(i, j int) bool { + return util.MustCidOf(&list[i].Sector).KeyString() < util.MustCidOf(&list[j].Sector).KeyString() + }) + arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for _, l := range list { + if err := arr.AppendContinuous(l); err != nil { + return cid.Undef, err + } + } + return arr.Root() +} + +type MinerInfo struct { + Info cid.Cid `cborgen:"info"` + Change core.ChangeType `cborgen:"change"` +} + +func MinerInfoHandler(ctx context.Context, store adt.Store, info *minerdiff.InfoChange) (cid.Cid, error) { + // ensure the miner info CID is the same as the CID found on chain. + infoCid, err := util.CidOf(&info.Info) + if err != nil { + return cid.Undef, err + } + infoChange := &MinerInfo{ + Info: infoCid, + Change: info.Change, + } + return store.Put(ctx, infoChange) +} diff --git a/pkg/util/cid.go b/pkg/util/cid.go new file mode 100644 index 000000000..25a4c14bd --- /dev/null +++ b/pkg/util/cid.go @@ -0,0 +1,34 @@ +package util + +import ( + "bytes" + + "github.com/filecoin-project/go-state-types/abi" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" +) + +func CidOf(v typegen.CBORMarshaler) (cid.Cid, error) { + buf := new(bytes.Buffer) + if err := v.MarshalCBOR(buf); err != nil { + return cid.Undef, err + } + c, err := abi.CidBuilder.Sum(buf.Bytes()) + if err != nil { + return cid.Undef, err + } + b, err := block.NewBlockWithCid(buf.Bytes(), c) + if err != nil { + return cid.Undef, err + } + return b.Cid(), nil +} + +func MustCidOf(v typegen.CBORMarshaler) cid.Cid { + c, err := CidOf(v) + if err != nil { + panic(err) + } + return c +} From 49241a6a6fbb2b4f16442e869dbed37901223a53 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 7 Dec 2022 19:29:21 -0800 Subject: [PATCH 07/48] generic actor map and array diffing :) --- chain/actors/builtin/miner/diff_deferred.go | 2 + pkg/core/array.go | 69 +++++++++++ pkg/core/map.go | 69 +++++++++++ pkg/extract/actors/generic/array.go | 69 +++++++++++ pkg/extract/actors/generic/map.go | 70 +++++++++++ pkg/extract/actors/minerdiff/infos.go | 2 - pkg/extract/actors/minerdiff/precommits.go | 65 ++++------ pkg/extract/actors/minerdiff/sectors.go | 60 +++------ pkg/extract/actors/minerdiff/state.go | 1 + pkg/extract/actors/verifregdiff/verifiers.go | 123 +++++++++++++++++++ 10 files changed, 444 insertions(+), 86 deletions(-) create mode 100644 pkg/core/array.go create mode 100644 pkg/core/map.go create mode 100644 pkg/extract/actors/generic/array.go create mode 100644 pkg/extract/actors/generic/map.go create mode 100644 pkg/extract/actors/verifregdiff/verifiers.go diff --git a/chain/actors/builtin/miner/diff_deferred.go b/chain/actors/builtin/miner/diff_deferred.go index 139499709..1e934b18a 100644 --- a/chain/actors/builtin/miner/diff_deferred.go +++ b/chain/actors/builtin/miner/diff_deferred.go @@ -22,6 +22,8 @@ type ChangeDiff struct { Current *typegen.Deferred } +// TODO there may be value in storing the map and array key in the change. + func DiffPreCommitsDeferred(ctx context.Context, store adt.Store, parent, child State) (*Changes, error) { parentMap, err := parent.PrecommitsMap() if err != nil { diff --git a/pkg/core/array.go b/pkg/core/array.go new file mode 100644 index 000000000..e5054ef29 --- /dev/null +++ b/pkg/core/array.go @@ -0,0 +1,69 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-amt-ipld/v4" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +type ArrayDiff struct { + Added []*ArrayChange + Modified []*ArrayModification + Removed []*ArrayChange +} + +func (m *ArrayDiff) Size() int { + return len(m.Added) + len(m.Removed) + len(m.Modified) +} + +type ArrayModification struct { + Key uint64 + Previous typegen.Deferred + Current typegen.Deferred +} + +type ArrayChange struct { + Key uint64 + Value typegen.Deferred +} + +func DiffArray(ctx context.Context, store adt.Store, child, parent adt.Array, childBw, parentBw int) (*ArrayDiff, error) { + // TODO handle different bitwidth + if childBw != parentBw { + panic("here") + } + changes, err := diff.Amt(ctx, parent, child, store, store, amt.UseTreeBitWidth(uint(childBw))) + if err != nil { + return nil, err + } + out := &ArrayDiff{ + Added: make([]*ArrayChange, 0, len(changes)), + Modified: make([]*ArrayModification, 0, len(changes)), + Removed: make([]*ArrayChange, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case amt.Add: + out.Added = append(out.Added, &ArrayChange{ + Key: change.Key, + Value: *change.After, + }) + case amt.Remove: + out.Removed = append(out.Removed, &ArrayChange{ + Key: change.Key, + Value: *change.Before, + }) + case amt.Modify: + out.Modified = append(out.Modified, &ArrayModification{ + Key: change.Key, + Previous: *change.Before, + Current: *change.After, + }) + } + } + return out, nil +} diff --git a/pkg/core/map.go b/pkg/core/map.go new file mode 100644 index 000000000..664103010 --- /dev/null +++ b/pkg/core/map.go @@ -0,0 +1,69 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-hamt-ipld/v3" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/adt/diff" +) + +type MapDiff struct { + Added []*MapChange + Modified []*MapModification + Removed []*MapChange +} + +func (m *MapDiff) Size() int { + return len(m.Added) + len(m.Removed) + len(m.Modified) +} + +type MapModification struct { + Key string + Previous typegen.Deferred + Current typegen.Deferred +} + +type MapChange struct { + Key string + Value typegen.Deferred +} + +func DiffMap(ctx context.Context, store adt.Store, child, parent adt.Map, childOpts, parentOpts *adt.MapOpts) (*MapDiff, error) { + // TODO handle different bitwidth and handFunctions + if !childOpts.Equal(parentOpts) { + panic("here") + } + changes, err := diff.Hamt(ctx, parent, child, store, store, hamt.UseHashFunction(hamt.HashFunc(childOpts.HashFunc)), hamt.UseTreeBitWidth(childOpts.Bitwidth)) + if err != nil { + return nil, err + } + out := &MapDiff{ + Added: make([]*MapChange, 0, len(changes)), + Modified: make([]*MapModification, 0, len(changes)), + Removed: make([]*MapChange, 0, len(changes)), + } + for _, change := range changes { + switch change.Type { + case hamt.Add: + out.Added = append(out.Added, &MapChange{ + Key: change.Key, + Value: *change.After, + }) + case hamt.Remove: + out.Removed = append(out.Removed, &MapChange{ + Key: change.Key, + Value: *change.Before, + }) + case hamt.Modify: + out.Modified = append(out.Modified, &MapModification{ + Key: change.Key, + Previous: *change.Before, + Current: *change.After, + }) + } + } + return out, nil +} diff --git a/pkg/extract/actors/generic/array.go b/pkg/extract/actors/generic/array.go new file mode 100644 index 000000000..e735d8810 --- /dev/null +++ b/pkg/extract/actors/generic/array.go @@ -0,0 +1,69 @@ +package generic + +import ( + "context" + + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type ActorStateArrayLoader = func(interface{}) (adt.Array, int, error) + +type ArrayChange struct { + Key uint64 + Value typegen.Deferred + Change core.ChangeType +} + +type ArrayChangeList = []*MapChange + +func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (*core.ArrayDiff, error) { + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentActor, err := actorStateLoader(api.Store(), act.Current) + if err != nil { + return nil, err + } + + currentArray, currentArrayBitWidth, err := actorArrayLoader(currentActor) + if err != nil { + return nil, err + } + if act.Type == core.ChangeTypeAdd { + out := &core.ArrayDiff{ + Added: make([]*core.ArrayChange, 0), + Modified: make([]*core.ArrayModification, 0), + Removed: make([]*core.ArrayChange, 0), + } + var v typegen.Deferred + if err := currentArray.ForEach(&v, func(key int64) error { + out.Added = append(out.Added, &core.ArrayChange{ + // TODO this type is inconsistent in specs-actors.. + Key: uint64(key), + Value: v, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedArray, executedArrayBitWidth, err := actorArrayLoader(executedActor) + if err != nil { + return nil, err + } + + return core.DiffArray(ctx, api.Store(), currentArray, executedArray, currentArrayBitWidth, executedArrayBitWidth) +} diff --git a/pkg/extract/actors/generic/map.go b/pkg/extract/actors/generic/map.go new file mode 100644 index 000000000..4e5e77277 --- /dev/null +++ b/pkg/extract/actors/generic/map.go @@ -0,0 +1,70 @@ +package generic + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type ActorStateLoader = func(adt.Store, *types.Actor) (interface{}, error) +type ActorStateMapLoader = func(interface{}) (adt.Map, *adt.MapOpts, error) + +type MapChange struct { + Key string + Value typegen.Deferred + Change core.ChangeType +} + +type MapChangeList = []*MapChange + +func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (*core.MapDiff, error) { + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentActor, err := actorStateLoader(api.Store(), act.Current) + if err != nil { + return nil, err + } + + currentMap, currentMapOpts, err := actorMapLoader(currentActor) + if err != nil { + return nil, err + } + if act.Type == core.ChangeTypeAdd { + out := &core.MapDiff{ + Added: make([]*core.MapChange, 0), + Modified: make([]*core.MapModification, 0), + Removed: make([]*core.MapChange, 0), + } + var v typegen.Deferred + if err := currentMap.ForEach(&v, func(key string) error { + out.Added = append(out.Added, &core.MapChange{ + Key: key, + Value: v, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil + } + + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + executedMap, executedMapOpts, err := actorMapLoader(executedActor) + if err != nil { + return nil, err + } + + return core.DiffMap(ctx, api.Store(), currentMap, executedMap, currentMapOpts, executedMapOpts) +} diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/infos.go index 92ec56aa4..d5620376c 100644 --- a/pkg/extract/actors/minerdiff/infos.go +++ b/pkg/extract/actors/minerdiff/infos.go @@ -4,7 +4,6 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/cbor" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" @@ -19,7 +18,6 @@ import ( var _ actors.ActorStateChange = (*InfoChange)(nil) type InfoChange struct { - cbor.Er Info typegen.Deferred `cborgen:"info"` Change core.ChangeType `cborgen:"change"` } diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/precommits.go index 7b64d7cd0..03fedad42 100644 --- a/pkg/extract/actors/minerdiff/precommits.go +++ b/pkg/extract/actors/minerdiff/precommits.go @@ -3,11 +3,14 @@ package minerdiff import ( "context" + "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" "github.com/filecoin-project/lily/tasks" ) @@ -29,70 +32,48 @@ func (p PreCommitChangeList) Kind() actors.ActorStateKind { type PreCommit struct{} func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - return DiffPreCommits(ctx, api, act) + return PreCommitDiff(ctx, api, act) } -func DiffPreCommits(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // the actor was removed, nothing has changes in the current state. - if act.Type == core.ChangeTypeRemove { - return nil, nil +func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + minerStateLoader := func(store adt.Store, act *types.Actor) (interface{}, error) { + return miner.Load(api.Store(), act) } - - currentMiner, err := miner.Load(api.Store(), act.Current) - if err != nil { - return nil, err - } - // the actor was added, everything is new in the current state. - if act.Type == core.ChangeTypeAdd { - var out PreCommitChangeList - pm, err := currentMiner.PrecommitsMap() + minerMapLoader := func(m interface{}) (adt.Map, *adt.MapOpts, error) { + minerState := m.(miner.State) + perCommitMap, err := minerState.PrecommitsMap() if err != nil { - return nil, err - } - var v typegen.Deferred - if err := pm.ForEach(&v, func(key string) error { - out = append(out, &PreCommitChange{ - PreCommit: v, - Change: core.ChangeTypeAdd, - }) - return nil - }); err != nil { - return nil, err + return nil, nil, err } - return out, nil + return perCommitMap, &adt.MapOpts{ + Bitwidth: minerState.PrecommitsMapBitWidth(), + HashFunc: minerState.PrecommitsMapHashFunction(), + }, nil } - - // actor was modified load executed state. - executedMiner, err := miner.Load(api.Store(), act.Executed) + mapChange, err := generic.DiffActorMap(ctx, api, act, minerStateLoader, minerMapLoader) if err != nil { return nil, err } - - preCommitChanges, err := miner.DiffPreCommitsDeferred(ctx, api.Store(), executedMiner, currentMiner) - if err != nil { - return nil, err - } - + out := make(PreCommitChangeList, mapChange.Size()) idx := 0 - out := make(PreCommitChangeList, len(preCommitChanges.Added)+len(preCommitChanges.Removed)+len(preCommitChanges.Modified)) - for _, change := range preCommitChanges.Added { + for _, change := range mapChange.Added { out[idx] = &PreCommitChange{ - PreCommit: *change, + PreCommit: change.Value, Change: core.ChangeTypeAdd, } idx++ } - for _, change := range preCommitChanges.Removed { + for _, change := range mapChange.Removed { out[idx] = &PreCommitChange{ - PreCommit: *change, + PreCommit: change.Value, Change: core.ChangeTypeRemove, } idx++ } // NB: PreCommits cannot be modified, but check anyway. - for _, change := range preCommitChanges.Modified { + for _, change := range mapChange.Modified { out[idx] = &PreCommitChange{ - PreCommit: *change.Current, + PreCommit: change.Current, Change: core.ChangeTypeModify, } idx++ diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/sectors.go index 52a166153..c37e87f54 100644 --- a/pkg/extract/actors/minerdiff/sectors.go +++ b/pkg/extract/actors/minerdiff/sectors.go @@ -3,11 +3,14 @@ package minerdiff import ( "context" + "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" "github.com/filecoin-project/lily/tasks" ) @@ -33,70 +36,43 @@ func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.Actor } func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // the actor was removed, nothing has changes in the current state. - if act.Type == core.ChangeTypeRemove { - return nil, nil + minerStateLoader := func(store adt.Store, act *types.Actor) (interface{}, error) { + return miner.Load(api.Store(), act) } - - currentMiner, err := miner.Load(api.Store(), act.Current) - if err != nil { - return nil, err - } - - // the actor was added, everything is new in the current state. - if act.Type == core.ChangeTypeAdd { - sa, err := currentMiner.SectorsArray() + minerArrayLoader := func(m interface{}) (adt.Array, int, error) { + minerState := m.(miner.State) + sectorArray, err := minerState.SectorsArray() if err != nil { - return nil, err - } - out := make(SectorChangeList, int(sa.Length())) - var v typegen.Deferred - if err := sa.ForEach(&v, func(idx int64) error { - out[idx] = &SectorChange{ - Sector: v, - Change: core.ChangeTypeAdd, - } - return nil - }); err != nil { - return nil, err + return nil, -1, err } - return out, nil + return sectorArray, minerState.SectorsAmtBitwidth(), nil } - - // the actor was modified, diff against executed state. - executedMiner, err := miner.Load(api.Store(), act.Executed) + arrayChange, err := generic.DiffActorArray(ctx, api, act, minerStateLoader, minerArrayLoader) if err != nil { return nil, err } - - sectorChanges, err := miner.DiffSectorsDeferred(ctx, api.Store(), executedMiner, currentMiner) - if err != nil { - return nil, err - } - idx := 0 - out := make(SectorChangeList, len(sectorChanges.Added)+len(sectorChanges.Removed)+len(sectorChanges.Modified)) - for _, change := range sectorChanges.Added { + out := make(SectorChangeList, arrayChange.Size()) + for _, change := range arrayChange.Added { out[idx] = &SectorChange{ - Sector: *change, + Sector: change.Value, Change: core.ChangeTypeAdd, } idx++ } - for _, change := range sectorChanges.Removed { + for _, change := range arrayChange.Removed { out[idx] = &SectorChange{ - Sector: *change, + Sector: change.Value, Change: core.ChangeTypeRemove, } idx++ } - for _, change := range sectorChanges.Modified { + for _, change := range arrayChange.Modified { out[idx] = &SectorChange{ - Sector: *change.Current, + Sector: change.Current, Change: core.ChangeTypeModify, } idx++ } return out, nil - } diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go index 289aa76d7..61e1f5c9f 100644 --- a/pkg/extract/actors/minerdiff/state.go +++ b/pkg/extract/actors/minerdiff/state.go @@ -21,6 +21,7 @@ type ActorStateKind string func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { var stateDiff = new(StateDiff) for _, f := range diffFns { + // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. stateChange, err := f.Diff(ctx, api, act) if err != nil { return nil, err diff --git a/pkg/extract/actors/verifregdiff/verifiers.go b/pkg/extract/actors/verifregdiff/verifiers.go new file mode 100644 index 000000000..8bbd95bb5 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/verifiers.go @@ -0,0 +1,123 @@ +package verifregdiff + +import ( + "context" + + "github.com/filecoin-project/go-address" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type VerifiersChange struct { + Verifier address.Address + DataCap typegen.Deferred + Change core.ChangeType +} + +type VerifiersChangeList []*VerifiersChange + +const KindVerifrefVerifiers = "verifreg_verifiers" + +func (v VerifiersChangeList) Kind() actors.ActorStateKind { + return KindVerifrefVerifiers +} + +type Verifiers struct{} + +func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DiffVerifiers(ctx, api, act) +} + +func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + if act.Type == core.ChangeTypeRemove { + return nil, nil + } + + currentVerifreg, err := verifreg.Load(api.Store(), act.Current) + if err != nil { + return nil, err + } + + // the actor was added, everything is new in the current state. + // NB: since this is a singleton actor it was added at genesis and we'll probably never hit this case on mainnet, a test net could in theory. + if act.Type == core.ChangeTypeAdd { + var out VerifiersChangeList + vm, err := currentVerifreg.VerifiersMap() + if err != nil { + return nil, err + } + var v typegen.Deferred + if err := vm.ForEach(&v, func(key string) error { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + out = append(out, &VerifiersChange{ + Verifier: addr, + DataCap: v, + Change: core.ChangeTypeAdd, + }) + return nil + }); err != nil { + return nil, err + } + } + + executedVerifreg, err := verifreg.Load(api.Store(), act.Executed) + if err != nil { + return nil, err + } + + verifierChanges, err := verifreg.DiffVerifiersDeferred(ctx, api.Store(), executedVerifreg, currentVerifreg) + if err != nil { + return nil, err + } + + idx := 0 + out := make(VerifiersChangeList, verifierChanges.Size()) + for _, change := range verifierChanges.Added { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &VerifiersChange{ + Verifier: addr, + DataCap: change.Value, + Change: core.ChangeTypeAdd, + } + idx++ + } + for _, change := range verifierChanges.Removed { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &VerifiersChange{ + Verifier: addr, + DataCap: change.Value, + Change: core.ChangeTypeRemove, + } + idx++ + } + for _, change := range verifierChanges.Modified { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &VerifiersChange{ + Verifier: addr, + DataCap: change.Current, + Change: core.ChangeTypeModify, + } + idx++ + } + return out, nil +} From 75fe34080dd7420bcc780ef82cf62ef3b1ffd81d Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 8 Dec 2022 16:50:46 -0800 Subject: [PATCH 08/48] verifireg extraction --- .../actors/builtin/verifreg/actor.go.template | 5 + .../actors/builtin/verifreg/state.go.template | 69 ++++++++ chain/actors/builtin/verifreg/v0.go | 35 ++++ chain/actors/builtin/verifreg/v2.go | 35 ++++ chain/actors/builtin/verifreg/v3.go | 35 ++++ chain/actors/builtin/verifreg/v4.go | 35 ++++ chain/actors/builtin/verifreg/v5.go | 35 ++++ chain/actors/builtin/verifreg/v6.go | 35 ++++ chain/actors/builtin/verifreg/v7.go | 35 ++++ chain/actors/builtin/verifreg/v8.go | 35 ++++ chain/actors/builtin/verifreg/v9.go | 53 ++++++ chain/actors/builtin/verifreg/verifreg.go | 5 + .../distributed/queue/tracing/carrier.go | 2 +- lens/lily/impl.go | 1 + pkg/extract/actors/minerdiff/load.go | 32 ++++ pkg/extract/actors/minerdiff/precommits.go | 20 +-- pkg/extract/actors/minerdiff/sectors.go | 17 +- pkg/extract/actors/minerdiff/state.go | 2 - pkg/extract/actors/verifregdiff/claims.go | 156 ++++++++++++++++++ pkg/extract/actors/verifregdiff/clients.go | 84 ++++++++++ pkg/extract/actors/verifregdiff/load.go | 49 ++++++ pkg/extract/actors/verifregdiff/state.go | 36 ++++ pkg/extract/actors/verifregdiff/verifiers.go | 56 +------ pkg/extract/procesor/process.go | 46 ++++-- 24 files changed, 818 insertions(+), 95 deletions(-) create mode 100644 pkg/extract/actors/minerdiff/load.go create mode 100644 pkg/extract/actors/verifregdiff/claims.go create mode 100644 pkg/extract/actors/verifregdiff/clients.go create mode 100644 pkg/extract/actors/verifregdiff/load.go create mode 100644 pkg/extract/actors/verifregdiff/state.go diff --git a/chain/actors/builtin/verifreg/actor.go.template b/chain/actors/builtin/verifreg/actor.go.template index 6882b000a..49da54270 100644 --- a/chain/actors/builtin/verifreg/actor.go.template +++ b/chain/actors/builtin/verifreg/actor.go.template @@ -72,6 +72,11 @@ type State interface { VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + ClaimsMap() (adt.Map, error) + ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) + ClaimsMapBitWidth() int + ClaimsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/actors/builtin/verifreg/state.go.template b/chain/actors/builtin/verifreg/state.go.template index 0a2512b3d..442558e1b 100644 --- a/chain/actors/builtin/verifreg/state.go.template +++ b/chain/actors/builtin/verifreg/state.go.template @@ -8,6 +8,10 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" +{{if (ge .v 9)}} + cbg "github.com/whyrusleeping/cbor-gen" +{{end}} + "github.com/filecoin-project/lily/chain/actors/adt" {{if (le .v 1)}} @@ -198,3 +202,68 @@ func (s *state{{.v}}) GetClaims(providerIdAddr address.Address) (map[verifreg9.C return s.LoadClaimsToMap(s.store, providerIdAddr) {{end}} } + +func (s *state{{.v}}) ClaimsMap() (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + return adt{{.v}}.AsMap(s.store, s.Claims, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + +// TODO this could return an error since not all versions have a claims map +func (s *state{{.v}}) ClaimsMapBitWidth() int { + {{if (ge .v 3)}} + return builtin{{.v}}.DefaultHamtBitwidth + {{else}} + return 5 + {{end}} +} + +// TODO this could return an error since not all versions have a claims map +func (s *state{{.v}}) ClaimsMapHashFunction() func(input []byte) []byte { + {{if (le .v 1)}} + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + {{else}} + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + {{end}} +} + +func (s *state{{.v}}) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(providerIdAddr), s.Claims, builtin{{.v}}.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + +func (s *state{{.v}}) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { +{{if (le .v 8)}} + return cid.Undef, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + actorToHamtMap, err := adt{{.v}}.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +{{end}} +} + diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index c48716228..1791dd447 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -153,3 +153,38 @@ func (s *state0) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v0") } + +func (s *state0) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state0) ClaimsMapBitWidth() int { + + return 5 + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state0) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + +} + +func (s *state0) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +func (s *state0) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v0") + +} diff --git a/chain/actors/builtin/verifreg/v2.go b/chain/actors/builtin/verifreg/v2.go index a1d388436..54ad540f2 100644 --- a/chain/actors/builtin/verifreg/v2.go +++ b/chain/actors/builtin/verifreg/v2.go @@ -153,3 +153,38 @@ func (s *state2) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v2") } + +func (s *state2) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state2) ClaimsMapBitWidth() int { + + return 5 + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state2) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state2) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +func (s *state2) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v2") + +} diff --git a/chain/actors/builtin/verifreg/v3.go b/chain/actors/builtin/verifreg/v3.go index 3be489f1f..f6dce7c04 100644 --- a/chain/actors/builtin/verifreg/v3.go +++ b/chain/actors/builtin/verifreg/v3.go @@ -155,3 +155,38 @@ func (s *state3) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v3") } + +func (s *state3) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state3) ClaimsMapBitWidth() int { + + return builtin3.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state3) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state3) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +func (s *state3) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v3") + +} diff --git a/chain/actors/builtin/verifreg/v4.go b/chain/actors/builtin/verifreg/v4.go index 819b547ad..748e3bffd 100644 --- a/chain/actors/builtin/verifreg/v4.go +++ b/chain/actors/builtin/verifreg/v4.go @@ -155,3 +155,38 @@ func (s *state4) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v4") } + +func (s *state4) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state4) ClaimsMapBitWidth() int { + + return builtin4.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state4) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state4) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +func (s *state4) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v4") + +} diff --git a/chain/actors/builtin/verifreg/v5.go b/chain/actors/builtin/verifreg/v5.go index 83a2496f5..325ffcce0 100644 --- a/chain/actors/builtin/verifreg/v5.go +++ b/chain/actors/builtin/verifreg/v5.go @@ -155,3 +155,38 @@ func (s *state5) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v5") } + +func (s *state5) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state5) ClaimsMapBitWidth() int { + + return builtin5.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state5) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state5) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +func (s *state5) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v5") + +} diff --git a/chain/actors/builtin/verifreg/v6.go b/chain/actors/builtin/verifreg/v6.go index 8aec727be..bf78ed5da 100644 --- a/chain/actors/builtin/verifreg/v6.go +++ b/chain/actors/builtin/verifreg/v6.go @@ -155,3 +155,38 @@ func (s *state6) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v6") } + +func (s *state6) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state6) ClaimsMapBitWidth() int { + + return builtin6.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state6) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state6) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +func (s *state6) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v6") + +} diff --git a/chain/actors/builtin/verifreg/v7.go b/chain/actors/builtin/verifreg/v7.go index 36b157b93..ef43ae36c 100644 --- a/chain/actors/builtin/verifreg/v7.go +++ b/chain/actors/builtin/verifreg/v7.go @@ -154,3 +154,38 @@ func (s *state7) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v7") } + +func (s *state7) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state7) ClaimsMapBitWidth() int { + + return builtin7.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state7) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state7) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +func (s *state7) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v7") + +} diff --git a/chain/actors/builtin/verifreg/v8.go b/chain/actors/builtin/verifreg/v8.go index ab2ade637..7bcf54765 100644 --- a/chain/actors/builtin/verifreg/v8.go +++ b/chain/actors/builtin/verifreg/v8.go @@ -153,3 +153,38 @@ func (s *state8) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return nil, fmt.Errorf("unsupported in actors v8") } + +func (s *state8) ClaimsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state8) ClaimsMapBitWidth() int { + + return builtin8.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state8) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state8) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +func (s *state8) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + return cid.Undef, fmt.Errorf("unsupported in actors v8") + +} diff --git a/chain/actors/builtin/verifreg/v9.go b/chain/actors/builtin/verifreg/v9.go index 3b8655b81..58e7e5365 100644 --- a/chain/actors/builtin/verifreg/v9.go +++ b/chain/actors/builtin/verifreg/v9.go @@ -8,6 +8,8 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/chain/actors/adt" "crypto/sha256" @@ -153,3 +155,54 @@ func (s *state9) GetClaims(providerIdAddr address.Address) (map[verifreg9.ClaimI return s.LoadClaimsToMap(s.store, providerIdAddr) } + +func (s *state9) ClaimsMap() (adt.Map, error) { + + return adt9.AsMap(s.store, s.Claims, builtin9.DefaultHamtBitwidth) + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state9) ClaimsMapBitWidth() int { + + return builtin9.DefaultHamtBitwidth + +} + +// TODO this could return an error since not all versions have a claims map +func (s *state9) ClaimsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state9) ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) { + + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(providerIdAddr), s.Claims, builtin9.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, builtin9.DefaultHamtBitwidth) + +} + +func (s *state9) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + + actorToHamtMap, err := adt9.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil + +} diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 104ff2d4e..31792b0b0 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -99,6 +99,11 @@ type State interface { VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + ClaimsMap() (adt.Map, error) + ClaimMapForProvider(providerIdAddr address.Address) (adt.Map, error) + ClaimsMapBitWidth() int + ClaimsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/indexer/distributed/queue/tracing/carrier.go b/chain/indexer/distributed/queue/tracing/carrier.go index caf106460..0c95126d2 100644 --- a/chain/indexer/distributed/queue/tracing/carrier.go +++ b/chain/indexer/distributed/queue/tracing/carrier.go @@ -32,7 +32,7 @@ type TraceCarrier struct { Remote bool `json:"remote"` } -//MarshalJSON converts TraceCarrier to a trace.SpanContext and marshals it to JSON. +// MarshalJSON converts TraceCarrier to a trace.SpanContext and marshals it to JSON. func (c *TraceCarrier) MarshalJSON() ([]byte, error) { return c.AsSpanContext().MarshalJSON() } diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 7d44dc51f..396ce66dd 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -171,6 +171,7 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf } changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + return nil, nil bs := blockstore.NewMemorySync() store := adt2.WrapBlockStore(ctx, bs) if _, err := cbor.ProcessActors(ctx, store, changes); err != nil { diff --git a/pkg/extract/actors/minerdiff/load.go b/pkg/extract/actors/minerdiff/load.go new file mode 100644 index 000000000..f804e9d03 --- /dev/null +++ b/pkg/extract/actors/minerdiff/load.go @@ -0,0 +1,32 @@ +package minerdiff + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" +) + +var MinerStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return miner.Load(store, act) +} +var MinerPreCommitMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + minerState := m.(miner.State) + perCommitMap, err := minerState.PrecommitsMap() + if err != nil { + return nil, nil, err + } + return perCommitMap, &adt.MapOpts{ + Bitwidth: minerState.PrecommitsMapBitWidth(), + HashFunc: minerState.PrecommitsMapHashFunction(), + }, nil +} + +var MinerSectorArrayLoader = func(m interface{}) (adt.Array, int, error) { + minerState := m.(miner.State) + sectorArray, err := minerState.SectorsArray() + if err != nil { + return nil, -1, err + } + return sectorArray, minerState.SectorsAmtBitwidth(), nil +} diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/precommits.go index 03fedad42..340e4a040 100644 --- a/pkg/extract/actors/minerdiff/precommits.go +++ b/pkg/extract/actors/minerdiff/precommits.go @@ -3,11 +3,8 @@ package minerdiff import ( "context" - "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -17,6 +14,7 @@ import ( var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) type PreCommitChange struct { + // TODO include sector ID PreCommit typegen.Deferred `cborgen:"pre_commit"` Change core.ChangeType `cborgen:"change"` } @@ -36,21 +34,7 @@ func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act } func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - minerStateLoader := func(store adt.Store, act *types.Actor) (interface{}, error) { - return miner.Load(api.Store(), act) - } - minerMapLoader := func(m interface{}) (adt.Map, *adt.MapOpts, error) { - minerState := m.(miner.State) - perCommitMap, err := minerState.PrecommitsMap() - if err != nil { - return nil, nil, err - } - return perCommitMap, &adt.MapOpts{ - Bitwidth: minerState.PrecommitsMapBitWidth(), - HashFunc: minerState.PrecommitsMapHashFunction(), - }, nil - } - mapChange, err := generic.DiffActorMap(ctx, api, act, minerStateLoader, minerMapLoader) + mapChange, err := generic.DiffActorMap(ctx, api, act, MinerStateLoader, MinerPreCommitMapLoader) if err != nil { return nil, err } diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/sectors.go index c37e87f54..0bc8ee4ea 100644 --- a/pkg/extract/actors/minerdiff/sectors.go +++ b/pkg/extract/actors/minerdiff/sectors.go @@ -3,11 +3,8 @@ package minerdiff import ( "context" - "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -17,6 +14,7 @@ import ( var _ actors.ActorStateChange = (*SectorChangeList)(nil) type SectorChange struct { + // TODO include sectorID key Sector typegen.Deferred `cborgen:"sector"` Change core.ChangeType `cborgen:"change"` } @@ -36,18 +34,7 @@ func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.Actor } func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - minerStateLoader := func(store adt.Store, act *types.Actor) (interface{}, error) { - return miner.Load(api.Store(), act) - } - minerArrayLoader := func(m interface{}) (adt.Array, int, error) { - minerState := m.(miner.State) - sectorArray, err := minerState.SectorsArray() - if err != nil { - return nil, -1, err - } - return sectorArray, minerState.SectorsAmtBitwidth(), nil - } - arrayChange, err := generic.DiffActorArray(ctx, api, act, minerStateLoader, minerArrayLoader) + arrayChange, err := generic.DiffActorArray(ctx, api, act, MinerStateLoader, MinerSectorArrayLoader) if err != nil { return nil, err } diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go index 61e1f5c9f..fd8c431d8 100644 --- a/pkg/extract/actors/minerdiff/state.go +++ b/pkg/extract/actors/minerdiff/state.go @@ -16,8 +16,6 @@ type StateDiff struct { SectorStatusChanges *SectorStatusChange } -type ActorStateKind string - func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { var stateDiff = new(StateDiff) for _, f := range diffFns { diff --git a/pkg/extract/actors/verifregdiff/claims.go b/pkg/extract/actors/verifregdiff/claims.go new file mode 100644 index 000000000..bdb859122 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/claims.go @@ -0,0 +1,156 @@ +package verifregdiff + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type ClaimsChange struct { + Provider address.Address + ClaimID uint64 + Claim typegen.Deferred + Change core.ChangeType +} + +type ClaimsChangeList []*ClaimsChange + +const KindVerifregClaims = "verifreg_claims" + +func (c ClaimsChangeList) Kind() actors.ActorStateKind { + return KindVerifregClaims +} + +type Claims struct{} + +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DiffClaims(ctx, api, act) +} + +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiiregClaimsMapLoader) + if err != nil { + return nil, err + } + out := make(ClaimsChangeList, 0) + for _, change := range mapChange.Added { + providerID, err := abi.ParseUIntKey(change.Key) + if err != nil { + return nil, err + } + providerAddress, err := address.NewIDAddress(providerID) + if err != nil { + return nil, err + } + added, err := diffSubMap(ctx, api, act, providerAddress) + if err != nil { + return nil, err + } + if len(added) > 0 { + out = append(out, added...) + } + } + for _, change := range mapChange.Removed { + providerID, err := abi.ParseUIntKey(change.Key) + if err != nil { + return nil, err + } + providerAddress, err := address.NewIDAddress(providerID) + if err != nil { + return nil, err + } + removed, err := diffSubMap(ctx, api, act, providerAddress) + if err != nil { + return nil, err + } + if len(removed) > 0 { + out = append(out, removed...) + } + } + for _, change := range mapChange.Modified { + providerID, err := abi.ParseUIntKey(change.Key) + if err != nil { + return nil, err + } + providerAddress, err := address.NewIDAddress(providerID) + if err != nil { + return nil, err + } + modified, err := diffSubMap(ctx, api, act, providerAddress) + if err != nil { + return nil, err + } + if len(modified) > 0 { + out = append(out, modified...) + } + } + + return out, nil +} + +func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerID address.Address) ([]*ClaimsChange, error) { + subMapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + providerID := providerID + verifregState := i.(verifreg.State) + providerClaimMap, err := verifregState.ClaimMapForProvider(providerID) + if err != nil { + return nil, nil, err + } + return providerClaimMap, &adt.MapOpts{ + Bitwidth: verifregState.ClaimsMapBitWidth(), + HashFunc: verifregState.ClaimsMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*ClaimsChange, 0, subMapChange.Size()) + for _, subChange := range subMapChange.Added { + claimID, err := abi.ParseUIntKey(subChange.Key) + if err != nil { + return nil, err + } + out = append(out, &ClaimsChange{ + Provider: providerID, + ClaimID: claimID, + Claim: subChange.Value, + Change: core.ChangeTypeAdd, + }) + } + for _, subChange := range subMapChange.Removed { + claimID, err := abi.ParseUIntKey(subChange.Key) + if err != nil { + return nil, err + } + out = append(out, &ClaimsChange{ + Provider: providerID, + ClaimID: claimID, + Claim: subChange.Value, + Change: core.ChangeTypeRemove, + }) + } + for _, subChange := range subMapChange.Modified { + claimID, err := abi.ParseUIntKey(subChange.Key) + if err != nil { + return nil, err + } + out = append(out, &ClaimsChange{ + Provider: providerID, + ClaimID: claimID, + Claim: subChange.Current, + Change: core.ChangeTypeModify, + }) + } + + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/clients.go b/pkg/extract/actors/verifregdiff/clients.go new file mode 100644 index 000000000..027828651 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/clients.go @@ -0,0 +1,84 @@ +package verifregdiff + +import ( + "context" + + "github.com/filecoin-project/go-address" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type ClientsChange struct { + Client address.Address + DataCap typegen.Deferred + Change core.ChangeType +} + +type ClientsChangeList []*ClientsChange + +const KindVerifregClients = "verifreg_clients" + +func (v ClientsChangeList) Kind() actors.ActorStateKind { + return KindVerifregClients +} + +type Clients struct{} + +func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + return DiffClients(ctx, api, act) +} + +func DiffClients(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregClientsMapLoader) + if err != nil { + return nil, err + } + + idx := 0 + out := make(ClientsChangeList, mapChange.Size()) + for _, change := range mapChange.Added { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &ClientsChange{ + Client: addr, + DataCap: change.Value, + Change: core.ChangeTypeAdd, + } + idx++ + } + for _, change := range mapChange.Removed { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &ClientsChange{ + Client: addr, + DataCap: change.Value, + Change: core.ChangeTypeRemove, + } + idx++ + } + for _, change := range mapChange.Modified { + // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct + addr, err := address.NewFromBytes([]byte(change.Key)) + if err != nil { + return nil, err + } + out[idx] = &ClientsChange{ + Client: addr, + DataCap: change.Current, + Change: core.ChangeTypeModify, + } + idx++ + } + return out, nil + +} diff --git a/pkg/extract/actors/verifregdiff/load.go b/pkg/extract/actors/verifregdiff/load.go new file mode 100644 index 000000000..27a5ad933 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/load.go @@ -0,0 +1,49 @@ +package verifregdiff + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" +) + +// VerifregStateLoader returns the verifiergistry actor state for `act`. +var VerifregStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return verifreg.Load(store, act) +} + +var VerifiregVerifiersMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + verifierMap, err := verifregState.VerifiersMap() + if err != nil { + return nil, nil, err + } + return verifierMap, &adt.MapOpts{ + Bitwidth: verifregState.VerifiersMapBitWidth(), + HashFunc: verifregState.VerifiersMapHashFunction(), + }, nil +} + +var VerifiregClientsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + clientsMap, err := verifregState.VerifiedClientsMap() + if err != nil { + return nil, nil, err + } + return clientsMap, &adt.MapOpts{ + Bitwidth: verifregState.VerifiedClientsMapBitWidth(), + HashFunc: verifregState.VerifiedClientsMapHashFunction(), + }, nil +} + +var VerifiiregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + claimsMap, err := verifregState.ClaimsMap() + if err != nil { + return nil, nil, err + } + return claimsMap, &adt.MapOpts{ + Bitwidth: verifregState.ClaimsMapBitWidth(), + HashFunc: verifregState.ClaimsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/verifregdiff/state.go b/pkg/extract/actors/verifregdiff/state.go new file mode 100644 index 000000000..480d01c68 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/state.go @@ -0,0 +1,36 @@ +package verifregdiff + +import ( + "context" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiff struct { + VerifierChanges VerifiersChangeList + ClientChanges ClientsChangeList + ClaimChanges ClaimsChangeList +} + +func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { + var stateDiff = new(StateDiff) + for _, f := range diffFns { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindVerifregClients: + stateDiff.ClientChanges = stateChange.(ClientsChangeList) + case KindVerifregClaims: + stateDiff.ClaimChanges = stateChange.(ClaimsChangeList) + case KindVerifregVerifiers: + stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + } + } + return stateDiff, nil +} diff --git a/pkg/extract/actors/verifregdiff/verifiers.go b/pkg/extract/actors/verifregdiff/verifiers.go index 8bbd95bb5..7dc288a4c 100644 --- a/pkg/extract/actors/verifregdiff/verifiers.go +++ b/pkg/extract/actors/verifregdiff/verifiers.go @@ -6,9 +6,9 @@ import ( "github.com/filecoin-project/go-address" typegen "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" "github.com/filecoin-project/lily/tasks" ) @@ -20,10 +20,10 @@ type VerifiersChange struct { type VerifiersChangeList []*VerifiersChange -const KindVerifrefVerifiers = "verifreg_verifiers" +const KindVerifregVerifiers = "verifreg_verifiers" func (v VerifiersChangeList) Kind() actors.ActorStateKind { - return KindVerifrefVerifiers + return KindVerifregVerifiers } type Verifiers struct{} @@ -33,54 +33,14 @@ func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act } func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - if act.Type == core.ChangeTypeRemove { - return nil, nil - } - - currentVerifreg, err := verifreg.Load(api.Store(), act.Current) - if err != nil { - return nil, err - } - - // the actor was added, everything is new in the current state. - // NB: since this is a singleton actor it was added at genesis and we'll probably never hit this case on mainnet, a test net could in theory. - if act.Type == core.ChangeTypeAdd { - var out VerifiersChangeList - vm, err := currentVerifreg.VerifiersMap() - if err != nil { - return nil, err - } - var v typegen.Deferred - if err := vm.ForEach(&v, func(key string) error { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - out = append(out, &VerifiersChange{ - Verifier: addr, - DataCap: v, - Change: core.ChangeTypeAdd, - }) - return nil - }); err != nil { - return nil, err - } - } - - executedVerifreg, err := verifreg.Load(api.Store(), act.Executed) - if err != nil { - return nil, err - } - - verifierChanges, err := verifreg.DiffVerifiersDeferred(ctx, api.Store(), executedVerifreg, currentVerifreg) + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregVerifiersMapLoader) if err != nil { return nil, err } idx := 0 - out := make(VerifiersChangeList, verifierChanges.Size()) - for _, change := range verifierChanges.Added { + out := make(VerifiersChangeList, mapChange.Size()) + for _, change := range mapChange.Added { // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct addr, err := address.NewFromBytes([]byte(change.Key)) if err != nil { @@ -93,7 +53,7 @@ func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorC } idx++ } - for _, change := range verifierChanges.Removed { + for _, change := range mapChange.Removed { // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct addr, err := address.NewFromBytes([]byte(change.Key)) if err != nil { @@ -106,7 +66,7 @@ func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorC } idx++ } - for _, change := range verifierChanges.Modified { + for _, change := range mapChange.Modified { // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct addr, err := address.NewFromBytes([]byte(change.Key)) if err != nil { diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index c7bb0fb4a..0503c5ab7 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -10,16 +10,19 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin/market" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/chain/actors/builtin/power" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" "github.com/filecoin-project/lily/pkg/extract/statetree" "github.com/filecoin-project/lily/tasks" ) var ( - MinerCodes = cid.NewSet() - PowerCodes = cid.NewSet() - MarketCodes = cid.NewSet() + MinerCodes = cid.NewSet() + PowerCodes = cid.NewSet() + MarketCodes = cid.NewSet() + VerifregCodes = cid.NewSet() ) func init() { @@ -32,13 +35,17 @@ func init() { for _, c := range market.AllCodes() { MarketCodes.Add(c) } + for _, c := range verifreg.AllCodes() { + VerifregCodes.Add(c) + } } type ActorStateChanges struct { - Current *types.TipSet - Executed *types.TipSet - Actors map[address.Address]statetree.ActorDiff - MinerActors map[address.Address]*minerdiff.StateDiff + Current *types.TipSet + Executed *types.TipSet + Actors map[address.Address]statetree.ActorDiff + MinerActors map[address.Address]*minerdiff.StateDiff + VerifregActor map[address.Address]*verifregdiff.StateDiff } func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ActorStateChanges, error) { @@ -47,10 +54,11 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current return nil, err } asc := &ActorStateChanges{ - Current: current, - Executed: executed, - Actors: actorChanges, - MinerActors: make(map[address.Address]*minerdiff.StateDiff, len(actorChanges)), // there are at most actorChanges entries + Current: current, + Executed: executed, + Actors: actorChanges, + MinerActors: make(map[address.Address]*minerdiff.StateDiff, len(actorChanges)), // there are at most actorChanges entries + VerifregActor: make(map[address.Address]*verifregdiff.StateDiff, len(actorChanges)), } for addr, change := range actorChanges { @@ -73,6 +81,22 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current } asc.MinerActors[addr] = minerChanges } + if VerifregCodes.Has(change.Current.Code) { + verifregChanges, err := verifregdiff.State(ctx, api, &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, + }, + //verifregdiff.Clients{}, + verifregdiff.Verifiers{}, + verifregdiff.Claims{}, + ) + if err != nil { + return nil, err + } + asc.VerifregActor[addr] = verifregChanges + } } return asc, nil From a1d55109e5c8e4a49b5ecaebd976e9c5540c2f0a Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 13 Dec 2022 17:31:16 -0800 Subject: [PATCH 09/48] things work --- go.mod | 4 + go.sum | 10 + lens/lily/impl.go | 40 ++-- pkg/core/array.go | 144 ++++++++++---- pkg/core/map.go | 158 +++++++++++----- pkg/extract/actors/generic/array.go | 53 +++--- pkg/extract/actors/generic/map.go | 56 +++--- pkg/extract/actors/generic/types.go | 14 ++ pkg/extract/actors/minerdiff/cbor_gen.go | 175 ++++++++++++++++-- pkg/extract/actors/minerdiff/debt.go | 6 + pkg/extract/actors/minerdiff/funds.go | 6 + pkg/extract/actors/minerdiff/infos.go | 6 + pkg/extract/actors/minerdiff/precommits.go | 42 ++--- pkg/extract/actors/minerdiff/sector_status.go | 6 + pkg/extract/actors/minerdiff/sectors.go | 41 ++-- pkg/extract/actors/minerdiff/state.go | 45 ++++- pkg/extract/actors/types.go | 22 +++ pkg/extract/actors/verifregdiff/claims.go | 120 ++++-------- pkg/extract/actors/verifregdiff/clients.go | 60 ++---- pkg/extract/actors/verifregdiff/state.go | 13 +- pkg/extract/actors/verifregdiff/verifiers.go | 59 ++---- pkg/extract/procesor/process.go | 125 +++++++++---- pkg/extract/statetree/diff.go | 104 +++++++---- pkg/gen/main.go | 11 +- pkg/transform/cbor/{ => miner}/cbor_gen.go | 131 ++++++++++++- pkg/transform/cbor/miner/state.go | 153 +++++++++++++++ pkg/transform/cbor/router.go | 131 +------------ 27 files changed, 1132 insertions(+), 603 deletions(-) create mode 100644 pkg/extract/actors/generic/types.go rename pkg/transform/cbor/{ => miner}/cbor_gen.go (72%) create mode 100644 pkg/transform/cbor/miner/state.go diff --git a/go.mod b/go.mod index 8aa62969c..44fed0f42 100644 --- a/go.mod +++ b/go.mod @@ -72,6 +72,8 @@ require ( github.com/multiformats/go-varint v0.0.6 go.opentelemetry.io/otel/trace v1.7.0 go.uber.org/atomic v1.10.0 + gorm.io/driver/sqlite v1.4.3 + gorm.io/gorm v1.24.2 ) require ( @@ -219,6 +221,7 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jessevdk/go-flags v1.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect @@ -259,6 +262,7 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect diff --git a/go.sum b/go.sum index 5cebb0093..ef5e87299 100644 --- a/go.sum +++ b/go.sum @@ -864,6 +864,9 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1249,6 +1252,8 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= +github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -2337,6 +2342,11 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= +gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= +gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.2 h1:9wR6CFD+G8nOusLdvkZelOEhpJVwwHzpQOUM+REd6U0= +gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 396ce66dd..573772e94 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -5,12 +5,12 @@ import ( "fmt" "strconv" "sync" + "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" network2 "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/state" @@ -21,13 +21,13 @@ import ( "github.com/filecoin-project/lotus/node/impl/full" "github.com/filecoin-project/lotus/node/impl/net" "github.com/filecoin-project/specs-actors/actors/util/adt" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/go-pg/pg/v10" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" + "go.uber.org/zap" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -44,7 +44,6 @@ import ( "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract/procesor" - "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -170,27 +169,32 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf return nil, err } + start := time.Now() changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) - return nil, nil - bs := blockstore.NewMemorySync() - store := adt2.WrapBlockStore(ctx, bs) - if _, err := cbor.ProcessActors(ctx, store, changes); err != nil { - return false, err + if err != nil { + return nil, err } - /* + log.Infow("Process Actor State Changes Complete", "duration", time.Since(start), zap.Inline(changes)) - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) - if err != nil { - return nil, err + /* + bs := blockstore.NewMemorySync() + store := adt2.WrapBlockStore(ctx, bs) + if _, err := cbor.ProcessActors(ctx, store, changes); err != nil { + return false, err } - ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) - if err != nil { - return nil, err - } + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } + + ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return nil, err + } - success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) + success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) */ diff --git a/pkg/core/array.go b/pkg/core/array.go index e5054ef29..89ca1ed41 100644 --- a/pkg/core/array.go +++ b/pkg/core/array.go @@ -10,60 +10,122 @@ import ( "github.com/filecoin-project/lily/chain/actors/adt/diff" ) -type ArrayDiff struct { - Added []*ArrayChange - Modified []*ArrayModification - Removed []*ArrayChange -} - -func (m *ArrayDiff) Size() int { - return len(m.Added) + len(m.Removed) + len(m.Modified) -} - type ArrayModification struct { Key uint64 - Previous typegen.Deferred - Current typegen.Deferred + Type ChangeType + Previous *typegen.Deferred + Current *typegen.Deferred } -type ArrayChange struct { - Key uint64 - Value typegen.Deferred -} +type ArrayModifications []*ArrayModification -func DiffArray(ctx context.Context, store adt.Store, child, parent adt.Array, childBw, parentBw int) (*ArrayDiff, error) { - // TODO handle different bitwidth +func DiffArray(ctx context.Context, store adt.Store, child, parent adt.Array, childBw, parentBw int) (ArrayModifications, error) { if childBw != parentBw { - panic("here") + diffContainer := &GenericArrayDiff{ + Added: []*ArrayModification{}, + Modified: []*ArrayModification{}, + Removed: []*ArrayModification{}, + } + if err := diff.CompareArray(child, parent, diffContainer); err != nil { + return nil, err + } + return diffContainer.AsArrayModifications() } changes, err := diff.Amt(ctx, parent, child, store, store, amt.UseTreeBitWidth(uint(childBw))) if err != nil { return nil, err } - out := &ArrayDiff{ - Added: make([]*ArrayChange, 0, len(changes)), - Modified: make([]*ArrayModification, 0, len(changes)), - Removed: make([]*ArrayChange, 0, len(changes)), + out := make(ArrayModifications, len(changes)) + for i, change := range changes { + out[i] = &ArrayModification{ + Key: change.Key, + Type: amtChangeTypeToGeneric(change.Type), + Previous: change.Before, + Current: change.After, + } } - for _, change := range changes { - switch change.Type { - case amt.Add: - out.Added = append(out.Added, &ArrayChange{ - Key: change.Key, - Value: *change.After, - }) - case amt.Remove: - out.Removed = append(out.Removed, &ArrayChange{ - Key: change.Key, - Value: *change.Before, - }) - case amt.Modify: - out.Modified = append(out.Modified, &ArrayModification{ - Key: change.Key, - Previous: *change.Before, - Current: *change.After, - }) + return out, nil +} + +func amtChangeTypeToGeneric(c amt.ChangeType) ChangeType { + switch c { + case amt.Add: + return ChangeTypeAdd + case amt.Remove: + return ChangeTypeRemove + case amt.Modify: + return ChangeTypeModify + } + panic("developer error") +} + +type GenericArrayDiff struct { + Added ArrayModifications + Modified ArrayModifications + Removed ArrayModifications +} + +func (t *GenericArrayDiff) AsArrayModifications() (ArrayModifications, error) { + out := make(ArrayModifications, len(t.Added)+len(t.Removed)+len(t.Modified)) + idx := 0 + for _, change := range t.Added { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeAdd, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Removed { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeRemove, + Previous: change.Previous, + Current: change.Current, } + idx++ + } + for _, change := range t.Modified { + out[idx] = &ArrayModification{ + Key: change.Key, + Type: ChangeTypeModify, + Previous: change.Previous, + Current: change.Current, + } + idx++ } return out, nil } + +var _ diff.ArrayDiffer = &GenericArrayDiff{} + +func (t *GenericArrayDiff) Add(key uint64, val *typegen.Deferred) error { + t.Added = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeAdd, + Previous: nil, + Current: val, + }) + return nil +} + +func (t *GenericArrayDiff) Modify(key uint64, from, to *typegen.Deferred) error { + t.Modified = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeModify, + Previous: from, + Current: to, + }) + return nil +} + +func (t *GenericArrayDiff) Remove(key uint64, val *typegen.Deferred) error { + t.Removed = append(t.Added, &ArrayModification{ + Key: key, + Type: ChangeTypeRemove, + Previous: val, + Current: nil, + }) + return nil +} diff --git a/pkg/core/map.go b/pkg/core/map.go index 664103010..fb9d1e52d 100644 --- a/pkg/core/map.go +++ b/pkg/core/map.go @@ -4,66 +4,140 @@ import ( "context" "github.com/filecoin-project/go-hamt-ipld/v3" + "github.com/filecoin-project/go-state-types/abi" typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/adt/diff" ) -type MapDiff struct { - Added []*MapChange - Modified []*MapModification - Removed []*MapChange -} - -func (m *MapDiff) Size() int { - return len(m.Added) + len(m.Removed) + len(m.Modified) -} - type MapModification struct { - Key string - Previous typegen.Deferred - Current typegen.Deferred + Key []byte + Type ChangeType + Previous *typegen.Deferred + Current *typegen.Deferred } -type MapChange struct { - Key string - Value typegen.Deferred -} +type MapModifications []*MapModification -func DiffMap(ctx context.Context, store adt.Store, child, parent adt.Map, childOpts, parentOpts *adt.MapOpts) (*MapDiff, error) { - // TODO handle different bitwidth and handFunctions +func DiffMap(ctx context.Context, store adt.Store, child, parent adt.Map, childOpts, parentOpts *adt.MapOpts) (MapModifications, error) { if !childOpts.Equal(parentOpts) { - panic("here") + diffContainer := &GenericMapDiff{ + Added: []*MapModification{}, + Modified: []*MapModification{}, + Removed: []*MapModification{}, + } + if err := diff.CompareMap(child, parent, diffContainer); err != nil { + return nil, err + } + return diffContainer.AsMapModifications() } changes, err := diff.Hamt(ctx, parent, child, store, store, hamt.UseHashFunction(hamt.HashFunc(childOpts.HashFunc)), hamt.UseTreeBitWidth(childOpts.Bitwidth)) if err != nil { return nil, err } - out := &MapDiff{ - Added: make([]*MapChange, 0, len(changes)), - Modified: make([]*MapModification, 0, len(changes)), - Removed: make([]*MapChange, 0, len(changes)), + out := make(MapModifications, len(changes)) + for i, change := range changes { + out[i] = &MapModification{ + Key: []byte(change.Key), + Type: hamtChangeTypeToGeneric(change.Type), + Previous: change.Before, + Current: change.After, + } + } + return out, nil +} + +func hamtChangeTypeToGeneric(c hamt.ChangeType) ChangeType { + switch c { + case hamt.Add: + return ChangeTypeAdd + case hamt.Remove: + return ChangeTypeRemove + case hamt.Modify: + return ChangeTypeModify + } + panic("developer error") +} + +type GenericMapDiff struct { + Added MapModifications + Modified MapModifications + Removed MapModifications +} + +func (t *GenericMapDiff) AsMapModifications() (MapModifications, error) { + out := make(MapModifications, len(t.Added)+len(t.Removed)+len(t.Modified)) + idx := 0 + for _, change := range t.Added { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeAdd, + Previous: change.Previous, + Current: change.Current, + } + idx++ + } + for _, change := range t.Removed { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeRemove, + Previous: change.Previous, + Current: change.Current, + } + idx++ } - for _, change := range changes { - switch change.Type { - case hamt.Add: - out.Added = append(out.Added, &MapChange{ - Key: change.Key, - Value: *change.After, - }) - case hamt.Remove: - out.Removed = append(out.Removed, &MapChange{ - Key: change.Key, - Value: *change.Before, - }) - case hamt.Modify: - out.Modified = append(out.Modified, &MapModification{ - Key: change.Key, - Previous: *change.Before, - Current: *change.After, - }) + for _, change := range t.Modified { + out[idx] = &MapModification{ + Key: change.Key, + Type: ChangeTypeModify, + Previous: change.Previous, + Current: change.Current, } + idx++ } return out, nil } + +var _ diff.MapDiffer = &GenericMapDiff{} + +// An adt.Map key that just preserves the underlying string. +type StringKey string + +func (k StringKey) Key() string { + return string(k) +} + +func (t *GenericMapDiff) AsKey(key string) (abi.Keyer, error) { + return StringKey(key), nil +} + +func (t *GenericMapDiff) Add(key string, val *typegen.Deferred) error { + t.Added = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeAdd, + Previous: nil, + Current: val, + }) + return nil +} + +func (t *GenericMapDiff) Modify(key string, from, to *typegen.Deferred) error { + t.Modified = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeModify, + Previous: from, + Current: to, + }) + return nil +} + +func (t *GenericMapDiff) Remove(key string, val *typegen.Deferred) error { + t.Removed = append(t.Added, &MapModification{ + Key: []byte(key), + Type: ChangeTypeRemove, + Previous: val, + Current: nil, + }) + return nil +} diff --git a/pkg/extract/actors/generic/array.go b/pkg/extract/actors/generic/array.go index e735d8810..75b706b6c 100644 --- a/pkg/extract/actors/generic/array.go +++ b/pkg/extract/actors/generic/array.go @@ -5,25 +5,38 @@ import ( typegen "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) -type ActorStateArrayLoader = func(interface{}) (adt.Array, int, error) - -type ArrayChange struct { - Key uint64 - Value typegen.Deferred - Change core.ChangeType -} +func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (core.ArrayModifications, error) { + if act.Type == core.ChangeTypeRemove { + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } -type ArrayChangeList = []*MapChange + executedArray, _, err := actorArrayLoader(executedActor) + if err != nil { + return nil, err + } -func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (*core.ArrayDiff, error) { - if act.Type == core.ChangeTypeRemove { - return nil, nil + var out core.ArrayModifications + var v typegen.Deferred + if err := executedArray.ForEach(&v, func(key int64) error { + value := v + out = append(out, &core.ArrayModification{ + Key: uint64(key), + Type: core.ChangeTypeRemove, + Previous: &value, + Current: nil, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil } currentActor, err := actorStateLoader(api.Store(), act.Current) @@ -36,17 +49,15 @@ func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.Actor return nil, err } if act.Type == core.ChangeTypeAdd { - out := &core.ArrayDiff{ - Added: make([]*core.ArrayChange, 0), - Modified: make([]*core.ArrayModification, 0), - Removed: make([]*core.ArrayChange, 0), - } + var out core.ArrayModifications var v typegen.Deferred if err := currentArray.ForEach(&v, func(key int64) error { - out.Added = append(out.Added, &core.ArrayChange{ - // TODO this type is inconsistent in specs-actors.. - Key: uint64(key), - Value: v, + value := v + out = append(out, &core.ArrayModification{ + Key: uint64(key), + Type: core.ChangeTypeAdd, + Previous: nil, + Current: &value, }) return nil }); err != nil { diff --git a/pkg/extract/actors/generic/map.go b/pkg/extract/actors/generic/map.go index 4e5e77277..96c0d1537 100644 --- a/pkg/extract/actors/generic/map.go +++ b/pkg/extract/actors/generic/map.go @@ -3,29 +3,40 @@ package generic import ( "context" - "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) -type ActorStateLoader = func(adt.Store, *types.Actor) (interface{}, error) -type ActorStateMapLoader = func(interface{}) (adt.Map, *adt.MapOpts, error) - -type MapChange struct { - Key string - Value typegen.Deferred - Change core.ChangeType -} +func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (core.MapModifications, error) { + if act.Type == core.ChangeTypeRemove { + executedActor, err := actorStateLoader(api.Store(), act.Executed) + if err != nil { + return nil, err + } -type MapChangeList = []*MapChange + executedMap, _, err := actorMapLoader(executedActor) + if err != nil { + return nil, err + } -func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (*core.MapDiff, error) { - if act.Type == core.ChangeTypeRemove { - return nil, nil + var out core.MapModifications + var v typegen.Deferred + if err := executedMap.ForEach(&v, func(key string) error { + value := v + out = append(out, &core.MapModification{ + Key: []byte(key), + Type: core.ChangeTypeRemove, + Previous: &value, + Current: nil, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil } currentActor, err := actorStateLoader(api.Store(), act.Current) @@ -37,23 +48,24 @@ func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.ActorCh if err != nil { return nil, err } + if act.Type == core.ChangeTypeAdd { - out := &core.MapDiff{ - Added: make([]*core.MapChange, 0), - Modified: make([]*core.MapModification, 0), - Removed: make([]*core.MapChange, 0), - } + var out core.MapModifications var v typegen.Deferred if err := currentMap.ForEach(&v, func(key string) error { - out.Added = append(out.Added, &core.MapChange{ - Key: key, - Value: v, + value := v + out = append(out, &core.MapModification{ + Key: []byte(key), + Type: core.ChangeTypeAdd, + Previous: nil, + Current: &value, }) return nil }); err != nil { return nil, err } return out, nil + } executedActor, err := actorStateLoader(api.Store(), act.Executed) diff --git a/pkg/extract/actors/generic/types.go b/pkg/extract/actors/generic/types.go new file mode 100644 index 000000000..5414dc6a7 --- /dev/null +++ b/pkg/extract/actors/generic/types.go @@ -0,0 +1,14 @@ +package generic + +import ( + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + + "github.com/filecoin-project/lily/chain/actors/adt" +) + +var log = logging.Logger("lily/extract/diff") + +type ActorStateArrayLoader = func(interface{}) (adt.Array, int, error) +type ActorStateLoader = func(adt.Store, *types.Actor) (interface{}, error) +type ActorStateMapLoader = func(interface{}) (adt.Map, *adt.MapOpts, error) diff --git a/pkg/extract/actors/minerdiff/cbor_gen.go b/pkg/extract/actors/minerdiff/cbor_gen.go index acf8bcb3d..8499d73ec 100644 --- a/pkg/extract/actors/minerdiff/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/cbor_gen.go @@ -192,23 +192,63 @@ func (t *PreCommitChange) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{162}); err != nil { + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorNumber ([]uint8) (slice) + if len("sector_number") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_number\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_number"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_number")); err != nil { + return err + } + + if len(t.SectorNumber) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.SectorNumber was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.SectorNumber))); err != nil { + return err + } + + if _, err := cw.Write(t.SectorNumber[:]); err != nil { return err } - // t.PreCommit (typegen.Deferred) (struct) - if len("pre_commit") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"pre_commit\" was too long") + // t.Current (typegen.Deferred) (struct) + if len("current_pre_commit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_pre_commit\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commit"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_pre_commit"))); err != nil { return err } - if _, err := io.WriteString(w, string("pre_commit")); err != nil { + if _, err := io.WriteString(w, string("current_pre_commit")); err != nil { return err } - if err := t.PreCommit.MarshalCBOR(cw); err != nil { + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous_pre_commit") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_pre_commit\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_pre_commit"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_pre_commit")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { return err } @@ -268,12 +308,47 @@ func (t *PreCommitChange) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.PreCommit (typegen.Deferred) (struct) - case "pre_commit": + // t.SectorNumber ([]uint8) (slice) + case "sector_number": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.SectorNumber: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.SectorNumber = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.SectorNumber[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current_pre_commit": { - if err := t.PreCommit.UnmarshalCBOR(cr); err != nil { + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_pre_commit": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("failed to read deferred field: %w", err) } } @@ -308,23 +383,55 @@ func (t *SectorChange) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{162}); err != nil { + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorNumber (uint64) (uint64) + if len("sector_number") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_number\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_number"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_number")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.SectorNumber)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current_sector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_sector\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_sector"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_sector")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { return err } - // t.Sector (typegen.Deferred) (struct) - if len("sector") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sector\" was too long") + // t.Previous (typegen.Deferred) (struct) + if len("previous_sector") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_sector\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_sector"))); err != nil { return err } - if _, err := io.WriteString(w, string("sector")); err != nil { + if _, err := io.WriteString(w, string("previous_sector")); err != nil { return err } - if err := t.Sector.MarshalCBOR(cw); err != nil { + if err := t.Previous.MarshalCBOR(cw); err != nil { return err } @@ -384,12 +491,40 @@ func (t *SectorChange) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.Sector (typegen.Deferred) (struct) - case "sector": + // t.SectorNumber (uint64) (uint64) + case "sector_number": { - if err := t.Sector.UnmarshalCBOR(cr); err != nil { + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.SectorNumber = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current_sector": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_sector": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("failed to read deferred field: %w", err) } } diff --git a/pkg/extract/actors/minerdiff/debt.go b/pkg/extract/actors/minerdiff/debt.go index 12eb4ae2e..eda0f38e7 100644 --- a/pkg/extract/actors/minerdiff/debt.go +++ b/pkg/extract/actors/minerdiff/debt.go @@ -2,8 +2,10 @@ package minerdiff import ( "context" + "time" "github.com/filecoin-project/go-state-types/abi" + "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -28,6 +30,10 @@ var _ actors.ActorDiffer = (*Debt)(nil) type Debt struct{} func (Debt) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerDebt, zap.Inline(act), "duration", time.Since(start)) + }() return DebtDiff(ctx, api, act) } diff --git a/pkg/extract/actors/minerdiff/funds.go b/pkg/extract/actors/minerdiff/funds.go index e8385e21d..3fa98f135 100644 --- a/pkg/extract/actors/minerdiff/funds.go +++ b/pkg/extract/actors/minerdiff/funds.go @@ -3,8 +3,10 @@ package minerdiff import ( "context" _ "embed" + "time" "github.com/filecoin-project/go-state-types/abi" + "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/pkg/core" @@ -32,6 +34,10 @@ var _ actors.ActorDiffer = (*Funds)(nil) type Funds struct{} func (Funds) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerFunds, zap.Inline(act), "duration", time.Since(start)) + }() return FundsDiff(ctx, api, act) } diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/infos.go index d5620376c..d52cda1c9 100644 --- a/pkg/extract/actors/minerdiff/infos.go +++ b/pkg/extract/actors/minerdiff/infos.go @@ -2,11 +2,13 @@ package minerdiff import ( "context" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -33,6 +35,10 @@ var _ actors.ActorDiffer = (*Info)(nil) type Info struct{} func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerInfo, zap.Inline(act), "duration", time.Since(start)) + }() return InfoDiff(ctx, api, act) } diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/precommits.go index 340e4a040..1b1060fc9 100644 --- a/pkg/extract/actors/minerdiff/precommits.go +++ b/pkg/extract/actors/minerdiff/precommits.go @@ -2,8 +2,10 @@ package minerdiff import ( "context" + "time" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -14,9 +16,10 @@ import ( var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) type PreCommitChange struct { - // TODO include sector ID - PreCommit typegen.Deferred `cborgen:"pre_commit"` - Change core.ChangeType `cborgen:"change"` + SectorNumber []byte `cborgen:"sector_number"` + Current *typegen.Deferred `cborgen:"current_pre_commit"` + Previous *typegen.Deferred `cborgen:"previous_pre_commit"` + Change core.ChangeType `cborgen:"change"` } type PreCommitChangeList []*PreCommitChange @@ -30,6 +33,10 @@ func (p PreCommitChangeList) Kind() actors.ActorStateKind { type PreCommit struct{} func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerPreCommit, zap.Inline(act), "duration", time.Since(start)) + }() return PreCommitDiff(ctx, api, act) } @@ -38,29 +45,14 @@ func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorC if err != nil { return nil, err } - out := make(PreCommitChangeList, mapChange.Size()) - idx := 0 - for _, change := range mapChange.Added { - out[idx] = &PreCommitChange{ - PreCommit: change.Value, - Change: core.ChangeTypeAdd, + out := make(PreCommitChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &PreCommitChange{ + SectorNumber: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, } - idx++ - } - for _, change := range mapChange.Removed { - out[idx] = &PreCommitChange{ - PreCommit: change.Value, - Change: core.ChangeTypeRemove, - } - idx++ - } - // NB: PreCommits cannot be modified, but check anyway. - for _, change := range mapChange.Modified { - out[idx] = &PreCommitChange{ - PreCommit: change.Current, - Change: core.ChangeTypeModify, - } - idx++ } return out, nil } diff --git a/pkg/extract/actors/minerdiff/sector_status.go b/pkg/extract/actors/minerdiff/sector_status.go index 228a03d6b..482596fd1 100644 --- a/pkg/extract/actors/minerdiff/sector_status.go +++ b/pkg/extract/actors/minerdiff/sector_status.go @@ -3,8 +3,10 @@ package minerdiff import ( "context" "fmt" + "time" "github.com/filecoin-project/go-bitfield" + "go.uber.org/zap" "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -34,6 +36,10 @@ func (s SectorStatusChange) Kind() actors.ActorStateKind { type SectorStatus struct{} func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerSectorStatus, zap.Inline(act), "duration", time.Since(start)) + }() child, err := api.MinerLoad(api.Store(), act.Current) if err != nil { return nil, err diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/sectors.go index 0bc8ee4ea..8ef59f4aa 100644 --- a/pkg/extract/actors/minerdiff/sectors.go +++ b/pkg/extract/actors/minerdiff/sectors.go @@ -2,8 +2,10 @@ package minerdiff import ( "context" + "time" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -14,9 +16,10 @@ import ( var _ actors.ActorStateChange = (*SectorChangeList)(nil) type SectorChange struct { - // TODO include sectorID key - Sector typegen.Deferred `cborgen:"sector"` - Change core.ChangeType `cborgen:"change"` + SectorNumber uint64 `cborgen:"sector_number"` + Current *typegen.Deferred `cborgen:"current_sector"` + Previous *typegen.Deferred `cborgen:"previous_sector"` + Change core.ChangeType `cborgen:"change"` } type SectorChangeList []*SectorChange @@ -30,6 +33,10 @@ func (s SectorChangeList) Kind() actors.ActorStateKind { type Sectors struct{} func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMinerSector, zap.Inline(act), "duration", time.Since(start)) + }() return DiffSectors(ctx, api, act) } @@ -38,28 +45,14 @@ func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorCha if err != nil { return nil, err } - idx := 0 - out := make(SectorChangeList, arrayChange.Size()) - for _, change := range arrayChange.Added { - out[idx] = &SectorChange{ - Sector: change.Value, - Change: core.ChangeTypeAdd, + out := make(SectorChangeList, len(arrayChange)) + for i, change := range arrayChange { + out[i] = &SectorChange{ + SectorNumber: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, } - idx++ - } - for _, change := range arrayChange.Removed { - out[idx] = &SectorChange{ - Sector: change.Value, - Change: core.ChangeTypeRemove, - } - idx++ - } - for _, change := range arrayChange.Modified { - out[idx] = &SectorChange{ - Sector: change.Current, - Change: core.ChangeTypeModify, - } - idx++ } return out, nil } diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go index fd8c431d8..0197e2037 100644 --- a/pkg/extract/actors/minerdiff/state.go +++ b/pkg/extract/actors/minerdiff/state.go @@ -3,10 +3,15 @@ package minerdiff import ( "context" + logging "github.com/ipfs/go-log/v2" + "golang.org/x/sync/errgroup" + "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) +var log = logging.Logger("extract/actors/miner") + type StateDiff struct { InfoChange *InfoChange FundsChange *FundsChange @@ -16,17 +21,38 @@ type StateDiff struct { SectorStatusChanges *SectorStatusChange } -func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { - var stateDiff = new(StateDiff) +func (s *StateDiff) Kind() string { + return "miner" +} + +func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (actors.ActorStateDiff, error) { + grp, grpCtx := errgroup.WithContext(ctx) + results := make(chan actors.ActorStateChange, len(diffFns)) + for _, f := range diffFns { - // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } - if stateChange == nil { - continue + f := f + grp.Go(func() error { + stateChange, err := f.Diff(grpCtx, api, act) + if err != nil { + return err + } + + // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. + if stateChange != nil { + results <- stateChange + } + return nil + }) + } + + go func() { + if err := grp.Wait(); err != nil { + log.Error(err) } + close(results) + }() + var stateDiff = new(StateDiff) + for stateChange := range results { switch stateChange.Kind() { case KindMinerInfo: stateDiff.InfoChange = stateChange.(*InfoChange) @@ -42,5 +68,6 @@ func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, d stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) } } + return stateDiff, nil } diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index 68ae2f3c4..d3a375094 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -5,7 +5,10 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + "github.com/filecoin-project/lily/chain/actors/builtin" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/tasks" ) @@ -27,4 +30,23 @@ type ActorChange struct { Type core.ChangeType } +func (a ActorChange) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("address", a.Address.String()), + attribute.String("type", builtin.ActorNameByCode(a.Current.Code)), + attribute.String("change", a.Type.String()), + } +} + +func (a ActorChange) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + type ActorChanges []*ActorChange + +type ActorStateDiff interface { + Kind() string +} diff --git a/pkg/extract/actors/verifregdiff/claims.go b/pkg/extract/actors/verifregdiff/claims.go index bdb859122..6399c4d58 100644 --- a/pkg/extract/actors/verifregdiff/claims.go +++ b/pkg/extract/actors/verifregdiff/claims.go @@ -2,10 +2,12 @@ package verifregdiff import ( "context" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" @@ -17,9 +19,10 @@ import ( ) type ClaimsChange struct { - Provider address.Address - ClaimID uint64 - Claim typegen.Deferred + Provider []byte + ClaimID []byte + Current *typegen.Deferred + Previous *typegen.Deferred Change core.ChangeType } @@ -34,6 +37,10 @@ func (c ClaimsChangeList) Kind() actors.ActorStateKind { type Claims struct{} func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregClaims, zap.Inline(act), "duration", time.Since(start)) + }() return DiffClaims(ctx, api, act) } @@ -42,67 +49,30 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan if err != nil { return nil, err } - out := make(ClaimsChangeList, 0) - for _, change := range mapChange.Added { - providerID, err := abi.ParseUIntKey(change.Key) + // map change is keyed on provider address with value adt.Map + out := make(ClaimsChangeList, 0, len(mapChange)) + for _, change := range mapChange { + subMapChanges, err := diffSubMap(ctx, api, act, change.Key) if err != nil { return nil, err } - providerAddress, err := address.NewIDAddress(providerID) - if err != nil { - return nil, err - } - added, err := diffSubMap(ctx, api, act, providerAddress) - if err != nil { - return nil, err - } - if len(added) > 0 { - out = append(out, added...) - } - } - for _, change := range mapChange.Removed { - providerID, err := abi.ParseUIntKey(change.Key) - if err != nil { - return nil, err - } - providerAddress, err := address.NewIDAddress(providerID) - if err != nil { - return nil, err - } - removed, err := diffSubMap(ctx, api, act, providerAddress) - if err != nil { - return nil, err - } - if len(removed) > 0 { - out = append(out, removed...) - } + out = append(out, subMapChanges...) } - for _, change := range mapChange.Modified { - providerID, err := abi.ParseUIntKey(change.Key) + return out, nil +} + +func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerKey []byte) ([]*ClaimsChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + providerID, err := abi.ParseUIntKey(string(providerKey)) if err != nil { - return nil, err + return nil, nil, err } providerAddress, err := address.NewIDAddress(providerID) if err != nil { - return nil, err - } - modified, err := diffSubMap(ctx, api, act, providerAddress) - if err != nil { - return nil, err - } - if len(modified) > 0 { - out = append(out, modified...) + return nil, nil, err } - } - - return out, nil -} - -func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerID address.Address) ([]*ClaimsChange, error) { - subMapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { - providerID := providerID verifregState := i.(verifreg.State) - providerClaimMap, err := verifregState.ClaimMapForProvider(providerID) + providerClaimMap, err := verifregState.ClaimMapForProvider(providerAddress) if err != nil { return nil, nil, err } @@ -114,43 +84,15 @@ func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChan if err != nil { return nil, err } - out := make([]*ClaimsChange, 0, subMapChange.Size()) - for _, subChange := range subMapChange.Added { - claimID, err := abi.ParseUIntKey(subChange.Key) - if err != nil { - return nil, err - } - out = append(out, &ClaimsChange{ - Provider: providerID, - ClaimID: claimID, - Claim: subChange.Value, - Change: core.ChangeTypeAdd, - }) - } - for _, subChange := range subMapChange.Removed { - claimID, err := abi.ParseUIntKey(subChange.Key) - if err != nil { - return nil, err - } + out := make([]*ClaimsChange, 0, len(mapChange)) + for _, change := range mapChange { out = append(out, &ClaimsChange{ - Provider: providerID, - ClaimID: claimID, - Claim: subChange.Value, - Change: core.ChangeTypeRemove, + Provider: providerKey, + ClaimID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, }) } - for _, subChange := range subMapChange.Modified { - claimID, err := abi.ParseUIntKey(subChange.Key) - if err != nil { - return nil, err - } - out = append(out, &ClaimsChange{ - Provider: providerID, - ClaimID: claimID, - Claim: subChange.Current, - Change: core.ChangeTypeModify, - }) - } - return out, nil } diff --git a/pkg/extract/actors/verifregdiff/clients.go b/pkg/extract/actors/verifregdiff/clients.go index 027828651..143f78767 100644 --- a/pkg/extract/actors/verifregdiff/clients.go +++ b/pkg/extract/actors/verifregdiff/clients.go @@ -2,9 +2,10 @@ package verifregdiff import ( "context" + "time" - "github.com/filecoin-project/go-address" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -13,9 +14,10 @@ import ( ) type ClientsChange struct { - Client address.Address - DataCap typegen.Deferred - Change core.ChangeType + Client []byte + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType } type ClientsChangeList []*ClientsChange @@ -29,6 +31,10 @@ func (v ClientsChangeList) Kind() actors.ActorStateKind { type Clients struct{} func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregClients, zap.Inline(act), "duration", time.Since(start)) + }() return DiffClients(ctx, api, act) } @@ -38,46 +44,14 @@ func DiffClients(ctx context.Context, api tasks.DataSource, act *actors.ActorCha return nil, err } - idx := 0 - out := make(ClientsChangeList, mapChange.Size()) - for _, change := range mapChange.Added { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err + out := make(ClientsChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &ClientsChange{ + Client: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, } - out[idx] = &ClientsChange{ - Client: addr, - DataCap: change.Value, - Change: core.ChangeTypeAdd, - } - idx++ - } - for _, change := range mapChange.Removed { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err - } - out[idx] = &ClientsChange{ - Client: addr, - DataCap: change.Value, - Change: core.ChangeTypeRemove, - } - idx++ - } - for _, change := range mapChange.Modified { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err - } - out[idx] = &ClientsChange{ - Client: addr, - DataCap: change.Current, - Change: core.ChangeTypeModify, - } - idx++ } return out, nil diff --git a/pkg/extract/actors/verifregdiff/state.go b/pkg/extract/actors/verifregdiff/state.go index 480d01c68..8bed113a6 100644 --- a/pkg/extract/actors/verifregdiff/state.go +++ b/pkg/extract/actors/verifregdiff/state.go @@ -2,18 +2,28 @@ package verifregdiff import ( "context" + "time" + + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) +var log = logging.Logger("extract/actors/verifreg") + type StateDiff struct { VerifierChanges VerifiersChangeList ClientChanges ClientsChangeList ClaimChanges ClaimsChangeList } -func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (*StateDiff, error) { +func (s *StateDiff) Kind() string { + return "verifreg" +} + +func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (actors.ActorStateDiff, error) { + start := time.Now() var stateDiff = new(StateDiff) for _, f := range diffFns { stateChange, err := f.Diff(ctx, api, act) @@ -32,5 +42,6 @@ func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, d stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) } } + log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/verifregdiff/verifiers.go b/pkg/extract/actors/verifregdiff/verifiers.go index 7dc288a4c..0074cf1c1 100644 --- a/pkg/extract/actors/verifregdiff/verifiers.go +++ b/pkg/extract/actors/verifregdiff/verifiers.go @@ -2,9 +2,10 @@ package verifregdiff import ( "context" + "time" - "github.com/filecoin-project/go-address" typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -13,8 +14,9 @@ import ( ) type VerifiersChange struct { - Verifier address.Address - DataCap typegen.Deferred + Verifier []byte + Current *typegen.Deferred + Previous *typegen.Deferred Change core.ChangeType } @@ -29,6 +31,10 @@ func (v VerifiersChangeList) Kind() actors.ActorStateKind { type Verifiers struct{} func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregVerifiers, zap.Inline(act), "duration", time.Since(start)) + }() return DiffVerifiers(ctx, api, act) } @@ -37,47 +43,14 @@ func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorC if err != nil { return nil, err } - - idx := 0 - out := make(VerifiersChangeList, mapChange.Size()) - for _, change := range mapChange.Added { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err - } - out[idx] = &VerifiersChange{ - Verifier: addr, - DataCap: change.Value, - Change: core.ChangeTypeAdd, - } - idx++ - } - for _, change := range mapChange.Removed { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err - } - out[idx] = &VerifiersChange{ - Verifier: addr, - DataCap: change.Value, - Change: core.ChangeTypeRemove, - } - idx++ - } - for _, change := range mapChange.Modified { - // TODO maybe we don't want to marshal these bytes to the address and leave them as bytes in the change struct - addr, err := address.NewFromBytes([]byte(change.Key)) - if err != nil { - return nil, err - } - out[idx] = &VerifiersChange{ - Verifier: addr, - DataCap: change.Current, - Change: core.ChangeTypeModify, + out := make(VerifiersChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &VerifiersChange{ + Verifier: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, } - idx++ } return out, nil } diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 0503c5ab7..05b40fff4 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -2,10 +2,15 @@ package procesor import ( "context" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/actors/builtin/market" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -18,6 +23,8 @@ import ( "github.com/filecoin-project/lily/tasks" ) +var log = logging.Logger("lily/extract/processor") + var ( MinerCodes = cid.NewSet() PowerCodes = cid.NewSet() @@ -48,6 +55,28 @@ type ActorStateChanges struct { VerifregActor map[address.Address]*verifregdiff.StateDiff } +func (a ActorStateChanges) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.Int64("current", int64(a.Current.Height())), + attribute.Int64("executed", int64(a.Executed.Height())), + attribute.Int("actor_change", len(a.Actors)), + attribute.Int("miner_changes", len(a.MinerActors)), + attribute.Int("verifreg_changes", len(a.VerifregActor)), + } +} + +func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +type StateDiffResult struct { + ActorDiff actors.ActorStateDiff + Address address.Address +} + func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { @@ -61,41 +90,73 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current VerifregActor: make(map[address.Address]*verifregdiff.StateDiff, len(actorChanges)), } + grp, grpCtx := errgroup.WithContext(ctx) + results := make(chan *StateDiffResult, len(actorChanges)) for addr, change := range actorChanges { - if MinerCodes.Has(change.Current.Code) { - minerChanges, err := minerdiff.State(ctx, api, &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - }, - minerdiff.Debt{}, - minerdiff.Funds{}, - minerdiff.Info{}, - minerdiff.PreCommit{}, - minerdiff.Sectors{}, - minerdiff.SectorStatus{}, - ) - if err != nil { - return nil, err + addr := addr + change := change + grp.Go(func() error { + if MinerCodes.Has(change.Current.Code) { + start := time.Now() + minerChanges, err := minerdiff.State(grpCtx, api, &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, + }, + minerdiff.Debt{}, + minerdiff.Funds{}, + minerdiff.Info{}, + minerdiff.PreCommit{}, + minerdiff.Sectors{}, + minerdiff.SectorStatus{}, + ) + if err != nil { + return err + } + log.Infow("Extract Miner", "address", addr, "duration", time.Since(start)) + results <- &StateDiffResult{ + ActorDiff: minerChanges, + Address: addr, + } } - asc.MinerActors[addr] = minerChanges - } - if VerifregCodes.Has(change.Current.Code) { - verifregChanges, err := verifregdiff.State(ctx, api, &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - }, - //verifregdiff.Clients{}, - verifregdiff.Verifiers{}, - verifregdiff.Claims{}, - ) - if err != nil { - return nil, err + if VerifregCodes.Has(change.Current.Code) { + start := time.Now() + verifregChanges, err := verifregdiff.State(grpCtx, api, &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, + }, + // TODO the functions handed to these methods should be based on the epoch of the chain. + //verifregdiff.Clients{}, + verifregdiff.Verifiers{}, + verifregdiff.Claims{}, + ) + if err != nil { + return err + } + log.Infow("Extract VerifiedRegistry", "address", addr, "duration", time.Since(start)) + results <- &StateDiffResult{ + ActorDiff: verifregChanges, + Address: addr, + } } - asc.VerifregActor[addr] = verifregChanges + return nil + }) + } + go func() { + if err := grp.Wait(); err != nil { + log.Error(err) + } + close(results) + }() + for stateDiff := range results { + switch v := stateDiff.ActorDiff.(type) { + case *minerdiff.StateDiff: + asc.MinerActors[stateDiff.Address] = v + case *verifregdiff.StateDiff: + asc.VerifregActor[stateDiff.Address] = v } } return asc, nil diff --git a/pkg/extract/statetree/diff.go b/pkg/extract/statetree/diff.go index 79c297e31..18b0109b9 100644 --- a/pkg/extract/statetree/diff.go +++ b/pkg/extract/statetree/diff.go @@ -5,9 +5,10 @@ import ( "context" "crypto/sha256" "fmt" + "time" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-hamt-ipld/v3" + "github.com/filecoin-project/go-state-types/builtin" "github.com/filecoin-project/lotus/chain/types" states0 "github.com/filecoin-project/specs-actors/actors/states" states2 "github.com/filecoin-project/specs-actors/v2/actors/states" @@ -15,44 +16,51 @@ import ( states4 "github.com/filecoin-project/specs-actors/v4/actors/states" states5 "github.com/filecoin-project/specs-actors/v5/actors/states" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/chain/actors/adt/diff" "github.com/filecoin-project/lily/pkg/core" ) +var log = logging.Logger("lily/extract/statetree") + type ActorDiff struct { Executed *types.Actor Current *types.Actor ChangeType core.ChangeType } -func ActorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet) (map[address.Address]ActorDiff, error) { +func DiffActorStateTree(ctx context.Context, store adt.Store, current, executed *types.TipSet) (core.MapModifications, error) { // we have this special method here to get the HAMT node root required by the faster diffing logic. I hate this. - oldRoot, oldVersion, err := getStateTreeHamtRootCIDAndVersion(ctx, store, executed.ParentState()) + executedMap, executedMapOps, err := getStateTreeHamtRootCIDAndVersion(ctx, store, executed.ParentState()) if err != nil { return nil, err } - newRoot, newVersion, err := getStateTreeHamtRootCIDAndVersion(ctx, store, current.ParentState()) + currentMap, currentMapOps, err := getStateTreeHamtRootCIDAndVersion(ctx, store, current.ParentState()) if err != nil { return nil, err } - if oldVersion != newVersion { - panic("here") + changes, err := core.DiffMap(ctx, store, currentMap, executedMap, currentMapOps, executedMapOps) + if err != nil { + return nil, err } + return changes, nil +} - changes, err := diff.Hamt(ctx, oldRoot, newRoot, store, store, hamt.UseTreeBitWidth(5), hamt.UseHashFunction(func(input []byte) []byte { - res := sha256.Sum256(input) - return res[:] - })) +func ActorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet) (map[address.Address]ActorDiff, error) { + start := time.Now() + defer func() { + log.Infow("Actor Changes", "duration", time.Since(start)) + }() + changes, err := DiffActorStateTree(ctx, store, current, executed) if err != nil { return nil, err } buf := bytes.NewReader(nil) out := map[address.Address]ActorDiff{} for _, change := range changes { - addr, err := address.NewFromBytes([]byte(change.Key)) + addr, err := address.NewFromBytes(change.Key) if err != nil { return nil, fmt.Errorf("address in state tree was not valid: %w", err) } @@ -62,34 +70,34 @@ func ActorChanges(ctx context.Context, store adt.Store, current, executed *types ChangeType: core.ChangeTypeUnknown, } switch change.Type { - case hamt.Add: - ch.ChangeType = core.ChangeTypeAdd - buf.Reset(change.After.Raw) + case core.ChangeTypeAdd: + ch.ChangeType = change.Type + buf.Reset(change.Current.Raw) err = ch.Current.UnmarshalCBOR(buf) buf.Reset(nil) if err != nil { return nil, err } - case hamt.Remove: - ch.ChangeType = core.ChangeTypeRemove - buf.Reset(change.Before.Raw) + case core.ChangeTypeRemove: + ch.ChangeType = change.Type + buf.Reset(change.Previous.Raw) err = ch.Executed.UnmarshalCBOR(buf) buf.Reset(nil) if err != nil { return nil, err } - case hamt.Modify: - ch.ChangeType = core.ChangeTypeModify - buf.Reset(change.Before.Raw) + case core.ChangeTypeModify: + ch.ChangeType = change.Type + buf.Reset(change.Previous.Raw) err = ch.Executed.UnmarshalCBOR(buf) buf.Reset(nil) if err != nil { return nil, err } - buf.Reset(change.After.Raw) + buf.Reset(change.Current.Raw) err = ch.Current.UnmarshalCBOR(buf) buf.Reset(nil) if err != nil { @@ -101,7 +109,7 @@ func ActorChanges(ctx context.Context, store adt.Store, current, executed *types return out, nil } -func getStateTreeHamtRootCIDAndVersion(ctx context.Context, store adt.Store, c cid.Cid) (adt.Map, types.StateTreeVersion, error) { +func getStateTreeHamtRootCIDAndVersion(ctx context.Context, store adt.Store, c cid.Cid) (adt.Map, *adt.MapOpts, error) { var root types.StateRoot // Try loading as a new-style state-tree (version/actors tuple). if err := store.Get(ctx, c, &root); err != nil { @@ -115,38 +123,66 @@ func getStateTreeHamtRootCIDAndVersion(ctx context.Context, store adt.Store, c c var tree *states0.Tree tree, err := states0.LoadTree(store, root.Actors) if err != nil { - return nil, 0, err + return nil, nil, err } - return tree.Map, root.Version, nil + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + case types.StateTreeVersion1: var tree *states2.Tree tree, err := states2.LoadTree(store, root.Actors) if err != nil { - return nil, 0, err + return nil, nil, err } - return tree.Map, root.Version, nil + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + case types.StateTreeVersion2: var tree *states3.Tree tree, err := states3.LoadTree(store, root.Actors) if err != nil { - return nil, 0, err + return nil, nil, err } - return tree.Map, root.Version, nil + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil + case types.StateTreeVersion3: var tree *states4.Tree tree, err := states4.LoadTree(store, root.Actors) if err != nil { - return nil, 0, err + return nil, nil, err } - return tree.Map, root.Version, nil + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil case types.StateTreeVersion4: var tree *states5.Tree tree, err := states5.LoadTree(store, root.Actors) if err != nil { - return nil, 0, err + return nil, nil, err } - return tree.Map, root.Version, nil + return tree.Map, &adt.MapOpts{ + Bitwidth: builtin.DefaultHamtBitwidth, + HashFunc: func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + }}, nil default: - return nil, 0, fmt.Errorf("unsupported state tree version: %d", root.Version) + return nil, nil, fmt.Errorf("unsupported state tree version: %d", root.Version) } } diff --git a/pkg/gen/main.go b/pkg/gen/main.go index b54a6627e..55a877b63 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -4,14 +4,14 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" - "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/miner" ) const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" const minerDiffPkg = "minerdiff" -const actorTransformPath = "pkg/transform/cbor/cbor_gen.go" -const actorTransformPkg = "cbor" +const minerTransformPath = "pkg/transform/miner/cbor_gen.go" +const minerTransformPkg = "miner" func main() { if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, @@ -24,8 +24,9 @@ func main() { panic(err) } - if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, - cbor.MinerStateChange{}, + if err := cbg.WriteMapEncodersToFile(minerTransformPath, minerTransformPkg, + miner.StateChange{}, + miner.Info{}, ); err != nil { panic(err) } diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/miner/cbor_gen.go similarity index 72% rename from pkg/transform/cbor/cbor_gen.go rename to pkg/transform/cbor/miner/cbor_gen.go index 127bdf069..2db8402b0 100644 --- a/pkg/transform/cbor/cbor_gen.go +++ b/pkg/transform/cbor/miner/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package cbor +package miner import ( "fmt" @@ -8,6 +8,7 @@ import ( "math" "sort" + core "github.com/filecoin-project/lily/pkg/core" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -19,7 +20,7 @@ var _ = cid.Undef var _ = math.E var _ = sort.Sort -func (t *MinerStateChange) MarshalCBOR(w io.Writer) error { +func (t *StateChange) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err @@ -164,8 +165,8 @@ func (t *MinerStateChange) MarshalCBOR(w io.Writer) error { return nil } -func (t *MinerStateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = MinerStateChange{} +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} cr := cbg.NewCborReader(r) @@ -184,7 +185,7 @@ func (t *MinerStateChange) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.MaxLength { - return fmt.Errorf("MinerStateChange: map struct too large (%d)", extra) + return fmt.Errorf("StateChange: map struct too large (%d)", extra) } var name string @@ -350,3 +351,123 @@ func (t *MinerStateChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *Info) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Info (cid.Cid) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *Info) UnmarshalCBOR(r io.Reader) (err error) { + *t = Info{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Info: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Info (cid.Cid) (struct) + case "info": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = c + + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/miner/state.go b/pkg/transform/cbor/miner/state.go new file mode 100644 index 000000000..38f1a62f5 --- /dev/null +++ b/pkg/transform/cbor/miner/state.go @@ -0,0 +1,153 @@ +package miner + +import ( + "bytes" + "context" + "fmt" + "sort" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/util" +) + +type StateChange struct { + // Miner is the address of the miner that changes. + Miner address.Address `cborgen:"miner"` + // Funds is the funds that changed for this miner or empty. + Funds *minerdiff.FundsChange `cborgen:"funds"` + // Debt is the debt that changed for this miner or empty. + Debt *minerdiff.DebtChange `cborgen:"debt"` + // SectorStatus is the sectors whose status changed for this miner or empty. + SectorStatus *minerdiff.SectorStatusChange `cborgen:"sector_status"` + // Info is the cid of the miner change info that changed for this miner or empty. + Info *cid.Cid `cborgen:"info"` + // PreCommits is an AMT of the pre commits that changed for this miner or empty. + PreCommits *cid.Cid `cborgen:"pre_commits"` + // Sectors is an AMT of the sectors that changed for this miner or empty. + Sectors *cid.Cid `cborgen:"sectors"` +} + +func HandleChanges(ctx context.Context, store adt.Store, actorHamt *adt.Map, miners map[address.Address]*minerdiff.StateDiff) error { + for addr, change := range miners { + msc, err := ChangeHandler(ctx, store, addr, change) + if err != nil { + return err + } + if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { + return err + } + fmt.Printf("%+v\n", msc) + } + return nil +} + +func ChangeHandler(ctx context.Context, store adt.Store, miner address.Address, change *minerdiff.StateDiff) (*StateChange, error) { + out := &StateChange{ + Miner: miner, + } + if change.FundsChange != nil { + out.Funds = change.FundsChange + } + if change.DebtChange != nil { + out.Debt = change.DebtChange + } + if change.SectorStatusChanges != nil { + out.SectorStatus = change.SectorStatusChanges + } + if change.InfoChange != nil { + c, err := InfoHandler(ctx, store, change.InfoChange) + if err != nil { + return nil, err + } + out.Info = &c + } + if change.SectorChanges != nil && len(change.SectorChanges) > 0 { + c, err := SectorHandler(ctx, store, change.SectorChanges) + if err != nil { + return nil, err + } + out.Sectors = &c + } + if change.PreCommitChanges != nil && len(change.PreCommitChanges) > 0 { + c, err := PreCommitHandler(ctx, store, change.PreCommitChanges) + if err != nil { + return nil, err + } + out.PreCommits = &c + } + return out, nil +} + +func PreCommitHandler(ctx context.Context, store adt.Store, list minerdiff.PreCommitChangeList) (cid.Cid, error) { + // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. + sort.Slice(list, func(i, j int) bool { + cmp := bytes.Compare(list[i].SectorNumber, list[j].SectorNumber) + switch cmp { + case 1: + return true + case -1: + return false + default: + panic("precommit with same ID changed twice in one epoch, not possible") + } + }) + arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for _, l := range list { + if err := arr.AppendContinuous(l); err != nil { + return cid.Undef, err + } + } + return arr.Root() +} + +func SectorHandler(ctx context.Context, store adt.Store, list minerdiff.SectorChangeList) (cid.Cid, error) { + // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. + sort.Slice(list, func(i, j int) bool { + return list[i].SectorNumber < list[j].SectorNumber + }) + arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for _, l := range list { + if err := arr.AppendContinuous(l); err != nil { + return cid.Undef, err + } + } + return arr.Root() +} + +type Info struct { + Info cid.Cid `cborgen:"info"` + Change core.ChangeType `cborgen:"change"` +} + +func InfoHandler(ctx context.Context, store adt.Store, info *minerdiff.InfoChange) (cid.Cid, error) { + // ensure the miner info CID is the same as the CID found on chain. + infoCid, err := util.CidOf(&info.Info) + if err != nil { + return cid.Undef, err + } + infoChange := &Info{ + Info: infoCid, + Change: info.Change, + } + mInfoCid, err := store.Put(ctx, infoCid) + if err != nil { + return cid.Undef, err + } + if !mInfoCid.Equals(infoCid) { + panic("here bad") + } + return store.Put(ctx, infoChange) +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 7041fd99f..1e23af176 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -2,18 +2,11 @@ package cbor import ( "context" - "fmt" - "sort" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" "github.com/filecoin-project/lily/pkg/extract/procesor" - "github.com/filecoin-project/lily/pkg/util" + "github.com/filecoin-project/lily/pkg/transform/cbor/miner" ) type ActorIPLDContainer struct { @@ -21,133 +14,13 @@ type ActorIPLDContainer struct { // HAMT of miner address to some big structure... } -type MinerStateChange struct { - // Miner is the address of the miner that changes. - Miner address.Address `cborgen:"miner"` - // Funds is the funds that changed for this miner or empty. - Funds *minerdiff.FundsChange `cborgen:"funds"` - // Debt is the debt that changed for this miner or empty. - Debt *minerdiff.DebtChange `cborgen:"debt"` - // SectorStatus is the sectors whose status changed for this miner or empty. - SectorStatus *minerdiff.SectorStatusChange `cborgen:"sector_status"` - // Info is the cid of the miner change info that changed for this miner or empty. - Info *cid.Cid `cborgen:"info"` - // PreCommits is an AMT of the pre commits that changed for this miner or empty. - PreCommits *cid.Cid `cborgen:"pre_commits"` - // Sectors is an AMT of the sectors that changed for this miner or empty. - Sectors *cid.Cid `cborgen:"sectors"` -} - func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.ActorStateChanges) (interface{}, error) { m, err := adt.MakeEmptyMap(store, 5 /*TODO*/) if err != nil { return nil, err } - if err := HandleMinerChanges(ctx, store, m, changes.MinerActors); err != nil { + if err := miner.HandleChanges(ctx, store, m, changes.MinerActors); err != nil { return nil, err } return nil, nil } - -func HandleMinerChanges(ctx context.Context, store adt.Store, actorHamt *adt.Map, miners map[address.Address]*minerdiff.StateDiff) error { - for addr, change := range miners { - msc, err := HandleMinerChange(ctx, store, addr, change) - if err != nil { - return err - } - if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { - return err - } - fmt.Printf("%+v\n", msc) - } - return nil -} - -func HandleMinerChange(ctx context.Context, store adt.Store, miner address.Address, change *minerdiff.StateDiff) (*MinerStateChange, error) { - out := &MinerStateChange{ - Miner: miner, - } - if change.FundsChange != nil { - out.Funds = change.FundsChange - } - if change.DebtChange != nil { - out.Debt = change.DebtChange - } - if change.SectorStatusChanges != nil { - out.SectorStatus = change.SectorStatusChanges - } - if change.InfoChange != nil { - c, err := MinerInfoHandler(ctx, store, change.InfoChange) - if err != nil { - return nil, err - } - out.Info = &c - } - if change.SectorChanges != nil && len(change.SectorChanges) > 0 { - c, err := MinerSectorHandler(ctx, store, change.SectorChanges) - if err != nil { - return nil, err - } - out.Sectors = &c - } - if change.PreCommitChanges != nil && len(change.PreCommitChanges) > 0 { - c, err := MinerPreCommitHandler(ctx, store, change.PreCommitChanges) - if err != nil { - return nil, err - } - out.PreCommits = &c - } - return out, nil -} - -func MinerPreCommitHandler(ctx context.Context, store adt.Store, list minerdiff.PreCommitChangeList) (cid.Cid, error) { - // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. - sort.Slice(list, func(i, j int) bool { - return util.MustCidOf(&list[i].PreCommit).KeyString() < util.MustCidOf(&list[j].PreCommit).KeyString() - }) - arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for _, l := range list { - if err := arr.AppendContinuous(l); err != nil { - return cid.Undef, err - } - } - return arr.Root() -} - -func MinerSectorHandler(ctx context.Context, store adt.Store, list minerdiff.SectorChangeList) (cid.Cid, error) { - // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. - sort.Slice(list, func(i, j int) bool { - return util.MustCidOf(&list[i].Sector).KeyString() < util.MustCidOf(&list[j].Sector).KeyString() - }) - arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for _, l := range list { - if err := arr.AppendContinuous(l); err != nil { - return cid.Undef, err - } - } - return arr.Root() -} - -type MinerInfo struct { - Info cid.Cid `cborgen:"info"` - Change core.ChangeType `cborgen:"change"` -} - -func MinerInfoHandler(ctx context.Context, store adt.Store, info *minerdiff.InfoChange) (cid.Cid, error) { - // ensure the miner info CID is the same as the CID found on chain. - infoCid, err := util.CidOf(&info.Info) - if err != nil { - return cid.Undef, err - } - infoChange := &MinerInfo{ - Info: infoCid, - Change: info.Change, - } - return store.Put(ctx, infoChange) -} From 13d76f36e8cb72403bbbc9e63db1dfab5ef79ec2 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 13 Dec 2022 20:15:04 -0800 Subject: [PATCH 10/48] write that fancy state decode method you always wanted --- lens/lily/impl.go | 59 +++--- pkg/extract/actors/minerdiff/debt.go | 9 +- pkg/extract/actors/minerdiff/funds.go | 9 +- pkg/extract/actors/minerdiff/infos.go | 8 +- pkg/extract/actors/minerdiff/sector_status.go | 5 + pkg/gen/main.go | 13 +- pkg/transform/cbor/cbor_gen.go | 184 ++++++++++++++++++ pkg/transform/cbor/miner/cbor_gen.go | 70 ++++--- pkg/transform/cbor/miner/state.go | 142 +++++++++++--- pkg/transform/cbor/router.go | 60 +++++- pkg/transform/timescale/processor.go | 87 +++++++++ 11 files changed, 568 insertions(+), 78 deletions(-) create mode 100644 pkg/transform/cbor/cbor_gen.go create mode 100644 pkg/transform/timescale/processor.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 573772e94..eb10fb002 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -3,9 +3,9 @@ package lily import ( "context" "fmt" + "os" "strconv" "sync" - "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/exitcode" @@ -27,7 +27,6 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" - "go.uber.org/zap" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -43,7 +42,7 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" - "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -159,6 +158,8 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf return nil, err } + _ = taskAPI + currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) if err != nil { return nil, err @@ -169,32 +170,46 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf return nil, err } - start := time.Now() - changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + /* + start := time.Now() + changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + if err != nil { + return nil, err + } + log.Infow("Process Actor State Changes Complete", "duration", time.Since(start), zap.Inline(changes)) + + f, err := os.Create(fmt.Sprintf("./%d_%s.car", currentTs.Height(), executedTs.Height())) + if err != nil { + return nil, err + } + defer f.Close() + if err := cbor.ProcessState(ctx, changes, f); err != nil { + return false, err + } + f.Seek(0, io.SeekStart) + + */ + f, err := os.Open(fmt.Sprintf("./%d_%s.car", currentTs.Height(), executedTs.Height())) if err != nil { return nil, err } - log.Infow("Process Actor State Changes Complete", "duration", time.Since(start), zap.Inline(changes)) - + if err := timescale.Process(ctx, f); err != nil { + return nil, err + } /* - bs := blockstore.NewMemorySync() - store := adt2.WrapBlockStore(ctx, bs) - if _, err := cbor.ProcessActors(ctx, store, changes); err != nil { - return false, err - } - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) - if err != nil { - return nil, err - } + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } - ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) - if err != nil { - return nil, err - } + ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return nil, err + } - success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) + success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) */ diff --git a/pkg/extract/actors/minerdiff/debt.go b/pkg/extract/actors/minerdiff/debt.go index eda0f38e7..a6c2adeba 100644 --- a/pkg/extract/actors/minerdiff/debt.go +++ b/pkg/extract/actors/minerdiff/debt.go @@ -5,6 +5,7 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" @@ -38,9 +39,13 @@ func (Debt) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorCha } func DebtDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // was removed, no change + // was removed, its debt is gone... + // TODO is this correct? Can a miner be removed from the state who has debt? Would it be better to persist its last known debt value? the modified case below will have persisted that. if act.Type == core.ChangeTypeRemove { - return nil, nil + return &DebtChange{ + FeeDebt: big.Zero(), + Change: core.ChangeTypeRemove, + }, nil } currentMiner, err := api.MinerLoad(api.Store(), act.Current) if err != nil { diff --git a/pkg/extract/actors/minerdiff/funds.go b/pkg/extract/actors/minerdiff/funds.go index 3fa98f135..83b307efd 100644 --- a/pkg/extract/actors/minerdiff/funds.go +++ b/pkg/extract/actors/minerdiff/funds.go @@ -6,6 +6,7 @@ import ( "time" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -44,7 +45,13 @@ func (Funds) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorCh func FundsDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { // was removed, no change if act.Type == core.ChangeTypeRemove { - return nil, nil + // TODO is this correct? Can a miner be removed from the state who still has funds? Would it be better to persist its last known funds value? the modified case below will have persisted that. + return &FundsChange{ + VestingFunds: big.Zero(), + InitialPledgeRequirement: big.Zero(), + PreCommitDeposit: big.Zero(), + Change: core.ChangeTypeRemove, + }, nil } currentMiner, err := api.MinerLoad(api.Store(), act.Current) if err != nil { diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/infos.go index d52cda1c9..261a759f0 100644 --- a/pkg/extract/actors/minerdiff/infos.go +++ b/pkg/extract/actors/minerdiff/infos.go @@ -20,8 +20,8 @@ import ( var _ actors.ActorStateChange = (*InfoChange)(nil) type InfoChange struct { - Info typegen.Deferred `cborgen:"info"` - Change core.ChangeType `cborgen:"change"` + Info *typegen.Deferred `cborgen:"info"` + Change core.ChangeType `cborgen:"change"` } const KindMinerInfo = "miner_info" @@ -68,7 +68,7 @@ func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*I // was added, info is new if act.Type == core.ChangeTypeAdd { return &InfoChange{ - Info: typegen.Deferred{Raw: infoBytes}, + Info: &typegen.Deferred{Raw: infoBytes}, Change: core.ChangeTypeAdd, }, nil } @@ -82,7 +82,7 @@ func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*I return nil, nil } return &InfoChange{ - Info: typegen.Deferred{Raw: infoBytes}, + Info: &typegen.Deferred{Raw: infoBytes}, Change: core.ChangeTypeModify, }, nil } diff --git a/pkg/extract/actors/minerdiff/sector_status.go b/pkg/extract/actors/minerdiff/sector_status.go index 482596fd1..c3a95bfa0 100644 --- a/pkg/extract/actors/minerdiff/sector_status.go +++ b/pkg/extract/actors/minerdiff/sector_status.go @@ -10,6 +10,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) @@ -40,6 +41,10 @@ func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors. defer func() { log.Debugw("Diff", "kind", KindMinerSectorStatus, zap.Inline(act), "duration", time.Since(start)) }() + // miner must have current and executed state for its sectors to have status. + if act.Type == core.ChangeTypeAdd || act.Type == core.ChangeTypeRemove { + return nil, nil + } child, err := api.MinerLoad(api.Store(), act.Current) if err != nil { return nil, err diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 55a877b63..acea02ed1 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -4,15 +4,19 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/miner" ) const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" const minerDiffPkg = "minerdiff" -const minerTransformPath = "pkg/transform/miner/cbor_gen.go" +const minerTransformPath = "pkg/transform/cbor/miner/cbor_gen.go" const minerTransformPkg = "miner" +const actorTransformPath = "pkg/transform/cbor/cbor_gen.go" +const actorTransformPkg = "cbor" + func main() { if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, minerdiff.SectorStatusChange{}, @@ -30,4 +34,11 @@ func main() { ); err != nil { panic(err) } + + if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, + cbor.ActorIPLDContainer{}, + ); err != nil { + panic(err) + } + } diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go new file mode 100644 index 000000000..4ec281db8 --- /dev/null +++ b/pkg/transform/cbor/cbor_gen.go @@ -0,0 +1,184 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package cbor + +import ( + "fmt" + "io" + "math" + "sort" + + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.CurrentTipSet (types.TipSet) (struct) + if len("CurrentTipSet") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"CurrentTipSet\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CurrentTipSet"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("CurrentTipSet")); err != nil { + return err + } + + if err := t.CurrentTipSet.MarshalCBOR(cw); err != nil { + return err + } + + // t.ExecutedTipSet (types.TipSet) (struct) + if len("ExecutedTipSet") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"ExecutedTipSet\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ExecutedTipSet"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("ExecutedTipSet")); err != nil { + return err + } + + if err := t.ExecutedTipSet.MarshalCBOR(cw); err != nil { + return err + } + + // t.MinerActors (cid.Cid) (struct) + if len("MinerActors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"MinerActors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MinerActors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("MinerActors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.MinerActors); err != nil { + return xerrors.Errorf("failed to write cid field t.MinerActors: %w", err) + } + + return nil +} + +func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.CurrentTipSet (types.TipSet) (struct) + case "CurrentTipSet": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.CurrentTipSet = new(types.TipSet) + if err := t.CurrentTipSet.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.CurrentTipSet pointer: %w", err) + } + } + + } + // t.ExecutedTipSet (types.TipSet) (struct) + case "ExecutedTipSet": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ExecutedTipSet = new(types.TipSet) + if err := t.ExecutedTipSet.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ExecutedTipSet pointer: %w", err) + } + } + + } + // t.MinerActors (cid.Cid) (struct) + case "MinerActors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MinerActors: %w", err) + } + + t.MinerActors = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/miner/cbor_gen.go b/pkg/transform/cbor/miner/cbor_gen.go index 2db8402b0..a1affaa20 100644 --- a/pkg/transform/cbor/miner/cbor_gen.go +++ b/pkg/transform/cbor/miner/cbor_gen.go @@ -9,7 +9,6 @@ import ( "sort" core "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" @@ -48,7 +47,7 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - // t.Funds (minerdiff.FundsChange) (struct) + // t.Funds (cid.Cid) (struct) if len("funds") > cbg.MaxLength { return xerrors.Errorf("Value in field \"funds\" was too long") } @@ -60,11 +59,17 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := t.Funds.MarshalCBOR(cw); err != nil { - return err + if t.Funds == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Funds); err != nil { + return xerrors.Errorf("failed to write cid field t.Funds: %w", err) + } } - // t.Debt (minerdiff.DebtChange) (struct) + // t.Debt (cid.Cid) (struct) if len("debt") > cbg.MaxLength { return xerrors.Errorf("Value in field \"debt\" was too long") } @@ -76,11 +81,17 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := t.Debt.MarshalCBOR(cw); err != nil { - return err + if t.Debt == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Debt); err != nil { + return xerrors.Errorf("failed to write cid field t.Debt: %w", err) + } } - // t.SectorStatus (minerdiff.SectorStatusChange) (struct) + // t.SectorStatus (cid.Cid) (struct) if len("sector_status") > cbg.MaxLength { return xerrors.Errorf("Value in field \"sector_status\" was too long") } @@ -92,8 +103,14 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := t.SectorStatus.MarshalCBOR(cw); err != nil { - return err + if t.SectorStatus == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { + return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) + } } // t.Info (cid.Cid) (struct) @@ -213,7 +230,7 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Funds (minerdiff.FundsChange) (struct) + // t.Funds (cid.Cid) (struct) case "funds": { @@ -226,14 +243,17 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Funds = new(minerdiff.FundsChange) - if err := t.Funds.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Funds pointer: %w", err) + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Funds: %w", err) } + + t.Funds = &c } } - // t.Debt (minerdiff.DebtChange) (struct) + // t.Debt (cid.Cid) (struct) case "debt": { @@ -246,14 +266,17 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Debt = new(minerdiff.DebtChange) - if err := t.Debt.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Debt pointer: %w", err) + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Debt: %w", err) } + + t.Debt = &c } } - // t.SectorStatus (minerdiff.SectorStatusChange) (struct) + // t.SectorStatus (cid.Cid) (struct) case "sector_status": { @@ -266,10 +289,13 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.SectorStatus = new(minerdiff.SectorStatusChange) - if err := t.SectorStatus.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.SectorStatus pointer: %w", err) + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) } + + t.SectorStatus = &c } } diff --git a/pkg/transform/cbor/miner/state.go b/pkg/transform/cbor/miner/state.go index 38f1a62f5..6b895634b 100644 --- a/pkg/transform/cbor/miner/state.go +++ b/pkg/transform/cbor/miner/state.go @@ -3,7 +3,6 @@ package miner import ( "bytes" "context" - "fmt" "sort" "github.com/filecoin-project/go-address" @@ -12,20 +11,88 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "gorm.io/driver/sqlite" + "gorm.io/gorm" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" - "github.com/filecoin-project/lily/pkg/util" ) +var DB *gorm.DB + +func init() { + var err error + DB, err = gorm.Open(sqlite.Open("test.db"), &gorm.Config{}) + if err != nil { + panic(err) + } + if err := DB.AutoMigrate(&StateChangeModel{}); err != nil { + panic(err) + } +} + +type StateChangeModel struct { + gorm.Model + Miner string + Funds string + Debt string + SectorStatus string + Info string + PreCommits string + Sectors string +} + +func (sc *StateChange) ToStorage(db *gorm.DB) error { + var ( + funds = "" + debt = "" + status = "" + info = "" + precommit = "" + sector = "" + ) + if sc.Funds != nil { + funds = sc.Funds.String() + } + if sc.Debt != nil { + debt = sc.Debt.String() + } + if sc.SectorStatus != nil { + status = sc.SectorStatus.String() + } + if sc.Info != nil { + info = sc.Info.String() + } + if sc.PreCommits != nil { + precommit = sc.PreCommits.String() + } + if sc.Sectors != nil { + sector = sc.Sectors.String() + } + tx := db.Create(&StateChangeModel{ + Miner: sc.Miner.String(), + Funds: funds, + Debt: debt, + SectorStatus: status, + Info: info, + PreCommits: precommit, + Sectors: sector, + }) + if tx.Error != nil { + return tx.Error + } + return nil +} + type StateChange struct { // Miner is the address of the miner that changes. Miner address.Address `cborgen:"miner"` // Funds is the funds that changed for this miner or empty. - Funds *minerdiff.FundsChange `cborgen:"funds"` + Funds *cid.Cid `cborgen:"funds"` // Debt is the debt that changed for this miner or empty. - Debt *minerdiff.DebtChange `cborgen:"debt"` + Debt *cid.Cid `cborgen:"debt"` // SectorStatus is the sectors whose status changed for this miner or empty. - SectorStatus *minerdiff.SectorStatusChange `cborgen:"sector_status"` + SectorStatus *cid.Cid `cborgen:"sector_status"` // Info is the cid of the miner change info that changed for this miner or empty. Info *cid.Cid `cborgen:"info"` // PreCommits is an AMT of the pre commits that changed for this miner or empty. @@ -40,28 +107,48 @@ func HandleChanges(ctx context.Context, store adt.Store, actorHamt *adt.Map, min if err != nil { return err } + if msc == nil { + continue + } + if err := msc.ToStorage(DB); err != nil { + return err + } if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { return err } - fmt.Printf("%+v\n", msc) } return nil } func ChangeHandler(ctx context.Context, store adt.Store, miner address.Address, change *minerdiff.StateDiff) (*StateChange, error) { - out := &StateChange{ - Miner: miner, - } + hasChangedState := false + out := &StateChange{} if change.FundsChange != nil { - out.Funds = change.FundsChange + hasChangedState = true + c, err := FundsHandler(ctx, store, change.FundsChange) + if err != nil { + return nil, err + } + out.Funds = &c } if change.DebtChange != nil { - out.Debt = change.DebtChange + hasChangedState = true + c, err := DebtHandler(ctx, store, change.DebtChange) + if err != nil { + return nil, err + } + out.Debt = &c } if change.SectorStatusChanges != nil { - out.SectorStatus = change.SectorStatusChanges + hasChangedState = true + c, err := SectorStatusHandler(ctx, store, change.SectorStatusChanges) + if err != nil { + return nil, err + } + out.SectorStatus = &c } if change.InfoChange != nil { + hasChangedState = true c, err := InfoHandler(ctx, store, change.InfoChange) if err != nil { return nil, err @@ -69,6 +156,7 @@ func ChangeHandler(ctx context.Context, store adt.Store, miner address.Address, out.Info = &c } if change.SectorChanges != nil && len(change.SectorChanges) > 0 { + hasChangedState = true c, err := SectorHandler(ctx, store, change.SectorChanges) if err != nil { return nil, err @@ -76,13 +164,30 @@ func ChangeHandler(ctx context.Context, store adt.Store, miner address.Address, out.Sectors = &c } if change.PreCommitChanges != nil && len(change.PreCommitChanges) > 0 { + hasChangedState = true c, err := PreCommitHandler(ctx, store, change.PreCommitChanges) if err != nil { return nil, err } out.PreCommits = &c } - return out, nil + if hasChangedState { + out.Miner = miner + return out, nil + } + return nil, nil +} + +func FundsHandler(ctx context.Context, store adt.Store, funds *minerdiff.FundsChange) (cid.Cid, error) { + return store.Put(ctx, funds) +} + +func DebtHandler(ctx context.Context, store adt.Store, debt *minerdiff.DebtChange) (cid.Cid, error) { + return store.Put(ctx, debt) +} + +func SectorStatusHandler(ctx context.Context, store adt.Store, sectors *minerdiff.SectorStatusChange) (cid.Cid, error) { + return store.Put(ctx, sectors) } func PreCommitHandler(ctx context.Context, store adt.Store, list minerdiff.PreCommitChangeList) (cid.Cid, error) { @@ -134,20 +239,13 @@ type Info struct { func InfoHandler(ctx context.Context, store adt.Store, info *minerdiff.InfoChange) (cid.Cid, error) { // ensure the miner info CID is the same as the CID found on chain. - infoCid, err := util.CidOf(&info.Info) + mInfoCid, err := store.Put(ctx, info.Info) if err != nil { return cid.Undef, err } infoChange := &Info{ - Info: infoCid, + Info: mInfoCid, Change: info.Change, } - mInfoCid, err := store.Put(ctx, infoCid) - if err != nil { - return cid.Undef, err - } - if !mInfoCid.Equals(infoCid) { - panic("here bad") - } return store.Put(ctx, infoChange) } diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 1e23af176..27d47d9d3 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -2,19 +2,62 @@ package cbor import ( "context" + "fmt" + "io" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + "github.com/ipld/go-car/util" "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/pkg/transform/cbor/miner" + + v1car "github.com/ipld/go-car" ) type ActorIPLDContainer struct { - // HAMT of actor addresses to their changes - // HAMT of miner address to some big structure... + CurrentTipSet *types.TipSet + ExecutedTipSet *types.TipSet + MinerActors cid.Cid // HAMT[Address]MinerStateChange +} + +func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { + bs := blockstore.NewMemorySync() + store := adt2.WrapBlockStore(ctx, bs) + actorStates, err := ProcessActors(ctx, store, changes) + if err != nil { + return err + } + actorStatesRoot, err := store.Put(ctx, actorStates) + if err != nil { + return err + } + if err := v1car.WriteHeader(&v1car.CarHeader{ + Roots: []cid.Cid{actorStatesRoot}, + Version: 1, + }, w); err != nil { + return err + } + keyCh, err := bs.AllKeysChan(ctx) + if err != nil { + return err + } + for key := range keyCh { + blk, err := bs.Get(ctx, key) + if err != nil { + return err + } + if err := util.LdWrite(w, blk.Cid().Bytes(), blk.RawData()); err != nil { + return err + } + } + return nil } -func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.ActorStateChanges) (interface{}, error) { +func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.ActorStateChanges) (*ActorIPLDContainer, error) { m, err := adt.MakeEmptyMap(store, 5 /*TODO*/) if err != nil { return nil, err @@ -22,5 +65,14 @@ func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.Actor if err := miner.HandleChanges(ctx, store, m, changes.MinerActors); err != nil { return nil, err } - return nil, nil + minerActorsRoot, err := m.Root() + if err != nil { + return nil, err + } + fmt.Println("MINER HAMT NODE") + return &ActorIPLDContainer{ + CurrentTipSet: changes.Current, + ExecutedTipSet: changes.Executed, + MinerActors: minerActorsRoot, + }, nil } diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go new file mode 100644 index 000000000..035ae2743 --- /dev/null +++ b/pkg/transform/timescale/processor.go @@ -0,0 +1,87 @@ +package timescale + +import ( + "bytes" + "context" + "fmt" + "io" + "reflect" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/transform/cbor" + cborminer "github.com/filecoin-project/lily/pkg/transform/cbor/miner" +) + +func Process(ctx context.Context, r io.Reader) error { + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + store := store.WrapBlockStore(ctx, bs) + var actorIPLDContainer cbor.ActorIPLDContainer + if err := store.Get(ctx, header.Roots[0], &actorIPLDContainer); err != nil { + return err + } + fmt.Println("miner hamt", actorIPLDContainer.MinerActors.String()) + minerMap, err := adt.AsMap(store, actorIPLDContainer.MinerActors, 5) + if err != nil { + return err + } + var minerState cborminer.StateChange + if err := minerMap.ForEach(&minerState, func(key string) error { + minerAddr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + fmt.Println(minerAddr.String()) + if minerState.Info != nil { + HandleMinerInfo(ctx, store, *minerState.Info) + } + return nil + }); err != nil { + return err + } + return nil +} + +func HandleMinerInfo(ctx context.Context, store adt.Store, info cid.Cid) (interface{}, error) { + var minerInfoChange cborminer.Info + if err := store.Get(ctx, info, &minerInfoChange); err != nil { + return nil, err + } + StateReader(ctx, store, minerInfoChange.Info, func(in *miner9.MinerInfo) error { + fmt.Printf("%+v", in) + return nil + }) + return nil, nil +} + +func StateReader(ctx context.Context, store adt.Store, c cid.Cid, fn interface{}) error { + var tmp typegen.Deferred + if err := store.Get(ctx, c, &tmp); err != nil { + return err + } + fnArg := reflect.TypeOf(fn).In(0) + fmt.Println(fnArg.String()) + if fnArg.Implements(reflect.TypeOf((*typegen.CBORUnmarshaler)(nil)).Elem()) { + p := reflect.New(fnArg.Elem()).Interface().(typegen.CBORUnmarshaler) + if err := p.UnmarshalCBOR(bytes.NewReader(tmp.Raw)); err != nil { + return err + } + results := reflect.ValueOf(fn).Call([]reflect.Value{reflect.ValueOf(p)}) + fmt.Println(results[0].Interface()) + } + return nil +} From 52506174fd7c9ceed3561225bdb5fa6e0f3ccef3 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Dec 2022 16:39:21 -0800 Subject: [PATCH 11/48] can convert car file of cbor into timescale models --- lens/lily/impl.go | 21 +-- pkg/core/reader.go | 39 +++++ pkg/transform/timescale/miner/router.go | 152 ++++++++++++++++++ pkg/transform/timescale/miner/v8/sectors.go | 23 +++ pkg/transform/timescale/miner/v9/debt.go | 22 +++ pkg/transform/timescale/miner/v9/funds.go | 24 +++ pkg/transform/timescale/miner/v9/info.go | 88 ++++++++++ .../timescale/miner/v9/precommits.go | 62 +++++++ pkg/transform/timescale/miner/v9/processor.go | 73 +++++++++ .../timescale/miner/v9/sector_deals.go | 87 ++++++++++ .../timescale/miner/v9/sector_events.go | 145 +++++++++++++++++ pkg/transform/timescale/miner/v9/sectors.go | 60 +++++++ pkg/transform/timescale/processor.go | 57 ++----- 13 files changed, 798 insertions(+), 55 deletions(-) create mode 100644 pkg/core/reader.go create mode 100644 pkg/transform/timescale/miner/router.go create mode 100644 pkg/transform/timescale/miner/v8/sectors.go create mode 100644 pkg/transform/timescale/miner/v9/debt.go create mode 100644 pkg/transform/timescale/miner/v9/funds.go create mode 100644 pkg/transform/timescale/miner/v9/info.go create mode 100644 pkg/transform/timescale/miner/v9/precommits.go create mode 100644 pkg/transform/timescale/miner/v9/processor.go create mode 100644 pkg/transform/timescale/miner/v9/sector_deals.go create mode 100644 pkg/transform/timescale/miner/v9/sector_events.go create mode 100644 pkg/transform/timescale/miner/v9/sectors.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index eb10fb002..d46d9cd28 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -135,23 +135,18 @@ func (m *LilyNodeAPI) StartTipSetWorker(_ context.Context, cfg *LilyTipSetWorker } func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { - /* - md := storage.Metadata{ - JobName: cfg.JobConfig.Name, - } + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } - */ // the context's passed to these methods live for the duration of the clients request, so make a new one. ctx := context.Background() // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. - /* - strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) - if err != nil { - return nil, err - } - - */ + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return nil, err + } taskAPI, err := datasource.NewDataSource(m) if err != nil { @@ -193,7 +188,7 @@ func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interf if err != nil { return nil, err } - if err := timescale.Process(ctx, f); err != nil { + if err := timescale.Process(ctx, f, strg); err != nil { return nil, err } /* diff --git a/pkg/core/reader.go b/pkg/core/reader.go new file mode 100644 index 000000000..bd67acd12 --- /dev/null +++ b/pkg/core/reader.go @@ -0,0 +1,39 @@ +package core + +import ( + "bytes" + "context" + "fmt" + "reflect" + + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/chain/actors/adt" +) + +func StateReader(ctx context.Context, store adt.Store, c cid.Cid, fn interface{}) error { + raw := new(typegen.Deferred) + if err := store.Get(ctx, c, raw); err != nil { + return err + } + return StateReadDeferred(ctx, raw, fn) +} + +func StateReadDeferred(ctx context.Context, raw *typegen.Deferred, fn interface{}) error { + fnArg := reflect.TypeOf(fn).In(0) + if fnArg.Implements(reflect.TypeOf((*typegen.CBORUnmarshaler)(nil)).Elem()) { + p := reflect.New(fnArg.Elem()).Interface().(typegen.CBORUnmarshaler) + if err := p.UnmarshalCBOR(bytes.NewReader(raw.Raw)); err != nil { + return err + } + results := reflect.ValueOf(fn).Call([]reflect.Value{reflect.ValueOf(p)}) + if results[0].IsNil() { + return nil + } else { + return fmt.Errorf("error: %s", results[0]) + } + } + return nil + +} diff --git a/pkg/transform/timescale/miner/router.go b/pkg/transform/timescale/miner/router.go new file mode 100644 index 000000000..038c371db --- /dev/null +++ b/pkg/transform/timescale/miner/router.go @@ -0,0 +1,152 @@ +package miner + +import ( + "context" + + "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + cborminer "github.com/filecoin-project/lily/pkg/transform/cbor/miner" +) + +func DecodeMinerStateDiff(ctx context.Context, store adt.Store, change cborminer.StateChange) (*minerdiff.StateDiff, error) { + out := &minerdiff.StateDiff{ + PreCommitChanges: minerdiff.PreCommitChangeList{}, + SectorChanges: minerdiff.SectorChangeList{}, + SectorStatusChanges: &minerdiff.SectorStatusChange{ + Removed: bitfield.New(), + Recovering: bitfield.New(), + Faulted: bitfield.New(), + Recovered: bitfield.New(), + }, + } + if c := change.Info; c != nil { + info, err := DecodeMinerInfoChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.InfoChange = info + } + + if c := change.Funds; c != nil { + funds, err := DecodeMinerFundsChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.FundsChange = funds + } + + if c := change.Debt; c != nil { + debt, err := DecodeMinerDebtChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.DebtChange = debt + } + + if c := change.SectorStatus; c != nil { + sectorStatus, err := DecodeMinerSectorStatusChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.SectorStatusChanges = sectorStatus + } + + if c := change.Sectors; c != nil { + sectors, err := DecodeMinerSectorChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.SectorChanges = sectors + } + + if c := change.PreCommits; c != nil { + precommits, err := DecodeMinerPreCommitChanges(ctx, store, *c) + if err != nil { + return nil, err + } + out.PreCommitChanges = precommits + } + return out, nil +} + +// TODO dubious on usage of pointers in the decode methods. + +func DecodeMinerInfoChanges(ctx context.Context, store adt.Store, info cid.Cid) (*minerdiff.InfoChange, error) { + var infoContainer cborminer.Info + if err := store.Get(ctx, info, &infoContainer); err != nil { + return nil, err + } + var minfo typegen.Deferred + if err := store.Get(ctx, infoContainer.Info, &minfo); err != nil { + return nil, err + } + return &minerdiff.InfoChange{ + // TODO this seems error prone, bad copy + Info: &minfo, + Change: infoContainer.Change, + }, nil +} + +func DecodeMinerFundsChanges(ctx context.Context, store adt.Store, funds cid.Cid) (*minerdiff.FundsChange, error) { + minerFundsChange := new(minerdiff.FundsChange) + if err := store.Get(ctx, funds, minerFundsChange); err != nil { + return nil, err + } + return minerFundsChange, nil +} + +func DecodeMinerDebtChanges(ctx context.Context, store adt.Store, debt cid.Cid) (*minerdiff.DebtChange, error) { + var minerDebtChange *minerdiff.DebtChange + if err := store.Get(ctx, debt, minerDebtChange); err != nil { + return nil, err + } + return minerDebtChange, nil +} + +func DecodeMinerSectorStatusChanges(ctx context.Context, store adt.Store, sectorStatus cid.Cid) (*minerdiff.SectorStatusChange, error) { + minerSectorStatusChange := new(minerdiff.SectorStatusChange) + if err := store.Get(ctx, sectorStatus, minerSectorStatusChange); err != nil { + return nil, err + } + return minerSectorStatusChange, nil +} + +func DecodeMinerSectorChanges(ctx context.Context, store adt.Store, sectors cid.Cid) (minerdiff.SectorChangeList, error) { + sectorArr, err := adt.AsArray(store, sectors, 5) + if err != nil { + return nil, err + } + sectorChangeList := make(minerdiff.SectorChangeList, 0, sectorArr.Length()) + sectorChange := new(minerdiff.SectorChange) + if err := sectorArr.ForEach(sectorChange, func(sectorNumber int64) error { + val := new(minerdiff.SectorChange) + *val = *sectorChange + sectorChangeList = append(sectorChangeList, val) + return nil + }); err != nil { + return nil, err + } + return sectorChangeList, nil +} + +func DecodeMinerPreCommitChanges(ctx context.Context, store adt.Store, precommits cid.Cid) (minerdiff.PreCommitChangeList, error) { + precommitArr, err := adt.AsArray(store, precommits, 5) + if err != nil { + return nil, err + } + precommitChangeList := make(minerdiff.PreCommitChangeList, 0, precommitArr.Length()) + precommitChange := new(minerdiff.PreCommitChange) + if err := precommitArr.ForEach(precommitChange, func(sectorNumber int64) error { + val := new(minerdiff.PreCommitChange) + *val = *precommitChange + precommitChangeList = append(precommitChangeList, val) + return nil + }); err != nil { + return nil, err + } + return precommitChangeList, nil +} diff --git a/pkg/transform/timescale/miner/v8/sectors.go b/pkg/transform/timescale/miner/v8/sectors.go new file mode 100644 index 000000000..688b3064d --- /dev/null +++ b/pkg/transform/timescale/miner/v8/sectors.go @@ -0,0 +1,23 @@ +package v8 + +import ( + "context" + + miner8 "github.com/filecoin-project/go-state-types/builtin/v8/miner" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleV8MinerSectorChanges(ctx context.Context, changes []minerdiff.SectorChange) ([]*miner8.SectorOnChainInfo, error) { + var sectors []*miner8.SectorOnChainInfo + for _, change := range changes { + if err := core.StateReadDeferred(ctx, *change.Current, func(sector *miner8.SectorOnChainInfo) error { + sectors = append(sectors, sector) + return nil + }); err != nil { + return nil, err + } + } + return sectors, nil +} diff --git a/pkg/transform/timescale/miner/v9/debt.go b/pkg/transform/timescale/miner/v9/debt.go new file mode 100644 index 000000000..f4533ef26 --- /dev/null +++ b/pkg/transform/timescale/miner/v9/debt.go @@ -0,0 +1,22 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerDebtChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.DebtChange) (model.Persistable, error) { + return &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: addr.String(), + FeeDebt: changes.FeeDebt.String(), + }, nil +} diff --git a/pkg/transform/timescale/miner/v9/funds.go b/pkg/transform/timescale/miner/v9/funds.go new file mode 100644 index 000000000..d08eb8d5e --- /dev/null +++ b/pkg/transform/timescale/miner/v9/funds.go @@ -0,0 +1,24 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerFundsChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.FundsChange) (model.Persistable, error) { + return &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: changes.VestingFunds.String(), + InitialPledge: changes.InitialPledgeRequirement.String(), + PreCommitDeposits: changes.PreCommitDeposit.String(), + }, nil +} diff --git a/pkg/transform/timescale/miner/v9/info.go b/pkg/transform/timescale/miner/v9/info.go new file mode 100644 index 000000000..cfb6a4ea3 --- /dev/null +++ b/pkg/transform/timescale/miner/v9/info.go @@ -0,0 +1,88 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/chain/types" + "github.com/libp2p/go-libp2p/core/peer" + maddr "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/miner" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerInfo(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, info *minerdiff.InfoChange) (model.Persistable, error) { + var out model.Persistable + var err error + if err := core.StateReadDeferred(ctx, info.Info, func(in *miner9.MinerInfo) error { + out, err = MinerInfoAsModel(ctx, current, executed, addr, in) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info *miner9.MinerInfo) (model.Persistable, error) { + return GenericMinerInfoAsModel(ctx, current, executed, addr, *info) +} + +func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner.MinerInfo) (model.Persistable, error) { + var newWorker string + var newWorkerEpoch int64 + if pendingWorkerKey := info.PendingWorkerKey; pendingWorkerKey != nil { + if pendingWorkerKey.NewWorker != address.Undef { + newWorker = pendingWorkerKey.NewWorker.String() + } + newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt) + } + + var newCtrlAddresses []string + for _, addr := range info.ControlAddresses { + newCtrlAddresses = append(newCtrlAddresses, addr.String()) + } + + // best effort to decode, we have no control over what miners put in this field, its just bytes. + var newMultiAddrs []string + for _, addr := range info.Multiaddrs { + newMaddr, err := maddr.NewMultiaddrBytes(addr) + if err == nil { + newMultiAddrs = append(newMultiAddrs, newMaddr.String()) + } else { + //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) + } + } + mi := &minermodel.MinerInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + OwnerID: info.Owner.String(), + WorkerID: info.Worker.String(), + NewWorker: newWorker, + WorkerChangeEpoch: newWorkerEpoch, + ConsensusFaultedElapsed: int64(info.ConsensusFaultElapsed), + ControlAddresses: newCtrlAddresses, + MultiAddresses: newMultiAddrs, + SectorSize: uint64(info.SectorSize), + } + + if info.PeerId != nil { + newPeerID, err := peer.IDFromBytes(info.PeerId) + if err != nil { + //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) + } else { + mi.PeerID = newPeerID.String() + } + } + + return mi, nil +} diff --git a/pkg/transform/timescale/miner/v9/precommits.go b/pkg/transform/timescale/miner/v9/precommits.go new file mode 100644 index 000000000..8c5155b96 --- /dev/null +++ b/pkg/transform/timescale/miner/v9/precommits.go @@ -0,0 +1,62 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerPreCommitChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes minerdiff.PreCommitChangeList) (model.Persistable, error) { + var precommits []*miner9.SectorPreCommitOnChainInfo + for _, change := range changes { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + if err := core.StateReadDeferred(ctx, change.Current, func(precommit *miner9.SectorPreCommitOnChainInfo) error { + precommits = append(precommits, precommit) + return nil + }); err != nil { + return nil, err + } + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner9.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoV9List, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + unSealedCID := "" + if preCommit.Info.UnsealedCid != nil { + unSealedCID = preCommit.Info.UnsealedCid.String() + } + preCommitModel[i] = &minermodel.MinerPreCommitInfoV9{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + DealIDS: deals, + UnsealedCID: unSealedCID, + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/miner/v9/processor.go b/pkg/transform/timescale/miner/v9/processor.go new file mode 100644 index 000000000..be77a6d15 --- /dev/null +++ b/pkg/transform/timescale/miner/v9/processor.go @@ -0,0 +1,73 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func ProcessMinerStateChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiff) (model.PersistableList, error) { + var ( + infoModel model.Persistable + fundsModel model.Persistable + debtModel model.Persistable + err error + ) + if changes.InfoChange != nil { + infoModel, err = HandleMinerInfo(ctx, store, current, executed, addr, changes.InfoChange) + if err != nil { + return nil, err + } + } + + if changes.FundsChange != nil { + fundsModel, err = HandleMinerFundsChange(ctx, store, current, executed, addr, changes.FundsChange) + if err != nil { + return nil, err + } + } + + if changes.DebtChange != nil { + debtModel, err = HandleMinerDebtChange(ctx, store, current, executed, addr, changes.DebtChange) + if err != nil { + return nil, err + } + } + + sectorModel, err := HandleMinerSectorChanges(ctx, store, current, executed, addr, changes.SectorChanges) + if err != nil { + return nil, err + } + + precommitModel, err := HandleMinerPreCommitChanges(ctx, store, current, executed, addr, changes.PreCommitChanges) + if err != nil { + return nil, err + } + + sectorEventModel, err := HandleMinerSectorEvents(ctx, store, current, executed, addr, changes.PreCommitChanges, changes.SectorChanges, changes.SectorStatusChanges) + if err != nil { + return nil, err + } + + sectorDealsModel, err := HandleMinerSectorDeals(ctx, store, current, executed, addr, changes.SectorChanges) + if err != nil { + return nil, err + } + return batchIfNotNil(infoModel, fundsModel, debtModel, sectorModel, precommitModel, sectorEventModel, sectorDealsModel), nil +} + +func batchIfNotNil(models ...model.Persistable) model.PersistableList { + out := make(model.PersistableList, 0, len(models)) + for _, m := range models { + if m == nil { + continue + } + out = append(out, m) + } + return out +} diff --git a/pkg/transform/timescale/miner/v9/sector_deals.go b/pkg/transform/timescale/miner/v9/sector_deals.go new file mode 100644 index 000000000..861d5bbca --- /dev/null +++ b/pkg/transform/timescale/miner/v9/sector_deals.go @@ -0,0 +1,87 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, sectors minerdiff.SectorChangeList) (model.Persistable, error) { + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner9.SectorOnChainInfo) error { + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + return nil + }); err != nil { + return nil, err + } + case core.ChangeTypeModify: + var previousSector *miner9.SectorOnChainInfo + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + var currentSector *miner9.SectorOnChainInfo + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range compareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} + +func compareDealIDs(cur, pre []abi.DealID) []abi.DealID { + var diff []abi.DealID + + // Loop two times, first to find cur dealIDs not in pre, + // second loop to find pre dealIDs not in cur + for i := 0; i < 2; i++ { + for _, s1 := range cur { + found := false + for _, s2 := range pre { + if s1 == s2 { + found = true + break + } + } + // DealID not found. We add it to return slice + if !found { + diff = append(diff, s1) + } + } + // Swap the slices, only if it was the first loop + if i == 0 { + cur, pre = pre, cur + } + } + + return diff +} diff --git a/pkg/transform/timescale/miner/v9/sector_events.go b/pkg/transform/timescale/miner/v9/sector_events.go new file mode 100644 index 000000000..2a25ade30 --- /dev/null +++ b/pkg/transform/timescale/miner/v9/sector_events.go @@ -0,0 +1,145 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, precommits minerdiff.PreCommitChangeList, sectors minerdiff.SectorChangeList, sectorstatus *minerdiff.SectorStatusChange) (model.Persistable, error) { + out := minermodel.MinerSectorEventList{} + height := int64(current.Height()) + minerAddr := addr.String() + stateRoot := current.ParentState().String() + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner9.SectorOnChainInfo) error { + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + return nil + }); err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + var previousSector *miner9.SectorOnChainInfo + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + var currentSector *miner9.SectorOnChainInfo + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorSnapped, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/miner/v9/sectors.go b/pkg/transform/timescale/miner/v9/sectors.go new file mode 100644 index 000000000..0d92f5ade --- /dev/null +++ b/pkg/transform/timescale/miner/v9/sectors.go @@ -0,0 +1,60 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" +) + +func HandleMinerSectorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes minerdiff.SectorChangeList) (model.Persistable, error) { + var sectors []*miner9.SectorOnChainInfo + for _, change := range changes { + // only care about modified and added sectors + if change.Change == core.ChangeTypeRemove { + continue + } + // change.Current is the newly added sector, or its state after modification. + if err := core.StateReadDeferred(ctx, change.Current, func(sector *miner9.SectorOnChainInfo) error { + sectors = append(sectors, sector) + return nil + }); err != nil { + return nil, err + } + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner9.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go index 035ae2743..0b8ff94fd 100644 --- a/pkg/transform/timescale/processor.go +++ b/pkg/transform/timescale/processor.go @@ -1,26 +1,23 @@ package timescale import ( - "bytes" "context" "fmt" "io" - "reflect" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" - "github.com/ipfs/go-cid" v1car "github.com/ipld/go-car" - typegen "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/transform/cbor" cborminer "github.com/filecoin-project/lily/pkg/transform/cbor/miner" + "github.com/filecoin-project/lily/pkg/transform/timescale/miner" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/miner/v9" ) -func Process(ctx context.Context, r io.Reader) error { +func Process(ctx context.Context, r io.Reader, strg model.Storage) error { bs := blockstore.NewMemorySync() header, err := v1car.LoadCar(ctx, bs, r) if err != nil { @@ -29,59 +26,35 @@ func Process(ctx context.Context, r io.Reader) error { if len(header.Roots) != 1 { return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) } + store := store.WrapBlockStore(ctx, bs) + var actorIPLDContainer cbor.ActorIPLDContainer if err := store.Get(ctx, header.Roots[0], &actorIPLDContainer); err != nil { return err } - fmt.Println("miner hamt", actorIPLDContainer.MinerActors.String()) + minerMap, err := adt.AsMap(store, actorIPLDContainer.MinerActors, 5) if err != nil { return err } + var minerState cborminer.StateChange + var minerModels model.PersistableList if err := minerMap.ForEach(&minerState, func(key string) error { - minerAddr, err := address.NewFromBytes([]byte(key)) + stateChange, err := miner.DecodeMinerStateDiff(ctx, store, minerState) if err != nil { return err } - fmt.Println(minerAddr.String()) - if minerState.Info != nil { - HandleMinerInfo(ctx, store, *minerState.Info) + minerModel, err := v9.ProcessMinerStateChanges(ctx, store, actorIPLDContainer.CurrentTipSet, actorIPLDContainer.ExecutedTipSet, minerState.Miner, stateChange) + if err != nil { + return err } + minerModels = append(minerModels, minerModel...) return nil }); err != nil { return err } - return nil -} - -func HandleMinerInfo(ctx context.Context, store adt.Store, info cid.Cid) (interface{}, error) { - var minerInfoChange cborminer.Info - if err := store.Get(ctx, info, &minerInfoChange); err != nil { - return nil, err - } - StateReader(ctx, store, minerInfoChange.Info, func(in *miner9.MinerInfo) error { - fmt.Printf("%+v", in) - return nil - }) - return nil, nil -} -func StateReader(ctx context.Context, store adt.Store, c cid.Cid, fn interface{}) error { - var tmp typegen.Deferred - if err := store.Get(ctx, c, &tmp); err != nil { - return err - } - fnArg := reflect.TypeOf(fn).In(0) - fmt.Println(fnArg.String()) - if fnArg.Implements(reflect.TypeOf((*typegen.CBORUnmarshaler)(nil)).Elem()) { - p := reflect.New(fnArg.Elem()).Interface().(typegen.CBORUnmarshaler) - if err := p.UnmarshalCBOR(bytes.NewReader(tmp.Raw)); err != nil { - return err - } - results := reflect.ValueOf(fn).Call([]reflect.Value{reflect.ValueOf(p)}) - fmt.Println(results[0].Interface()) - } - return nil + return strg.PersistBatch(ctx, minerModels...) } From 45ee51f2e28c293d0a7f59882698398883d65b60 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Dec 2022 17:35:08 -0800 Subject: [PATCH 12/48] add ipld index command --- commands/job/index.go | 45 +++++++++++++ lens/lily/api.go | 9 +++ lens/lily/impl.go | 88 ++++++++++++------------- lens/lily/struct.go | 13 ++-- pkg/transform/cbor/router.go | 2 - pkg/transform/timescale/miner/router.go | 11 ++-- 6 files changed, 109 insertions(+), 59 deletions(-) diff --git a/commands/job/index.go b/commands/job/index.go index 0a36fa877..b57268200 100644 --- a/commands/job/index.go +++ b/commands/job/index.go @@ -3,6 +3,7 @@ package job import ( "fmt" "strings" + "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" @@ -52,6 +53,7 @@ var IndexTipSetCmd = &cli.Command{ Usage: "Index the state of a tipset from the filecoin blockchain by tipset key.", Subcommands: []*cli.Command{ IndexNotifyCmd, + IndexIPLDCmd, }, Flags: []cli.Flag{ &cli.StringFlag{ @@ -107,6 +109,7 @@ var IndexHeightCmd = &cli.Command{ }, Subcommands: []*cli.Command{ IndexNotifyCmd, + IndexIPLDCmd, }, Before: func(cctx *cli.Context) error { ctx := lotuscli.ReqContext(cctx) @@ -150,6 +153,48 @@ var IndexHeightCmd = &cli.Command{ }, } +var IndexIPLDCmd = &cli.Command{ + Name: "ipld", + Usage: "Index the state of a tipset from the filecoin blockchain persisted as a car file.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to persist state to.", + Value: fmt.Sprintf("lily_state_%s.car", time.Now()), + }, + &cli.IntFlag{ + Name: "write-buffer", + Usage: "specify write buffer size", + Value: 1 << 20, + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lotuscli.ReqContext(cctx) + + api, closer, err := commands.GetAPI(ctx) + if err != nil { + return err + } + defer closer() + + cfg := &lily.LilyIndexIPLDConfig{ + JobConfig: RunFlags.ParseJobConfig("index-ipld"), + TipSet: indexFlags.tipsetKey, + Path: cctx.String("filename"), + WriteBufferSize: cctx.Int("write-buffer"), + } + + success, err := api.LilyIndexIPLD(ctx, cfg) + fmt.Println("success: ", success) + if err != nil { + return err + } + + return nil + + }, +} + var IndexNotifyCmd = &cli.Command{ Name: "notify", Usage: "notify the provided queueing system of the tipset to index allowing tipset-workers to perform the indexing.", diff --git a/lens/lily/api.go b/lens/lily/api.go index 2ccf8d47e..655c484a5 100644 --- a/lens/lily/api.go +++ b/lens/lily/api.go @@ -20,6 +20,7 @@ type LilyAPI interface { AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) LilyIndex(ctx context.Context, cfg *LilyIndexConfig) (interface{}, error) + LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) LilyWatch(ctx context.Context, cfg *LilyWatchConfig) (*schedule.JobSubmitResult, error) LilyWalk(ctx context.Context, cfg *LilyWalkConfig) (*schedule.JobSubmitResult, error) LilySurvey(ctx context.Context, cfg *LilySurveyConfig) (*schedule.JobSubmitResult, error) @@ -160,6 +161,14 @@ type LilyIndexConfig struct { TipSet types.TipSetKey } +type LilyIndexIPLDConfig struct { + JobConfig LilyJobConfig + + TipSet types.TipSetKey + Path string + WriteBufferSize int +} + type LilyIndexNotifyConfig struct { IndexConfig LilyIndexConfig diff --git a/lens/lily/impl.go b/lens/lily/impl.go index d46d9cd28..0a1227134 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -42,7 +42,8 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" - "github.com/filecoin-project/lily/pkg/transform/timescale" + "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -134,81 +135,74 @@ func (m *LilyNodeAPI) StartTipSetWorker(_ context.Context, cfg *LilyTipSetWorker return res, nil } -func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { - md := storage.Metadata{ - JobName: cfg.JobConfig.Name, - } - +func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { // the context's passed to these methods live for the duration of the clients request, so make a new one. ctx := context.Background() - // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. - strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + // TODO add buffering + f, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) + defer f.Close() if err != nil { - return nil, err + return false, err } taskAPI, err := datasource.NewDataSource(m) if err != nil { - return nil, err + return false, err } - _ = taskAPI - currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) if err != nil { - return nil, err + return false, err } executedTs, err := m.ChainGetTipSet(ctx, currentTs.Parents()) if err != nil { - return nil, err + return false, err } - /* - start := time.Now() - changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) - if err != nil { - return nil, err - } - log.Infow("Process Actor State Changes Complete", "duration", time.Since(start), zap.Inline(changes)) + changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + if err != nil { + return false, err + } + if err := cbor.ProcessState(ctx, changes, f); err != nil { + return false, err + } + return true, nil +} - f, err := os.Create(fmt.Sprintf("./%d_%s.car", currentTs.Height(), executedTs.Height())) - if err != nil { - return nil, err - } - defer f.Close() - if err := cbor.ProcessState(ctx, changes, f); err != nil { - return false, err - } - f.Seek(0, io.SeekStart) +func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() - */ - f, err := os.Open(fmt.Sprintf("./%d_%s.car", currentTs.Height(), executedTs.Height())) + // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) if err != nil { return nil, err } - if err := timescale.Process(ctx, f, strg); err != nil { + + taskAPI, err := datasource.NewDataSource(m) + if err != nil { return nil, err } - /* - - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) - if err != nil { - return nil, err - } - ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) - if err != nil { - return nil, err - } + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + im, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } - success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) + ts, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return nil, err + } - */ + success, err := im.TipSet(ctx, ts, indexer.WithTasks(cfg.JobConfig.Tasks)) - return true, err + return success, err } func (m *LilyNodeAPI) LilyIndexNotify(_ context.Context, cfg *LilyIndexNotifyConfig) (interface{}, error) { diff --git a/lens/lily/struct.go b/lens/lily/struct.go index 64f497118..19a637daa 100644 --- a/lens/lily/struct.go +++ b/lens/lily/struct.go @@ -27,10 +27,11 @@ type LilyAPIStruct struct { Internal struct { Store func() adt.Store `perm:"read"` - LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` - LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` + LilyIndexIPLD func(ctx context.Context, config *LilyIndexIPLDConfig) (bool, error) `perm:"read"` + LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` LilyIndexNotify func(ctx context.Context, config *LilyIndexNotifyConfig) (interface{}, error) `perm:"read"` LilyWatchNotify func(ctx context.Context, config *LilyWatchNotifyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` @@ -81,6 +82,10 @@ type LilyAPIStruct struct { } } +func (s *LilyAPIStruct) LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { + return s.Internal.LilyIndexIPLD(ctx, cfg) +} + func (s *LilyAPIStruct) StateCompute(ctx context.Context, tsk types.TipSetKey) (interface{}, error) { return s.Internal.StateCompute(ctx, tsk) } diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 27d47d9d3..4a8dd5951 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -2,7 +2,6 @@ package cbor import ( "context" - "fmt" "io" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" @@ -69,7 +68,6 @@ func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.Actor if err != nil { return nil, err } - fmt.Println("MINER HAMT NODE") return &ActorIPLDContainer{ CurrentTipSet: changes.Current, ExecutedTipSet: changes.Executed, diff --git a/pkg/transform/timescale/miner/router.go b/pkg/transform/timescale/miner/router.go index 038c371db..9153e0b5e 100644 --- a/pkg/transform/timescale/miner/router.go +++ b/pkg/transform/timescale/miner/router.go @@ -73,20 +73,17 @@ func DecodeMinerStateDiff(ctx context.Context, store adt.Store, change cborminer return out, nil } -// TODO dubious on usage of pointers in the decode methods. - func DecodeMinerInfoChanges(ctx context.Context, store adt.Store, info cid.Cid) (*minerdiff.InfoChange, error) { var infoContainer cborminer.Info if err := store.Get(ctx, info, &infoContainer); err != nil { return nil, err } - var minfo typegen.Deferred - if err := store.Get(ctx, infoContainer.Info, &minfo); err != nil { + minfo := new(typegen.Deferred) + if err := store.Get(ctx, infoContainer.Info, minfo); err != nil { return nil, err } return &minerdiff.InfoChange{ - // TODO this seems error prone, bad copy - Info: &minfo, + Info: minfo, Change: infoContainer.Change, }, nil } @@ -123,6 +120,7 @@ func DecodeMinerSectorChanges(ctx context.Context, store adt.Store, sectors cid. sectorChangeList := make(minerdiff.SectorChangeList, 0, sectorArr.Length()) sectorChange := new(minerdiff.SectorChange) if err := sectorArr.ForEach(sectorChange, func(sectorNumber int64) error { + // make a copy val := new(minerdiff.SectorChange) *val = *sectorChange sectorChangeList = append(sectorChangeList, val) @@ -141,6 +139,7 @@ func DecodeMinerPreCommitChanges(ctx context.Context, store adt.Store, precommit precommitChangeList := make(minerdiff.PreCommitChangeList, 0, precommitArr.Length()) precommitChange := new(minerdiff.PreCommitChange) if err := precommitArr.ForEach(precommitChange, func(sectorNumber int64) error { + // make a copy val := new(minerdiff.PreCommitChange) *val = *precommitChange precommitChangeList = append(precommitChangeList, val) From eee4ae873302f2989050e5f2ba5cab8f8111445c Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Dec 2022 17:51:19 -0800 Subject: [PATCH 13/48] add command to index car file into timescale --- commands/job/index.go | 38 ++++++++++++++++++- lens/lily/api.go | 7 ++++ lens/lily/impl.go | 28 ++++++++++++++ lens/lily/struct.go | 15 +++++--- .../timescale/miner/v9/sector_deals.go | 4 +- .../timescale/miner/v9/sector_events.go | 4 +- 6 files changed, 86 insertions(+), 10 deletions(-) diff --git a/commands/job/index.go b/commands/job/index.go index b57268200..54edb18e5 100644 --- a/commands/job/index.go +++ b/commands/job/index.go @@ -34,6 +34,7 @@ The index command may be used to index a single tipset from the filecoin blockch Subcommands: []*cli.Command{ IndexTipSetCmd, IndexHeightCmd, + IndexFileCmd, }, Before: func(cctx *cli.Context) error { tasks := RunFlags.Tasks.Value() @@ -48,6 +49,41 @@ The index command may be used to index a single tipset from the filecoin blockch }, } +var IndexFileCmd = &cli.Command{ + Name: "file", + Usage: "Index the state of a car file produced by lily.", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to persist state to.", + Value: fmt.Sprintf("lily_state_%.car", time.Now().Unix()), + }, + }, + Action: func(cctx *cli.Context) error { + ctx := lotuscli.ReqContext(cctx) + + api, closer, err := commands.GetAPI(ctx) + if err != nil { + return err + } + defer closer() + + cfg := &lily.LilyIndexIPLDFileConfig{ + JobConfig: RunFlags.ParseJobConfig("index-file"), + Path: cctx.String("filename"), + } + + success, err := api.LilyIndexIPLDFile(ctx, cfg) + fmt.Println("success: ", success) + if err != nil { + return err + } + + return nil + + }, +} + var IndexTipSetCmd = &cli.Command{ Name: "tipset", Usage: "Index the state of a tipset from the filecoin blockchain by tipset key.", @@ -160,7 +196,7 @@ var IndexIPLDCmd = &cli.Command{ &cli.StringFlag{ Name: "filename", Usage: "Name of the file to persist state to.", - Value: fmt.Sprintf("lily_state_%s.car", time.Now()), + Value: fmt.Sprintf("lily_state_%d.car", time.Now().Unix()), }, &cli.IntFlag{ Name: "write-buffer", diff --git a/lens/lily/api.go b/lens/lily/api.go index 655c484a5..67779bd20 100644 --- a/lens/lily/api.go +++ b/lens/lily/api.go @@ -21,6 +21,7 @@ type LilyAPI interface { LilyIndex(ctx context.Context, cfg *LilyIndexConfig) (interface{}, error) LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) + LilyIndexIPLDFile(ctx context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) LilyWatch(ctx context.Context, cfg *LilyWatchConfig) (*schedule.JobSubmitResult, error) LilyWalk(ctx context.Context, cfg *LilyWalkConfig) (*schedule.JobSubmitResult, error) LilySurvey(ctx context.Context, cfg *LilySurveyConfig) (*schedule.JobSubmitResult, error) @@ -161,6 +162,12 @@ type LilyIndexConfig struct { TipSet types.TipSetKey } +type LilyIndexIPLDFileConfig struct { + JobConfig LilyJobConfig + + Path string +} + type LilyIndexIPLDConfig struct { JobConfig LilyJobConfig diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 0a1227134..67d18e512 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -44,6 +44,7 @@ import ( "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -171,6 +172,33 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return true, nil } +func (m *LilyNodeAPI) LilyIndexIPLDFile(_ context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) { + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } + + // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return false, err + } + + // TODO add buffering + f, err := os.OpenFile(cfg.Path, os.O_RDONLY, 0o644) + defer f.Close() + if err != nil { + return false, err + } + + if err := timescale.Process(ctx, f, strg); err != nil { + return false, err + } + + return true, nil +} + func (m *LilyNodeAPI) LilyIndex(_ context.Context, cfg *LilyIndexConfig) (interface{}, error) { md := storage.Metadata{ JobName: cfg.JobConfig.Name, diff --git a/lens/lily/struct.go b/lens/lily/struct.go index 19a637daa..1f54053af 100644 --- a/lens/lily/struct.go +++ b/lens/lily/struct.go @@ -27,11 +27,12 @@ type LilyAPIStruct struct { Internal struct { Store func() adt.Store `perm:"read"` - LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` - LilyIndexIPLD func(ctx context.Context, config *LilyIndexIPLDConfig) (bool, error) `perm:"read"` - LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` - LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyIndex func(ctx context.Context, config *LilyIndexConfig) (interface{}, error) `perm:"read"` + LilyIndexIPLD func(ctx context.Context, config *LilyIndexIPLDConfig) (bool, error) `perm:"read"` + LilyIndexIPLDFile func(ctx context.Context, config *LilyIndexIPLDFileConfig) (bool, error) `perm:"read"` + LilyWatch func(context.Context, *LilyWatchConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilyWalk func(context.Context, *LilyWalkConfig) (*schedule.JobSubmitResult, error) `perm:"read"` + LilySurvey func(context.Context, *LilySurveyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` LilyIndexNotify func(ctx context.Context, config *LilyIndexNotifyConfig) (interface{}, error) `perm:"read"` LilyWatchNotify func(ctx context.Context, config *LilyWatchNotifyConfig) (*schedule.JobSubmitResult, error) `perm:"read"` @@ -82,6 +83,10 @@ type LilyAPIStruct struct { } } +func (s *LilyAPIStruct) LilyIndexIPLDFile(ctx context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) { + return s.Internal.LilyIndexIPLDFile(ctx, cfg) +} + func (s *LilyAPIStruct) LilyIndexIPLD(ctx context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { return s.Internal.LilyIndexIPLD(ctx, cfg) } diff --git a/pkg/transform/timescale/miner/v9/sector_deals.go b/pkg/transform/timescale/miner/v9/sector_deals.go index 861d5bbca..119dbd402 100644 --- a/pkg/transform/timescale/miner/v9/sector_deals.go +++ b/pkg/transform/timescale/miner/v9/sector_deals.go @@ -37,11 +37,11 @@ func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, execu return nil, err } case core.ChangeTypeModify: - var previousSector *miner9.SectorOnChainInfo + previousSector := new(miner9.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - var currentSector *miner9.SectorOnChainInfo + currentSector := new(miner9.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } diff --git a/pkg/transform/timescale/miner/v9/sector_events.go b/pkg/transform/timescale/miner/v9/sector_events.go index 2a25ade30..44a1bbaa3 100644 --- a/pkg/transform/timescale/miner/v9/sector_events.go +++ b/pkg/transform/timescale/miner/v9/sector_events.go @@ -58,11 +58,11 @@ func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, exec Event: event, }) case core.ChangeTypeModify: - var previousSector *miner9.SectorOnChainInfo + previousSector := new(miner9.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - var currentSector *miner9.SectorOnChainInfo + currentSector := new(miner9.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } From 648028526ed6dcd1aed2db3ad17df87b39c6844a Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 14 Dec 2022 18:04:45 -0800 Subject: [PATCH 14/48] fix miner infor handler --- pkg/core/reader.go | 1 + pkg/transform/timescale/miner/v9/info.go | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/core/reader.go b/pkg/core/reader.go index bd67acd12..19a0f6660 100644 --- a/pkg/core/reader.go +++ b/pkg/core/reader.go @@ -34,6 +34,7 @@ func StateReadDeferred(ctx context.Context, raw *typegen.Deferred, fn interface{ return fmt.Errorf("error: %s", results[0]) } } + panic("here") return nil } diff --git a/pkg/transform/timescale/miner/v9/info.go b/pkg/transform/timescale/miner/v9/info.go index cfb6a4ea3..2afe008ba 100644 --- a/pkg/transform/timescale/miner/v9/info.go +++ b/pkg/transform/timescale/miner/v9/info.go @@ -10,7 +10,6 @@ import ( maddr "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" @@ -21,7 +20,7 @@ func HandleMinerInfo(ctx context.Context, store adt.Store, current, executed *ty var out model.Persistable var err error if err := core.StateReadDeferred(ctx, info.Info, func(in *miner9.MinerInfo) error { - out, err = MinerInfoAsModel(ctx, current, executed, addr, in) + out, err = MinerInfoAsModel(ctx, current, executed, addr, *in) if err != nil { return err } @@ -32,11 +31,11 @@ func HandleMinerInfo(ctx context.Context, store adt.Store, current, executed *ty return out, nil } -func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info *miner9.MinerInfo) (model.Persistable, error) { - return GenericMinerInfoAsModel(ctx, current, executed, addr, *info) +func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner9.MinerInfo) (model.Persistable, error) { + return GenericMinerInfoAsModel(ctx, current, executed, addr, info) } -func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner.MinerInfo) (model.Persistable, error) { +func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner9.MinerInfo) (model.Persistable, error) { var newWorker string var newWorkerEpoch int64 if pendingWorkerKey := info.PendingWorkerKey; pendingWorkerKey != nil { From b523d7e7fe5c8e67e23dcbbb4243a1316ca9cee4 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 15 Dec 2022 20:16:23 -0800 Subject: [PATCH 15/48] large refactor, can still produce car files and persist them to database --- lens/lily/impl.go | 6 +- pkg/core/versions.go | 15 + pkg/extract/actors/minerdiff/state.go | 73 --- .../actors/minerdiff/{ => v0}/cbor_gen.go | 423 ++++++++++++++- pkg/extract/actors/minerdiff/{ => v0}/debt.go | 39 +- .../actors/minerdiff/{ => v0}/funds.go | 39 +- .../actors/minerdiff/{ => v0}/infos.go | 40 +- pkg/extract/actors/minerdiff/{ => v0}/load.go | 2 +- .../actors/minerdiff/{ => v0}/precommits.go | 46 +- .../minerdiff/{ => v0}/sector_status.go | 38 +- .../actors/minerdiff/{ => v0}/sectors.go | 44 +- pkg/extract/actors/minerdiff/v0/state.go | 299 +++++++++++ pkg/extract/actors/minerdiff/v2/debt.go | 6 + pkg/extract/actors/minerdiff/v2/funds.go | 6 + pkg/extract/actors/minerdiff/v2/info.go | 6 + pkg/extract/actors/minerdiff/v2/precommits.go | 7 + .../actors/minerdiff/v2/sector_status.go | 6 + pkg/extract/actors/minerdiff/v2/sectors.go | 7 + pkg/extract/actors/minerdiff/v2/state.go | 7 + pkg/extract/actors/minerdiff/v3/debt.go | 6 + pkg/extract/actors/minerdiff/v3/funds.go | 6 + pkg/extract/actors/minerdiff/v3/info.go | 6 + pkg/extract/actors/minerdiff/v3/precommits.go | 7 + .../actors/minerdiff/v3/sector_status.go | 6 + pkg/extract/actors/minerdiff/v3/sectors.go | 7 + pkg/extract/actors/minerdiff/v3/state.go | 7 + pkg/extract/actors/minerdiff/v4/debt.go | 6 + pkg/extract/actors/minerdiff/v4/funds.go | 6 + pkg/extract/actors/minerdiff/v4/info.go | 6 + pkg/extract/actors/minerdiff/v4/precommits.go | 7 + .../actors/minerdiff/v4/sector_status.go | 6 + pkg/extract/actors/minerdiff/v4/sectors.go | 7 + pkg/extract/actors/minerdiff/v4/state.go | 7 + pkg/extract/actors/minerdiff/v5/debt.go | 6 + pkg/extract/actors/minerdiff/v5/funds.go | 6 + pkg/extract/actors/minerdiff/v5/info.go | 6 + pkg/extract/actors/minerdiff/v5/precommits.go | 7 + .../actors/minerdiff/v5/sector_status.go | 6 + pkg/extract/actors/minerdiff/v5/sectors.go | 7 + pkg/extract/actors/minerdiff/v5/state.go | 7 + pkg/extract/actors/minerdiff/v6/debt.go | 6 + pkg/extract/actors/minerdiff/v6/funds.go | 6 + pkg/extract/actors/minerdiff/v6/info.go | 6 + pkg/extract/actors/minerdiff/v6/precommits.go | 7 + .../actors/minerdiff/v6/sector_status.go | 6 + pkg/extract/actors/minerdiff/v6/sectors.go | 7 + pkg/extract/actors/minerdiff/v6/state.go | 7 + pkg/extract/actors/minerdiff/v7/debt.go | 6 + pkg/extract/actors/minerdiff/v7/funds.go | 6 + pkg/extract/actors/minerdiff/v7/info.go | 6 + pkg/extract/actors/minerdiff/v7/precommits.go | 7 + .../actors/minerdiff/v7/sector_status.go | 6 + pkg/extract/actors/minerdiff/v7/sectors.go | 7 + pkg/extract/actors/minerdiff/v7/state.go | 7 + pkg/extract/actors/minerdiff/v8/debt.go | 6 + pkg/extract/actors/minerdiff/v8/funds.go | 6 + pkg/extract/actors/minerdiff/v8/info.go | 6 + pkg/extract/actors/minerdiff/v8/precommits.go | 7 + .../actors/minerdiff/v8/sector_status.go | 6 + pkg/extract/actors/minerdiff/v8/sectors.go | 7 + pkg/extract/actors/minerdiff/v8/state.go | 7 + pkg/extract/actors/minerdiff/v9/debt.go | 6 + pkg/extract/actors/minerdiff/v9/funds.go | 6 + pkg/extract/actors/minerdiff/v9/info.go | 6 + pkg/extract/actors/minerdiff/v9/precommits.go | 7 + .../actors/minerdiff/v9/sector_status.go | 6 + pkg/extract/actors/minerdiff/v9/sectors.go | 7 + pkg/extract/actors/minerdiff/v9/state.go | 7 + pkg/extract/actors/minerdiff/version.go | 116 ++++ pkg/extract/actors/types.go | 17 +- .../actors/verifregdiff/{ => v0}/clients.go | 7 +- .../actors/verifregdiff/{ => v0}/load.go | 2 +- .../actors/verifregdiff/{ => v0}/state.go | 34 +- .../actors/verifregdiff/{ => v0}/verifiers.go | 3 +- pkg/extract/actors/verifregdiff/v2/clients.go | 7 + pkg/extract/actors/verifregdiff/v2/state.go | 5 + .../actors/verifregdiff/v2/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v3/clients.go | 7 + pkg/extract/actors/verifregdiff/v3/state.go | 5 + .../actors/verifregdiff/v3/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v4/clients.go | 7 + pkg/extract/actors/verifregdiff/v4/state.go | 5 + .../actors/verifregdiff/v4/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v5/clients.go | 7 + pkg/extract/actors/verifregdiff/v5/state.go | 5 + .../actors/verifregdiff/v5/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v6/clients.go | 7 + pkg/extract/actors/verifregdiff/v6/state.go | 5 + .../actors/verifregdiff/v6/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v7/clients.go | 7 + pkg/extract/actors/verifregdiff/v7/state.go | 5 + .../actors/verifregdiff/v7/verifiers.go | 7 + pkg/extract/actors/verifregdiff/v8/clients.go | 7 + pkg/extract/actors/verifregdiff/v8/state.go | 5 + .../actors/verifregdiff/v8/verifiers.go | 9 + .../actors/verifregdiff/v9/allocations.go | 46 ++ .../actors/verifregdiff/{ => v9}/claims.go | 8 +- pkg/extract/actors/verifregdiff/v9/state.go | 60 +++ .../actors/verifregdiff/v9/verifiers.go | 9 + pkg/extract/actors/verifregdiff/version.go | 81 +++ pkg/extract/procesor/process.go | 84 +-- pkg/gen/main.go | 41 +- pkg/transform/cbor/miner/cbor_gen.go | 499 ------------------ pkg/transform/cbor/miner/state.go | 240 +-------- pkg/transform/cbor/router.go | 40 +- pkg/transform/cbor/verifreg/state.go | 16 + .../timescale/actors/miner/router.go | 1 + .../timescale/actors/miner/v0/debt.go | 23 + .../{miner/v9 => actors/miner/v0}/funds.go | 6 +- .../timescale/actors/miner/v0/info.go | 87 +++ .../timescale/actors/miner/v0/precommits.go | 59 +++ .../timescale/actors/miner/v0/processor.go | 70 +++ .../timescale/actors/miner/v0/sector_deals.go | 87 +++ .../actors/miner/v0/sector_events.go | 149 ++++++ .../timescale/actors/miner/v0/sectors.go | 55 ++ .../timescale/actors/miner/v2/processor.go | 69 +++ .../timescale/{ => actors}/miner/v9/debt.go | 12 +- .../timescale/actors/miner/v9/funds.go | 29 + .../timescale/{ => actors}/miner/v9/info.go | 13 +- .../{ => actors}/miner/v9/precommits.go | 9 +- .../timescale/actors/miner/v9/processor.go | 78 +++ .../{ => actors}/miner/v9/sector_deals.go | 8 +- .../{ => actors}/miner/v9/sector_events.go | 20 +- .../{ => actors}/miner/v9/sectors.go | 15 +- pkg/transform/timescale/actors/processor.go | 71 +++ pkg/transform/timescale/actors/types.go | 1 + pkg/transform/timescale/miner/router.go | 151 ------ pkg/transform/timescale/miner/v8/sectors.go | 23 - pkg/transform/timescale/miner/v9/processor.go | 73 --- pkg/transform/timescale/processor.go | 60 --- 130 files changed, 2778 insertions(+), 1253 deletions(-) create mode 100644 pkg/core/versions.go delete mode 100644 pkg/extract/actors/minerdiff/state.go rename pkg/extract/actors/minerdiff/{ => v0}/cbor_gen.go (68%) rename pkg/extract/actors/minerdiff/{ => v0}/debt.go (73%) rename pkg/extract/actors/minerdiff/{ => v0}/funds.go (80%) rename pkg/extract/actors/minerdiff/{ => v0}/infos.go (74%) rename pkg/extract/actors/minerdiff/{ => v0}/load.go (98%) rename pkg/extract/actors/minerdiff/{ => v0}/precommits.go (60%) rename pkg/extract/actors/minerdiff/{ => v0}/sector_status.go (88%) rename pkg/extract/actors/minerdiff/{ => v0}/sectors.go (61%) create mode 100644 pkg/extract/actors/minerdiff/v0/state.go create mode 100644 pkg/extract/actors/minerdiff/v2/debt.go create mode 100644 pkg/extract/actors/minerdiff/v2/funds.go create mode 100644 pkg/extract/actors/minerdiff/v2/info.go create mode 100644 pkg/extract/actors/minerdiff/v2/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v2/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v2/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v2/state.go create mode 100644 pkg/extract/actors/minerdiff/v3/debt.go create mode 100644 pkg/extract/actors/minerdiff/v3/funds.go create mode 100644 pkg/extract/actors/minerdiff/v3/info.go create mode 100644 pkg/extract/actors/minerdiff/v3/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v3/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v3/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v3/state.go create mode 100644 pkg/extract/actors/minerdiff/v4/debt.go create mode 100644 pkg/extract/actors/minerdiff/v4/funds.go create mode 100644 pkg/extract/actors/minerdiff/v4/info.go create mode 100644 pkg/extract/actors/minerdiff/v4/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v4/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v4/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v4/state.go create mode 100644 pkg/extract/actors/minerdiff/v5/debt.go create mode 100644 pkg/extract/actors/minerdiff/v5/funds.go create mode 100644 pkg/extract/actors/minerdiff/v5/info.go create mode 100644 pkg/extract/actors/minerdiff/v5/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v5/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v5/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v5/state.go create mode 100644 pkg/extract/actors/minerdiff/v6/debt.go create mode 100644 pkg/extract/actors/minerdiff/v6/funds.go create mode 100644 pkg/extract/actors/minerdiff/v6/info.go create mode 100644 pkg/extract/actors/minerdiff/v6/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v6/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v6/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v6/state.go create mode 100644 pkg/extract/actors/minerdiff/v7/debt.go create mode 100644 pkg/extract/actors/minerdiff/v7/funds.go create mode 100644 pkg/extract/actors/minerdiff/v7/info.go create mode 100644 pkg/extract/actors/minerdiff/v7/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v7/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v7/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v7/state.go create mode 100644 pkg/extract/actors/minerdiff/v8/debt.go create mode 100644 pkg/extract/actors/minerdiff/v8/funds.go create mode 100644 pkg/extract/actors/minerdiff/v8/info.go create mode 100644 pkg/extract/actors/minerdiff/v8/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v8/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v8/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v8/state.go create mode 100644 pkg/extract/actors/minerdiff/v9/debt.go create mode 100644 pkg/extract/actors/minerdiff/v9/funds.go create mode 100644 pkg/extract/actors/minerdiff/v9/info.go create mode 100644 pkg/extract/actors/minerdiff/v9/precommits.go create mode 100644 pkg/extract/actors/minerdiff/v9/sector_status.go create mode 100644 pkg/extract/actors/minerdiff/v9/sectors.go create mode 100644 pkg/extract/actors/minerdiff/v9/state.go create mode 100644 pkg/extract/actors/minerdiff/version.go rename pkg/extract/actors/verifregdiff/{ => v0}/clients.go (86%) rename pkg/extract/actors/verifregdiff/{ => v0}/load.go (98%) rename pkg/extract/actors/verifregdiff/{ => v0}/state.go (53%) rename pkg/extract/actors/verifregdiff/{ => v0}/verifiers.go (97%) create mode 100644 pkg/extract/actors/verifregdiff/v2/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v2/state.go create mode 100644 pkg/extract/actors/verifregdiff/v2/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v3/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v3/state.go create mode 100644 pkg/extract/actors/verifregdiff/v3/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v4/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v4/state.go create mode 100644 pkg/extract/actors/verifregdiff/v4/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v5/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v5/state.go create mode 100644 pkg/extract/actors/verifregdiff/v5/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v6/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v6/state.go create mode 100644 pkg/extract/actors/verifregdiff/v6/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v7/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v7/state.go create mode 100644 pkg/extract/actors/verifregdiff/v7/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v8/clients.go create mode 100644 pkg/extract/actors/verifregdiff/v8/state.go create mode 100644 pkg/extract/actors/verifregdiff/v8/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/v9/allocations.go rename pkg/extract/actors/verifregdiff/{ => v9}/claims.go (88%) create mode 100644 pkg/extract/actors/verifregdiff/v9/state.go create mode 100644 pkg/extract/actors/verifregdiff/v9/verifiers.go create mode 100644 pkg/extract/actors/verifregdiff/version.go delete mode 100644 pkg/transform/cbor/miner/cbor_gen.go create mode 100644 pkg/transform/cbor/verifreg/state.go create mode 100644 pkg/transform/timescale/actors/miner/router.go create mode 100644 pkg/transform/timescale/actors/miner/v0/debt.go rename pkg/transform/timescale/{miner/v9 => actors/miner/v0}/funds.go (85%) create mode 100644 pkg/transform/timescale/actors/miner/v0/info.go create mode 100644 pkg/transform/timescale/actors/miner/v0/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v0/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v0/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v0/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v0/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v2/processor.go rename pkg/transform/timescale/{ => actors}/miner/v9/debt.go (53%) create mode 100644 pkg/transform/timescale/actors/miner/v9/funds.go rename pkg/transform/timescale/{ => actors}/miner/v9/info.go (88%) rename pkg/transform/timescale/{ => actors}/miner/v9/precommits.go (85%) create mode 100644 pkg/transform/timescale/actors/miner/v9/processor.go rename pkg/transform/timescale/{ => actors}/miner/v9/sector_deals.go (87%) rename pkg/transform/timescale/{ => actors}/miner/v9/sector_events.go (86%) rename pkg/transform/timescale/{ => actors}/miner/v9/sectors.go (80%) create mode 100644 pkg/transform/timescale/actors/processor.go create mode 100644 pkg/transform/timescale/actors/types.go delete mode 100644 pkg/transform/timescale/miner/router.go delete mode 100644 pkg/transform/timescale/miner/v8/sectors.go delete mode 100644 pkg/transform/timescale/miner/v9/processor.go delete mode 100644 pkg/transform/timescale/processor.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 67d18e512..81ae6efad 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -44,7 +44,7 @@ import ( "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/pkg/transform/cbor" - "github.com/filecoin-project/lily/pkg/transform/timescale" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -162,7 +162,7 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } - changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs) + changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) if err != nil { return false, err } @@ -192,7 +192,7 @@ func (m *LilyNodeAPI) LilyIndexIPLDFile(_ context.Context, cfg *LilyIndexIPLDFil return false, err } - if err := timescale.Process(ctx, f, strg); err != nil { + if err := actors.Process(ctx, f, strg, m.StateManager.GetNetworkVersion); err != nil { return false, err } diff --git a/pkg/core/versions.go b/pkg/core/versions.go new file mode 100644 index 000000000..1408db07e --- /dev/null +++ b/pkg/core/versions.go @@ -0,0 +1,15 @@ +package core + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/types" +) + +func ActorVersionForTipSet(ctx context.Context, ts *types.TipSet, ntwkVersionGetter func(ctx context.Context, epoch abi.ChainEpoch) network.Version) (actorstypes.Version, error) { + ntwkVersion := ntwkVersionGetter(ctx, ts.Height()) + return actorstypes.VersionForNetwork(ntwkVersion) +} diff --git a/pkg/extract/actors/minerdiff/state.go b/pkg/extract/actors/minerdiff/state.go deleted file mode 100644 index 0197e2037..000000000 --- a/pkg/extract/actors/minerdiff/state.go +++ /dev/null @@ -1,73 +0,0 @@ -package minerdiff - -import ( - "context" - - logging "github.com/ipfs/go-log/v2" - "golang.org/x/sync/errgroup" - - "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" -) - -var log = logging.Logger("extract/actors/miner") - -type StateDiff struct { - InfoChange *InfoChange - FundsChange *FundsChange - DebtChange *DebtChange - PreCommitChanges PreCommitChangeList - SectorChanges SectorChangeList - SectorStatusChanges *SectorStatusChange -} - -func (s *StateDiff) Kind() string { - return "miner" -} - -func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (actors.ActorStateDiff, error) { - grp, grpCtx := errgroup.WithContext(ctx) - results := make(chan actors.ActorStateChange, len(diffFns)) - - for _, f := range diffFns { - f := f - grp.Go(func() error { - stateChange, err := f.Diff(grpCtx, api, act) - if err != nil { - return err - } - - // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. - if stateChange != nil { - results <- stateChange - } - return nil - }) - } - - go func() { - if err := grp.Wait(); err != nil { - log.Error(err) - } - close(results) - }() - var stateDiff = new(StateDiff) - for stateChange := range results { - switch stateChange.Kind() { - case KindMinerInfo: - stateDiff.InfoChange = stateChange.(*InfoChange) - case KindMinerSector: - stateDiff.SectorChanges = stateChange.(SectorChangeList) - case KindMinerPreCommit: - stateDiff.PreCommitChanges = stateChange.(PreCommitChangeList) - case KindMinerFunds: - stateDiff.FundsChange = stateChange.(*FundsChange) - case KindMinerDebt: - stateDiff.DebtChange = stateChange.(*DebtChange) - case KindMinerSectorStatus: - stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) - } - } - - return stateDiff, nil -} diff --git a/pkg/extract/actors/minerdiff/cbor_gen.go b/pkg/extract/actors/minerdiff/v0/cbor_gen.go similarity index 68% rename from pkg/extract/actors/minerdiff/cbor_gen.go rename to pkg/extract/actors/minerdiff/v0/cbor_gen.go index 8499d73ec..6db935a72 100644 --- a/pkg/extract/actors/minerdiff/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/v0/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package minerdiff +package v0 import ( "fmt" @@ -8,10 +8,11 @@ import ( "math" "sort" - core "github.com/filecoin-project/lily/pkg/core" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" + + core "github.com/filecoin-project/lily/pkg/core" ) var _ = xerrors.Errorf @@ -837,3 +838,421 @@ func (t *DebtChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *InfoChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Info (typegen.Deferred) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if err := t.Info.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *InfoChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = InfoChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("InfoChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Info (typegen.Deferred) (struct) + case "info": + + { + + t.Info = new(cbg.Deferred) + + if err := t.Info.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{166}); err != nil { + return err + } + + // t.Funds (cid.Cid) (struct) + if len("funds") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"funds\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("funds"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("funds")); err != nil { + return err + } + + if t.Funds == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Funds); err != nil { + return xerrors.Errorf("failed to write cid field t.Funds: %w", err) + } + } + + // t.Debt (cid.Cid) (struct) + if len("debt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"debt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("debt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("debt")); err != nil { + return err + } + + if t.Debt == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Debt); err != nil { + return xerrors.Errorf("failed to write cid field t.Debt: %w", err) + } + } + + // t.SectorStatus (cid.Cid) (struct) + if len("sector_status") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_status\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_status")); err != nil { + return err + } + + if t.SectorStatus == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { + return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) + } + } + + // t.Info (cid.Cid) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if t.Info == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + } + + // t.PreCommits (cid.Cid) (struct) + if len("pre_commits") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commits\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commits")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.PreCommits); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) + } + + // t.Sectors (cid.Cid) (struct) + if len("sectors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sectors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sectors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Sectors); err != nil { + return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Funds (cid.Cid) (struct) + case "funds": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Funds: %w", err) + } + + t.Funds = &c + } + + } + // t.Debt (cid.Cid) (struct) + case "debt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Debt: %w", err) + } + + t.Debt = &c + } + + } + // t.SectorStatus (cid.Cid) (struct) + case "sector_status": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) + } + + t.SectorStatus = &c + } + + } + // t.Info (cid.Cid) (struct) + case "info": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = &c + } + + } + // t.PreCommits (cid.Cid) (struct) + case "pre_commits": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) + } + + t.PreCommits = c + + } + // t.Sectors (cid.Cid) (struct) + case "sectors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) + } + + t.Sectors = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/minerdiff/debt.go b/pkg/extract/actors/minerdiff/v0/debt.go similarity index 73% rename from pkg/extract/actors/minerdiff/debt.go rename to pkg/extract/actors/minerdiff/v0/debt.go index a6c2adeba..d02951d28 100644 --- a/pkg/extract/actors/minerdiff/debt.go +++ b/pkg/extract/actors/minerdiff/v0/debt.go @@ -1,11 +1,14 @@ -package minerdiff +package v0 import ( + "bytes" "context" "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "go.uber.org/zap" "github.com/filecoin-project/lily/pkg/core" @@ -20,13 +23,45 @@ type DebtChange struct { Change core.ChangeType `cborgen:"change"` } +func (d *DebtChange) Serialize() ([]byte, error) { + buf := new(bytes.Buffer) + if err := d.MarshalCBOR(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (d *DebtChange) ToStorageBlock() (block.Block, error) { + data, err := d.Serialize() + if err != nil { + return nil, err + } + + c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) + if err != nil { + return nil, err + } + + return block.NewBlockWithCid(data, c) +} + +func DecodeDebt(b []byte) (*DebtChange, error) { + var debt DebtChange + if err := debt.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + return nil, err + } + + return &debt, nil +} + const KindMinerDebt = "miner_debt" func (d *DebtChange) Kind() actors.ActorStateKind { return KindMinerDebt } -var _ actors.ActorDiffer = (*Debt)(nil) +var _ actors.ActorStateDiff = (*Debt)(nil) type Debt struct{} diff --git a/pkg/extract/actors/minerdiff/funds.go b/pkg/extract/actors/minerdiff/v0/funds.go similarity index 80% rename from pkg/extract/actors/minerdiff/funds.go rename to pkg/extract/actors/minerdiff/v0/funds.go index 83b307efd..7eb01165e 100644 --- a/pkg/extract/actors/minerdiff/funds.go +++ b/pkg/extract/actors/minerdiff/v0/funds.go @@ -1,12 +1,15 @@ -package minerdiff +package v0 import ( + "bytes" "context" _ "embed" "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/big" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/miner" @@ -24,13 +27,45 @@ type FundsChange struct { Change core.ChangeType `cborgen:"change"` } +func (f *FundsChange) Serialize() ([]byte, error) { + buf := new(bytes.Buffer) + if err := f.MarshalCBOR(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (f *FundsChange) ToStorageBlock() (block.Block, error) { + data, err := f.Serialize() + if err != nil { + return nil, err + } + + c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) + if err != nil { + return nil, err + } + + return block.NewBlockWithCid(data, c) +} + +func DecodeFunds(b []byte) (*FundsChange, error) { + var funds FundsChange + if err := funds.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + return nil, err + } + + return &funds, nil +} + const KindMinerFunds = "miner_funds" func (f *FundsChange) Kind() actors.ActorStateKind { return KindMinerFunds } -var _ actors.ActorDiffer = (*Funds)(nil) +var _ actors.ActorStateDiff = (*Funds)(nil) type Funds struct{} diff --git a/pkg/extract/actors/minerdiff/infos.go b/pkg/extract/actors/minerdiff/v0/infos.go similarity index 74% rename from pkg/extract/actors/minerdiff/infos.go rename to pkg/extract/actors/minerdiff/v0/infos.go index 261a759f0..422335d6b 100644 --- a/pkg/extract/actors/minerdiff/infos.go +++ b/pkg/extract/actors/minerdiff/v0/infos.go @@ -1,11 +1,14 @@ -package minerdiff +package v0 import ( + "bytes" "context" "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -24,13 +27,46 @@ type InfoChange struct { Change core.ChangeType `cborgen:"change"` } +func (i *InfoChange) Serialize() ([]byte, error) { + buf := new(bytes.Buffer) + if err := i.MarshalCBOR(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil + +} + +func (i *InfoChange) ToStorageBlock() (block.Block, error) { + data, err := i.Serialize() + if err != nil { + return nil, err + } + + c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) + if err != nil { + return nil, err + } + + return block.NewBlockWithCid(data, c) +} + +func DecodeInfo(b []byte) (*InfoChange, error) { + var info InfoChange + if err := info.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + return nil, err + } + + return &info, nil +} + const KindMinerInfo = "miner_info" func (i *InfoChange) Kind() actors.ActorStateKind { return KindMinerInfo } -var _ actors.ActorDiffer = (*Info)(nil) +var _ actors.ActorStateDiff = (*Info)(nil) type Info struct{} diff --git a/pkg/extract/actors/minerdiff/load.go b/pkg/extract/actors/minerdiff/v0/load.go similarity index 98% rename from pkg/extract/actors/minerdiff/load.go rename to pkg/extract/actors/minerdiff/v0/load.go index f804e9d03..974fbed48 100644 --- a/pkg/extract/actors/minerdiff/load.go +++ b/pkg/extract/actors/minerdiff/v0/load.go @@ -1,4 +1,4 @@ -package minerdiff +package v0 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/minerdiff/precommits.go b/pkg/extract/actors/minerdiff/v0/precommits.go similarity index 60% rename from pkg/extract/actors/minerdiff/precommits.go rename to pkg/extract/actors/minerdiff/v0/precommits.go index 1b1060fc9..01867a4f2 100644 --- a/pkg/extract/actors/minerdiff/precommits.go +++ b/pkg/extract/actors/minerdiff/v0/precommits.go @@ -1,12 +1,16 @@ -package minerdiff +package v0 import ( "context" "time" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -15,6 +19,8 @@ import ( var _ actors.ActorStateChange = (*PreCommitChangeList)(nil) +var _ abi.Keyer = (*PreCommitChange)(nil) + type PreCommitChange struct { SectorNumber []byte `cborgen:"sector_number"` Current *typegen.Deferred `cborgen:"current_pre_commit"` @@ -22,8 +28,46 @@ type PreCommitChange struct { Change core.ChangeType `cborgen:"change"` } +func (t *PreCommitChange) Key() string { + return core.StringKey(t.SectorNumber).Key() +} + type PreCommitChangeList []*PreCommitChange +func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +func (p *PreCommitChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { + precommitMap, err := adt.AsMap(store, root, 5) + if err != nil { + return err + } + + precommits := new(PreCommitChangeList) + precommitChange := new(PreCommitChange) + if err := precommitMap.ForEach(precommitChange, func(sectorNumber string) error { + val := new(PreCommitChange) + *val = *precommitChange + *precommits = append(*precommits, val) + return nil + }); err != nil { + return err + } + *p = *precommits + return nil + +} + const KindMinerPreCommit = "miner_precommit" func (p PreCommitChangeList) Kind() actors.ActorStateKind { diff --git a/pkg/extract/actors/minerdiff/sector_status.go b/pkg/extract/actors/minerdiff/v0/sector_status.go similarity index 88% rename from pkg/extract/actors/minerdiff/sector_status.go rename to pkg/extract/actors/minerdiff/v0/sector_status.go index c3a95bfa0..2533b6c7c 100644 --- a/pkg/extract/actors/minerdiff/sector_status.go +++ b/pkg/extract/actors/minerdiff/v0/sector_status.go @@ -1,11 +1,15 @@ -package minerdiff +package v0 import ( + "bytes" "context" "fmt" "time" "github.com/filecoin-project/go-bitfield" + "github.com/filecoin-project/go-state-types/abi" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -28,6 +32,38 @@ type SectorStatusChange struct { Recovered bitfield.BitField `cborgen:"recovered"` } +func (s *SectorStatusChange) Serialize() ([]byte, error) { + buf := new(bytes.Buffer) + if err := s.MarshalCBOR(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (s *SectorStatusChange) ToStorageBlock() (block.Block, error) { + data, err := s.Serialize() + if err != nil { + return nil, err + } + + c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) + if err != nil { + return nil, err + } + + return block.NewBlockWithCid(data, c) +} + +func DecodeSectorStatus(b []byte) (*SectorStatusChange, error) { + var ss SectorStatusChange + if err := ss.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + return nil, err + } + + return &ss, nil +} + const KindMinerSectorStatus = "miner_sector_status" func (s SectorStatusChange) Kind() actors.ActorStateKind { diff --git a/pkg/extract/actors/minerdiff/sectors.go b/pkg/extract/actors/minerdiff/v0/sectors.go similarity index 61% rename from pkg/extract/actors/minerdiff/sectors.go rename to pkg/extract/actors/minerdiff/v0/sectors.go index 8ef59f4aa..342a42d77 100644 --- a/pkg/extract/actors/minerdiff/sectors.go +++ b/pkg/extract/actors/minerdiff/v0/sectors.go @@ -1,9 +1,12 @@ -package minerdiff +package v0 import ( "context" "time" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -15,6 +18,8 @@ import ( var _ actors.ActorStateChange = (*SectorChangeList)(nil) +var _ abi.Keyer = (*SectorChange)(nil) + type SectorChange struct { SectorNumber uint64 `cborgen:"sector_number"` Current *typegen.Deferred `cborgen:"current_sector"` @@ -22,8 +27,45 @@ type SectorChange struct { Change core.ChangeType `cborgen:"change"` } +func (t *SectorChange) Key() string { + return abi.UIntKey(t.SectorNumber).Key() +} + type SectorChangeList []*SectorChange +func (s SectorChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range s { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + +func (s *SectorChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { + sectorMap, err := adt.AsMap(store, root, 5) + if err != nil { + return err + } + + sectors := new(SectorChangeList) + sectorChange := new(SectorChange) + if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { + val := new(SectorChange) + *val = *sectorChange + *sectors = append(*sectors, val) + return nil + }); err != nil { + return err + } + *s = *sectors + return nil +} + const KindMinerSector = "miner_sector" func (s SectorChangeList) Kind() actors.ActorStateKind { diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v0/state.go new file mode 100644 index 000000000..45bc0654d --- /dev/null +++ b/pkg/extract/actors/minerdiff/v0/state.go @@ -0,0 +1,299 @@ +package v0 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/miner") + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + grp, grpCtx := errgroup.WithContext(ctx) + results := make(chan actors.ActorStateChange, len(s.DiffMethods)) + + for _, f := range s.DiffMethods { + f := f + grp.Go(func() error { + stateChange, err := f.Diff(grpCtx, api, act) + if err != nil { + return err + } + + // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. + if stateChange != nil { + results <- stateChange + } + return nil + }) + } + + go func() { + if err := grp.Wait(); err != nil { + log.Error(err) + } + close(results) + }() + var stateDiff = new(StateDiffResult) + for stateChange := range results { + switch stateChange.Kind() { + case KindMinerInfo: + stateDiff.InfoChange = stateChange.(*InfoChange) + case KindMinerSector: + stateDiff.SectorChanges = stateChange.(SectorChangeList) + case KindMinerPreCommit: + stateDiff.PreCommitChanges = stateChange.(PreCommitChangeList) + case KindMinerFunds: + stateDiff.FundsChange = stateChange.(*FundsChange) + case KindMinerDebt: + stateDiff.DebtChange = stateChange.(*DebtChange) + case KindMinerSectorStatus: + stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) + default: + return nil, fmt.Errorf("unknown state change %s", stateChange.Kind()) + } + } + + return stateDiff, nil +} + +type StateDiffResult struct { + InfoChange *InfoChange + FundsChange *FundsChange + DebtChange *DebtChange + PreCommitChanges PreCommitChangeList + SectorChanges SectorChangeList + SectorStatusChanges *SectorStatusChange +} + +func (s *StateDiffResult) Kind() string { + return "miner" +} + +type StateChange struct { + // Funds is the funds that changed for this miner or empty. + Funds *cid.Cid `cborgen:"funds"` + // Debt is the debt that changed for this miner or empty. + Debt *cid.Cid `cborgen:"debt"` + // SectorStatus is the sectors whose status changed for this miner or empty. + SectorStatus *cid.Cid `cborgen:"sector_status"` + // Info is the cid of the miner change info that changed for this miner or empty. + Info *cid.Cid `cborgen:"info"` + // PreCommits is an HAMT of the pre commits that changed for this miner. + PreCommits cid.Cid `cborgen:"pre_commits"` + // Sectors is an HAMT of the sectors that changed for this miner. + Sectors cid.Cid `cborgen:"sectors"` +} + +func (sc *StateChange) StateChangeAsStateDiffResult(ctx context.Context, bs blockstore.Blockstore) (*StateDiffResult, error) { + out := &StateDiffResult{} + adtStore := store.WrapBlockStore(ctx, bs) + + // + // SectorChangeList + { + sectorChangeList := &SectorChangeList{} + if err := sectorChangeList.FromAdtMap(adtStore, sc.Sectors, 5); err != nil { + return nil, err + } + out.SectorChanges = *sectorChangeList + /* + sectorMap, err := adt.AsMap(adtStore, sc.Sectors, 5) + if err != nil { + return nil, err + } + + sectorChangeList := SectorChangeList{} + sectorChange := new(SectorChange) + if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { + val := new(SectorChange) + *val = *sectorChange + sectorChangeList = append(sectorChangeList, val) + return nil + }); err != nil { + return nil, err + } + out.SectorChanges = sectorChangeList + + */ + } + + // + // PrecommitChangeList + { + precommitChangeList := &PreCommitChangeList{} + if err := precommitChangeList.FromAdtMap(adtStore, sc.PreCommits, 5); err != nil { + return nil, err + } + out.PreCommitChanges = *precommitChangeList + /* + preCommitMap, err := adt.AsMap(adtStore, sc.PreCommits, 5) + if err != nil { + return nil, err + } + + preCommitChangeList := PreCommitChangeList{} + preCommitChange := new(PreCommitChange) + if err := preCommitMap.ForEach(preCommitChange, func(sectorNumber string) error { + val := new(PreCommitChange) + *val = *preCommitChange + preCommitChangeList = append(preCommitChangeList, val) + return nil + }); err != nil { + return nil, err + } + out.PreCommitChanges = preCommitChangeList + + */ + } + + // + // Info + { + if sc.Info != nil { + blk, err := bs.Get(ctx, *sc.Info) + if err != nil { + return nil, err + } + info, err := DecodeInfo(blk.RawData()) + if err != nil { + return nil, err + } + out.InfoChange = info + } + } + + // + // Funds + { + if sc.Funds != nil { + blk, err := bs.Get(ctx, *sc.Funds) + if err != nil { + return nil, err + } + funds, err := DecodeFunds(blk.RawData()) + if err != nil { + return nil, err + } + out.FundsChange = funds + } + } + + // + // Debt + { + if sc.Debt != nil { + blk, err := bs.Get(ctx, *sc.Debt) + if err != nil { + return nil, err + } + debt, err := DecodeDebt(blk.RawData()) + if err != nil { + return nil, err + } + out.DebtChange = debt + } + } + + // + // SectorStatus + { + + if sc.SectorStatus != nil { + blk, err := bs.Get(ctx, *sc.SectorStatus) + if err != nil { + return nil, err + } + ss, err := DecodeSectorStatus(blk.RawData()) + if err != nil { + return nil, err + } + out.SectorStatusChanges = ss + } + } + + return out, nil +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) { + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if sectors := sd.SectorChanges; sectors != nil { + root, err := sectors.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Sectors = root + } + + if precommits := sd.PreCommitChanges; precommits != nil { + root, err := precommits.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.PreCommits = root + } + + if info := sd.InfoChange; info != nil { + blk, err := info.ToStorageBlock() + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + c := blk.Cid() + out.Info = &c + } + + if funds := sd.FundsChange; funds != nil { + blk, err := funds.ToStorageBlock() + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + c := blk.Cid() + out.Funds = &c + } + + if debt := sd.DebtChange; debt != nil { + blk, err := debt.ToStorageBlock() + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + c := blk.Cid() + out.Debt = &c + } + + if sectorstatus := sd.SectorStatusChanges; sectorstatus != nil { + blk, err := sectorstatus.ToStorageBlock() + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + c := blk.Cid() + out.SectorStatus = &c + } + return out, nil +} diff --git a/pkg/extract/actors/minerdiff/v2/debt.go b/pkg/extract/actors/minerdiff/v2/debt.go new file mode 100644 index 000000000..bbb3f4e83 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/debt.go @@ -0,0 +1,6 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v2/funds.go b/pkg/extract/actors/minerdiff/v2/funds.go new file mode 100644 index 000000000..81b65096a --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/funds.go @@ -0,0 +1,6 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v2/info.go b/pkg/extract/actors/minerdiff/v2/info.go new file mode 100644 index 000000000..9d76c0ed4 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/info.go @@ -0,0 +1,6 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v2/precommits.go b/pkg/extract/actors/minerdiff/v2/precommits.go new file mode 100644 index 000000000..518943873 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/precommits.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v2/sector_status.go b/pkg/extract/actors/minerdiff/v2/sector_status.go new file mode 100644 index 000000000..1f247e995 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/sector_status.go @@ -0,0 +1,6 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v2/sectors.go b/pkg/extract/actors/minerdiff/v2/sectors.go new file mode 100644 index 000000000..9e76fd2c9 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/sectors.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v2/state.go b/pkg/extract/actors/minerdiff/v2/state.go new file mode 100644 index 000000000..69c74dac0 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v2/state.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v3/debt.go b/pkg/extract/actors/minerdiff/v3/debt.go new file mode 100644 index 000000000..03ba59b58 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/debt.go @@ -0,0 +1,6 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v3/funds.go b/pkg/extract/actors/minerdiff/v3/funds.go new file mode 100644 index 000000000..45fa6c116 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/funds.go @@ -0,0 +1,6 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v3/info.go b/pkg/extract/actors/minerdiff/v3/info.go new file mode 100644 index 000000000..4c0ae4f5f --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/info.go @@ -0,0 +1,6 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v3/precommits.go b/pkg/extract/actors/minerdiff/v3/precommits.go new file mode 100644 index 000000000..eb6127fd6 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/precommits.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v3/sector_status.go b/pkg/extract/actors/minerdiff/v3/sector_status.go new file mode 100644 index 000000000..df36d2bca --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/sector_status.go @@ -0,0 +1,6 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v3/sectors.go b/pkg/extract/actors/minerdiff/v3/sectors.go new file mode 100644 index 000000000..cae0b0584 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/sectors.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v3/state.go b/pkg/extract/actors/minerdiff/v3/state.go new file mode 100644 index 000000000..d3dd5a70b --- /dev/null +++ b/pkg/extract/actors/minerdiff/v3/state.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v4/debt.go b/pkg/extract/actors/minerdiff/v4/debt.go new file mode 100644 index 000000000..8e8999f48 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/debt.go @@ -0,0 +1,6 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v4/funds.go b/pkg/extract/actors/minerdiff/v4/funds.go new file mode 100644 index 000000000..8fc0bb558 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/funds.go @@ -0,0 +1,6 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v4/info.go b/pkg/extract/actors/minerdiff/v4/info.go new file mode 100644 index 000000000..67260a2d8 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/info.go @@ -0,0 +1,6 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v4/precommits.go b/pkg/extract/actors/minerdiff/v4/precommits.go new file mode 100644 index 000000000..f07b79a0f --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/precommits.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v4/sector_status.go b/pkg/extract/actors/minerdiff/v4/sector_status.go new file mode 100644 index 000000000..c62dd8f86 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/sector_status.go @@ -0,0 +1,6 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v4/sectors.go b/pkg/extract/actors/minerdiff/v4/sectors.go new file mode 100644 index 000000000..6756b06cd --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/sectors.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v4/state.go b/pkg/extract/actors/minerdiff/v4/state.go new file mode 100644 index 000000000..ea583d7fe --- /dev/null +++ b/pkg/extract/actors/minerdiff/v4/state.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v5/debt.go b/pkg/extract/actors/minerdiff/v5/debt.go new file mode 100644 index 000000000..f7e7b6fd5 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/debt.go @@ -0,0 +1,6 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v5/funds.go b/pkg/extract/actors/minerdiff/v5/funds.go new file mode 100644 index 000000000..8380ed56d --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/funds.go @@ -0,0 +1,6 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v5/info.go b/pkg/extract/actors/minerdiff/v5/info.go new file mode 100644 index 000000000..45135d5c7 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/info.go @@ -0,0 +1,6 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v5/precommits.go b/pkg/extract/actors/minerdiff/v5/precommits.go new file mode 100644 index 000000000..e7ff3e384 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/precommits.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v5/sector_status.go b/pkg/extract/actors/minerdiff/v5/sector_status.go new file mode 100644 index 000000000..b01fd8186 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/sector_status.go @@ -0,0 +1,6 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v5/sectors.go b/pkg/extract/actors/minerdiff/v5/sectors.go new file mode 100644 index 000000000..f1ce97e65 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/sectors.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v5/state.go b/pkg/extract/actors/minerdiff/v5/state.go new file mode 100644 index 000000000..95b671e65 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v5/state.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v6/debt.go b/pkg/extract/actors/minerdiff/v6/debt.go new file mode 100644 index 000000000..8b9d1e99a --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/debt.go @@ -0,0 +1,6 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v6/funds.go b/pkg/extract/actors/minerdiff/v6/funds.go new file mode 100644 index 000000000..d3fa00306 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/funds.go @@ -0,0 +1,6 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v6/info.go b/pkg/extract/actors/minerdiff/v6/info.go new file mode 100644 index 000000000..c38170de7 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/info.go @@ -0,0 +1,6 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v6/precommits.go b/pkg/extract/actors/minerdiff/v6/precommits.go new file mode 100644 index 000000000..fdee78a84 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/precommits.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v6/sector_status.go b/pkg/extract/actors/minerdiff/v6/sector_status.go new file mode 100644 index 000000000..ddf9a6762 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/sector_status.go @@ -0,0 +1,6 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v6/sectors.go b/pkg/extract/actors/minerdiff/v6/sectors.go new file mode 100644 index 000000000..409ec8488 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/sectors.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v6/state.go b/pkg/extract/actors/minerdiff/v6/state.go new file mode 100644 index 000000000..c3545513b --- /dev/null +++ b/pkg/extract/actors/minerdiff/v6/state.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v7/debt.go b/pkg/extract/actors/minerdiff/v7/debt.go new file mode 100644 index 000000000..58f1f242d --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/debt.go @@ -0,0 +1,6 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v7/funds.go b/pkg/extract/actors/minerdiff/v7/funds.go new file mode 100644 index 000000000..7dc10c926 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/funds.go @@ -0,0 +1,6 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v7/info.go b/pkg/extract/actors/minerdiff/v7/info.go new file mode 100644 index 000000000..63c42fc76 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/info.go @@ -0,0 +1,6 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v7/precommits.go b/pkg/extract/actors/minerdiff/v7/precommits.go new file mode 100644 index 000000000..1b8ef337e --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/precommits.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v7/sector_status.go b/pkg/extract/actors/minerdiff/v7/sector_status.go new file mode 100644 index 000000000..fc3fb8ea2 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/sector_status.go @@ -0,0 +1,6 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v7/sectors.go b/pkg/extract/actors/minerdiff/v7/sectors.go new file mode 100644 index 000000000..2a2697b3f --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/sectors.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v7/state.go b/pkg/extract/actors/minerdiff/v7/state.go new file mode 100644 index 000000000..dac14967c --- /dev/null +++ b/pkg/extract/actors/minerdiff/v7/state.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v8/debt.go b/pkg/extract/actors/minerdiff/v8/debt.go new file mode 100644 index 000000000..f16daccac --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/debt.go @@ -0,0 +1,6 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v8/funds.go b/pkg/extract/actors/minerdiff/v8/funds.go new file mode 100644 index 000000000..e5f1d0189 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/funds.go @@ -0,0 +1,6 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v8/info.go b/pkg/extract/actors/minerdiff/v8/info.go new file mode 100644 index 000000000..38690dca1 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/info.go @@ -0,0 +1,6 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v8/precommits.go b/pkg/extract/actors/minerdiff/v8/precommits.go new file mode 100644 index 000000000..be9925bab --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/precommits.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v8/sector_status.go b/pkg/extract/actors/minerdiff/v8/sector_status.go new file mode 100644 index 000000000..53f1f5cdb --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/sector_status.go @@ -0,0 +1,6 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v8/sectors.go b/pkg/extract/actors/minerdiff/v8/sectors.go new file mode 100644 index 000000000..bd706479f --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/sectors.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v8/state.go b/pkg/extract/actors/minerdiff/v8/state.go new file mode 100644 index 000000000..d2a72308d --- /dev/null +++ b/pkg/extract/actors/minerdiff/v8/state.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v9/debt.go b/pkg/extract/actors/minerdiff/v9/debt.go new file mode 100644 index 000000000..b286f512c --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/debt.go @@ -0,0 +1,6 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type DebtChange = v0.DebtChange +type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v9/funds.go b/pkg/extract/actors/minerdiff/v9/funds.go new file mode 100644 index 000000000..dd0f1c55e --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/funds.go @@ -0,0 +1,6 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type FundsChange = v0.FundsChange +type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v9/info.go b/pkg/extract/actors/minerdiff/v9/info.go new file mode 100644 index 000000000..e1c74d061 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/info.go @@ -0,0 +1,6 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type InfoChange = v0.InfoChange +type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v9/precommits.go b/pkg/extract/actors/minerdiff/v9/precommits.go new file mode 100644 index 000000000..d65a4cf36 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/precommits.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type PreCommitChange = v0.PreCommitChange +type PreCommitChangeList = v0.PreCommitChangeList +type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v9/sector_status.go b/pkg/extract/actors/minerdiff/v9/sector_status.go new file mode 100644 index 000000000..4286eb9fc --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/sector_status.go @@ -0,0 +1,6 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorStatusChange = v0.SectorStatusChange +type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v9/sectors.go b/pkg/extract/actors/minerdiff/v9/sectors.go new file mode 100644 index 000000000..2961334e3 --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/sectors.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type SectorChange = v0.SectorChange +type SectorChangeList = v0.SectorChangeList +type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v9/state.go b/pkg/extract/actors/minerdiff/v9/state.go new file mode 100644 index 000000000..287cffc2c --- /dev/null +++ b/pkg/extract/actors/minerdiff/v9/state.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/version.go b/pkg/extract/actors/minerdiff/version.go new file mode 100644 index 000000000..b149e19b1 --- /dev/null +++ b/pkg/extract/actors/minerdiff/version.go @@ -0,0 +1,116 @@ +package minerdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + v3 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + v4 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + v5 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + v6 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + v7 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version0: + return &v0.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v0.Debt{}, + v0.Funds{}, + v0.Info{}, + v0.PreCommit{}, + v0.SectorStatus{}, + v0.Sectors{}, + }}, nil + case actortypes.Version2: + return &v2.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v2.Debt{}, + v2.Funds{}, + v2.Info{}, + v2.PreCommit{}, + v2.SectorStatus{}, + v2.Sectors{}, + }}, nil + case actortypes.Version3: + return &v3.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v3.Debt{}, + v3.Funds{}, + v3.Info{}, + v3.PreCommit{}, + v3.SectorStatus{}, + v3.Sectors{}, + }}, nil + case actortypes.Version4: + return &v4.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v4.Debt{}, + v4.Funds{}, + v4.Info{}, + v4.PreCommit{}, + v4.SectorStatus{}, + v4.Sectors{}, + }}, nil + case actortypes.Version5: + return &v5.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v5.Debt{}, + v5.Funds{}, + v5.Info{}, + v5.PreCommit{}, + v5.SectorStatus{}, + v5.Sectors{}, + }}, nil + case actortypes.Version6: + return &v6.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v6.Debt{}, + v6.Funds{}, + v6.Info{}, + v6.PreCommit{}, + v6.SectorStatus{}, + v6.Sectors{}, + }}, nil + case actortypes.Version7: + return &v7.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v7.Debt{}, + v7.Funds{}, + v7.Info{}, + v7.PreCommit{}, + v7.SectorStatus{}, + v7.Sectors{}, + }}, nil + case actortypes.Version8: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v8.Debt{}, + v8.Funds{}, + v8.Info{}, + v8.PreCommit{}, + v8.SectorStatus{}, + v8.Sectors{}, + }}, nil + case actortypes.Version9: + return &v9.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Debt{}, + v9.Funds{}, + v9.Info{}, + v9.PreCommit{}, + v9.SectorStatus{}, + v9.Sectors{}, + }}, nil + case actortypes.Version10: + panic("Not yet implemented") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index d3a375094..6de41aad4 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -4,7 +4,9 @@ import ( "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" + typegen "github.com/whyrusleeping/cbor-gen" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" @@ -13,7 +15,16 @@ import ( "github.com/filecoin-project/lily/tasks" ) -type ActorDiffer interface { +type ActorDiffResult interface { + Kind() string + MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) +} + +type ActorDiff interface { + State(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) +} + +type ActorStateDiff interface { Diff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorStateChange, error) } @@ -46,7 +57,3 @@ func (a ActorChange) MarshalLogObject(enc zapcore.ObjectEncoder) error { } type ActorChanges []*ActorChange - -type ActorStateDiff interface { - Kind() string -} diff --git a/pkg/extract/actors/verifregdiff/clients.go b/pkg/extract/actors/verifregdiff/v0/clients.go similarity index 86% rename from pkg/extract/actors/verifregdiff/clients.go rename to pkg/extract/actors/verifregdiff/v0/clients.go index 143f78767..22982e52f 100644 --- a/pkg/extract/actors/verifregdiff/clients.go +++ b/pkg/extract/actors/verifregdiff/v0/clients.go @@ -1,9 +1,10 @@ -package verifregdiff +package v0 import ( "context" "time" + logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -13,6 +14,10 @@ import ( "github.com/filecoin-project/lily/tasks" ) +// TODO this log name could be confusing since its reused by subsequent versions +var log = logging.Logger("lily/extract/actors/verifreg/v0") + +// TODO add cbor gen tags type ClientsChange struct { Client []byte Current *typegen.Deferred diff --git a/pkg/extract/actors/verifregdiff/load.go b/pkg/extract/actors/verifregdiff/v0/load.go similarity index 98% rename from pkg/extract/actors/verifregdiff/load.go rename to pkg/extract/actors/verifregdiff/v0/load.go index 27a5ad933..e0bc4bed6 100644 --- a/pkg/extract/actors/verifregdiff/load.go +++ b/pkg/extract/actors/verifregdiff/v0/load.go @@ -1,4 +1,4 @@ -package verifregdiff +package v0 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/verifregdiff/state.go b/pkg/extract/actors/verifregdiff/v0/state.go similarity index 53% rename from pkg/extract/actors/verifregdiff/state.go rename to pkg/extract/actors/verifregdiff/v0/state.go index 8bed113a6..004e48a06 100644 --- a/pkg/extract/actors/verifregdiff/state.go +++ b/pkg/extract/actors/verifregdiff/v0/state.go @@ -1,31 +1,43 @@ -package verifregdiff +package v0 import ( "context" "time" - logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" ) -var log = logging.Logger("extract/actors/verifreg") - -type StateDiff struct { +type StateDiffResult struct { VerifierChanges VerifiersChangeList ClientChanges ClientsChangeList - ClaimChanges ClaimsChangeList } -func (s *StateDiff) Kind() string { +func (s *StateDiffResult) Kind() string { return "verifreg" } -func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns ...actors.ActorDiffer) (actors.ActorStateDiff, error) { +type StateChange struct { + Verifiers cid.Cid + Clients cid.Cid +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) { + panic("TODO") +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { start := time.Now() - var stateDiff = new(StateDiff) - for _, f := range diffFns { + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { stateChange, err := f.Diff(ctx, api, act) if err != nil { return nil, err @@ -36,8 +48,6 @@ func State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, d switch stateChange.Kind() { case KindVerifregClients: stateDiff.ClientChanges = stateChange.(ClientsChangeList) - case KindVerifregClaims: - stateDiff.ClaimChanges = stateChange.(ClaimsChangeList) case KindVerifregVerifiers: stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) } diff --git a/pkg/extract/actors/verifregdiff/verifiers.go b/pkg/extract/actors/verifregdiff/v0/verifiers.go similarity index 97% rename from pkg/extract/actors/verifregdiff/verifiers.go rename to pkg/extract/actors/verifregdiff/v0/verifiers.go index 0074cf1c1..574ab3e3a 100644 --- a/pkg/extract/actors/verifregdiff/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v0/verifiers.go @@ -1,4 +1,4 @@ -package verifregdiff +package v0 import ( "context" @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lily/tasks" ) +// TODO add cbor gen tags type VerifiersChange struct { Verifier []byte Current *typegen.Deferred diff --git a/pkg/extract/actors/verifregdiff/v2/clients.go b/pkg/extract/actors/verifregdiff/v2/clients.go new file mode 100644 index 000000000..501b0bbb5 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/clients.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v2/state.go b/pkg/extract/actors/verifregdiff/v2/state.go new file mode 100644 index 000000000..7aebd90b3 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/state.go @@ -0,0 +1,5 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v2/verifiers.go b/pkg/extract/actors/verifregdiff/v2/verifiers.go new file mode 100644 index 000000000..71df2621a --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v2/verifiers.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v3/clients.go b/pkg/extract/actors/verifregdiff/v3/clients.go new file mode 100644 index 000000000..c17bcd44d --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v3/clients.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v3/state.go b/pkg/extract/actors/verifregdiff/v3/state.go new file mode 100644 index 000000000..d4587fd5d --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v3/state.go @@ -0,0 +1,5 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v3/verifiers.go b/pkg/extract/actors/verifregdiff/v3/verifiers.go new file mode 100644 index 000000000..abae8239a --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v3/verifiers.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v4/clients.go b/pkg/extract/actors/verifregdiff/v4/clients.go new file mode 100644 index 000000000..c20b0c368 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v4/clients.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v4/state.go b/pkg/extract/actors/verifregdiff/v4/state.go new file mode 100644 index 000000000..cee9a4dfa --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v4/state.go @@ -0,0 +1,5 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v4/verifiers.go b/pkg/extract/actors/verifregdiff/v4/verifiers.go new file mode 100644 index 000000000..9b45da959 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v4/verifiers.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v5/clients.go b/pkg/extract/actors/verifregdiff/v5/clients.go new file mode 100644 index 000000000..f767999b3 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v5/clients.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v5/state.go b/pkg/extract/actors/verifregdiff/v5/state.go new file mode 100644 index 000000000..3632eab46 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v5/state.go @@ -0,0 +1,5 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v5/verifiers.go b/pkg/extract/actors/verifregdiff/v5/verifiers.go new file mode 100644 index 000000000..1ed52d298 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v5/verifiers.go @@ -0,0 +1,7 @@ +package v5 + +import "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v6/clients.go b/pkg/extract/actors/verifregdiff/v6/clients.go new file mode 100644 index 000000000..637882000 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v6/clients.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v6/state.go b/pkg/extract/actors/verifregdiff/v6/state.go new file mode 100644 index 000000000..41901625c --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v6/state.go @@ -0,0 +1,5 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v6/verifiers.go b/pkg/extract/actors/verifregdiff/v6/verifiers.go new file mode 100644 index 000000000..0c3e02d8a --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v6/verifiers.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v7/clients.go b/pkg/extract/actors/verifregdiff/v7/clients.go new file mode 100644 index 000000000..ef01f2763 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v7/clients.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v7/state.go b/pkg/extract/actors/verifregdiff/v7/state.go new file mode 100644 index 000000000..35af0c1a3 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v7/state.go @@ -0,0 +1,5 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v7/verifiers.go b/pkg/extract/actors/verifregdiff/v7/verifiers.go new file mode 100644 index 000000000..3573cb58b --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v7/verifiers.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v8/clients.go b/pkg/extract/actors/verifregdiff/v8/clients.go new file mode 100644 index 000000000..1090cabf5 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v8/clients.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type ClientsChange = v0.ClientsChange +type ClientsChangeList = v0.ClientsChangeList +type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v8/state.go b/pkg/extract/actors/verifregdiff/v8/state.go new file mode 100644 index 000000000..9f295f3e3 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v8/state.go @@ -0,0 +1,5 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v8/verifiers.go b/pkg/extract/actors/verifregdiff/v8/verifiers.go new file mode 100644 index 000000000..9a8a8e9ad --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v8/verifiers.go @@ -0,0 +1,9 @@ +package v8 + +import ( + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +) + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v9/allocations.go b/pkg/extract/actors/verifregdiff/v9/allocations.go new file mode 100644 index 000000000..0a5009efb --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v9/allocations.go @@ -0,0 +1,46 @@ +package v9 + +import ( + "context" + "time" + + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +// TODO add cbor gen tags +type AllocationsChange struct { + Provider []byte + ClaimID []byte + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType +} + +type AllocationsChangeList []*AllocationsChange + +const KindVerifregAllocations = "verifreg_allocations" + +func (c AllocationsChangeList) Kind() actors.ActorStateKind { + return KindVerifregAllocations +} + +type Allocations struct{} + +func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindVerifregAllocations, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffAllocations(ctx, api, act) +} + +func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + // TODO this will look a lot like the DiffClaims method, except diffing allocations + // - need to add method to the actor that exposes the allocations HAMT and its sub HAMT AllocationMapForProvider() + panic("NYI") +} diff --git a/pkg/extract/actors/verifregdiff/claims.go b/pkg/extract/actors/verifregdiff/v9/claims.go similarity index 88% rename from pkg/extract/actors/verifregdiff/claims.go rename to pkg/extract/actors/verifregdiff/v9/claims.go index 6399c4d58..4b370da0c 100644 --- a/pkg/extract/actors/verifregdiff/claims.go +++ b/pkg/extract/actors/verifregdiff/v9/claims.go @@ -1,4 +1,4 @@ -package verifregdiff +package v9 import ( "context" @@ -10,6 +10,7 @@ import ( "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" @@ -18,6 +19,7 @@ import ( "github.com/filecoin-project/lily/tasks" ) +// TODO add cbor gen tags type ClaimsChange struct { Provider []byte ClaimID []byte @@ -45,7 +47,7 @@ func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorC } func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiiregClaimsMapLoader) + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, v0.VerifiiregClaimsMapLoader) if err != nil { return nil, err } @@ -62,7 +64,7 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan } func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerKey []byte) ([]*ClaimsChange, error) { - mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { providerID, err := abi.ParseUIntKey(string(providerKey)) if err != nil { return nil, nil, err diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go new file mode 100644 index 000000000..6fe65bf3a --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -0,0 +1,60 @@ +package v9 + +import ( + "context" + "time" + + "github.com/filecoin-project/lotus/blockstore" + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v8" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/verifreg") + +type StateDiffResult struct { + VerifierChanges v8.VerifiersChangeList + ClaimChanges ClaimsChangeList + AllocationsChanges AllocationsChangeList +} + +func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + //TODO implement me + panic("implement me") +} + +func (s *StateDiffResult) Kind() string { + return "verifreg" +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case v0.KindVerifregVerifiers: + stateDiff.VerifierChanges = stateChange.(v0.VerifiersChangeList) + case KindVerifregClaims: + stateDiff.ClaimChanges = stateChange.(ClaimsChangeList) + case KindVerifregAllocations: + stateDiff.AllocationsChanges = stateChange.(AllocationsChangeList) + } + } + log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} diff --git a/pkg/extract/actors/verifregdiff/v9/verifiers.go b/pkg/extract/actors/verifregdiff/v9/verifiers.go new file mode 100644 index 000000000..b65a39e8a --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v9/verifiers.go @@ -0,0 +1,9 @@ +package v9 + +import ( + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +) + +type VerifiersChange = v0.VerifiersChange +type VerifiersChangeList = v0.VerifiersChangeList +type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go new file mode 100644 index 000000000..1f9c19e8b --- /dev/null +++ b/pkg/extract/actors/verifregdiff/version.go @@ -0,0 +1,81 @@ +package verifregdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" + v3 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v3" + v4 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v4" + v5 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v5" + v6 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v6" + v7 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v7" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v8" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version0: + return &v0.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v0.Clients{}, + v0.Verifiers{}, + }}, nil + case actortypes.Version2: + return &v2.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v2.Clients{}, + v2.Verifiers{}, + }}, nil + case actortypes.Version3: + return &v3.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v3.Clients{}, + v3.Verifiers{}, + }}, nil + case actortypes.Version4: + return &v4.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v4.Clients{}, + v4.Verifiers{}, + }}, nil + case actortypes.Version5: + return &v5.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v5.Clients{}, + v5.Verifiers{}, + }}, nil + case actortypes.Version6: + return &v6.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v6.Clients{}, + v6.Verifiers{}, + }}, nil + case actortypes.Version7: + return &v7.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v7.Clients{}, + v7.Verifiers{}, + }}, nil + case actortypes.Version8: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v8.Clients{}, + v8.Verifiers{}, + }}, nil + case actortypes.Version9: + return &v9.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Verifiers{}, + v9.Claims{}, + v9.Allocations{}, + }}, nil + case actortypes.Version10: + panic("Not yet implemented") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 05b40fff4..4c735f141 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -5,6 +5,8 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -16,6 +18,7 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/chain/actors/builtin/power" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" @@ -51,8 +54,8 @@ type ActorStateChanges struct { Current *types.TipSet Executed *types.TipSet Actors map[address.Address]statetree.ActorDiff - MinerActors map[address.Address]*minerdiff.StateDiff - VerifregActor map[address.Address]*verifregdiff.StateDiff + MinerActors map[address.Address]actors.ActorDiffResult + VerifregActor actors.ActorDiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { @@ -61,7 +64,6 @@ func (a ActorStateChanges) Attributes() []attribute.KeyValue { attribute.Int64("executed", int64(a.Executed.Height())), attribute.Int("actor_change", len(a.Actors)), attribute.Int("miner_changes", len(a.MinerActors)), - attribute.Int("verifreg_changes", len(a.VerifregActor)), } } @@ -73,21 +75,27 @@ func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { } type StateDiffResult struct { - ActorDiff actors.ActorStateDiff + ActorDiff actors.ActorDiffResult Address address.Address } -func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ActorStateChanges, error) { +type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version + +func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, nvg NetworkVersionGetter) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { return nil, err } asc := &ActorStateChanges{ - Current: current, - Executed: executed, - Actors: actorChanges, - MinerActors: make(map[address.Address]*minerdiff.StateDiff, len(actorChanges)), // there are at most actorChanges entries - VerifregActor: make(map[address.Address]*verifregdiff.StateDiff, len(actorChanges)), + Current: current, + Executed: executed, + Actors: actorChanges, + MinerActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries + } + + actorVersion, err := core.ActorVersionForTipSet(ctx, current, nvg) + if err != nil { + return nil, err } grp, grpCtx := errgroup.WithContext(ctx) @@ -95,22 +103,22 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current for addr, change := range actorChanges { addr := addr change := change + act := &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, + } grp.Go(func() error { if MinerCodes.Has(change.Current.Code) { start := time.Now() - minerChanges, err := minerdiff.State(grpCtx, api, &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - }, - minerdiff.Debt{}, - minerdiff.Funds{}, - minerdiff.Info{}, - minerdiff.PreCommit{}, - minerdiff.Sectors{}, - minerdiff.SectorStatus{}, - ) + // construct the state differ required by this actor version + actorDiff, err := minerdiff.StateDiffFor(actorVersion) + if err != nil { + return err + } + // diff the actors state and collect changes + minerChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { return err } @@ -121,18 +129,15 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current } } if VerifregCodes.Has(change.Current.Code) { + return nil start := time.Now() - verifregChanges, err := verifregdiff.State(grpCtx, api, &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - }, - // TODO the functions handed to these methods should be based on the epoch of the chain. - //verifregdiff.Clients{}, - verifregdiff.Verifiers{}, - verifregdiff.Claims{}, - ) + // construct the state differ required by this actor version + actorDiff, err := verifregdiff.StateDiffFor(actorVersion) + if err != nil { + return err + } + // diff the actors state and collect changes + verifregChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { return err } @@ -152,11 +157,12 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current close(results) }() for stateDiff := range results { - switch v := stateDiff.ActorDiff.(type) { - case *minerdiff.StateDiff: - asc.MinerActors[stateDiff.Address] = v - case *verifregdiff.StateDiff: - asc.VerifregActor[stateDiff.Address] = v + switch stateDiff.ActorDiff.Kind() { + + case "miner": + asc.MinerActors[stateDiff.Address] = stateDiff.ActorDiff + case "verifreg": + asc.VerifregActor = stateDiff.ActorDiff } } return asc, nil diff --git a/pkg/gen/main.go b/pkg/gen/main.go index acea02ed1..30d37815e 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -3,9 +3,8 @@ package main import ( cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" - "github.com/filecoin-project/lily/pkg/transform/cbor" - "github.com/filecoin-project/lily/pkg/transform/cbor/miner" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + "github.com/filecoin-project/lily/pkg/transform/cbor/miner/v9" ) const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" @@ -19,26 +18,30 @@ const actorTransformPkg = "cbor" func main() { if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, - minerdiff.SectorStatusChange{}, - minerdiff.PreCommitChange{}, - minerdiff.SectorChange{}, - minerdiff.FundsChange{}, - minerdiff.DebtChange{}, + v0.SectorStatusChange{}, + v0.PreCommitChange{}, + v0.SectorChange{}, + v0.FundsChange{}, + v0.DebtChange{}, + v0.InfoChange{}, + v9.StateChange{}, ); err != nil { panic(err) } - if err := cbg.WriteMapEncodersToFile(minerTransformPath, minerTransformPkg, - miner.StateChange{}, - miner.Info{}, - ); err != nil { - panic(err) - } + /* + if err := cbg.WriteMapEncodersToFile(minerTransformPath, minerTransformPkg, + v9.StateChange{}, + ); err != nil { + panic(err) + } - if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, - cbor.ActorIPLDContainer{}, - ); err != nil { - panic(err) - } + if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, + cbor.ActorIPLDContainer{}, + ); err != nil { + panic(err) + } + + */ } diff --git a/pkg/transform/cbor/miner/cbor_gen.go b/pkg/transform/cbor/miner/cbor_gen.go deleted file mode 100644 index a1affaa20..000000000 --- a/pkg/transform/cbor/miner/cbor_gen.go +++ /dev/null @@ -1,499 +0,0 @@ -// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. - -package miner - -import ( - "fmt" - "io" - "math" - "sort" - - core "github.com/filecoin-project/lily/pkg/core" - cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -var _ = xerrors.Errorf -var _ = cid.Undef -var _ = math.E -var _ = sort.Sort - -func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{167}); err != nil { - return err - } - - // t.Miner (address.Address) (struct) - if len("miner") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"miner\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("miner")); err != nil { - return err - } - - if err := t.Miner.MarshalCBOR(cw); err != nil { - return err - } - - // t.Funds (cid.Cid) (struct) - if len("funds") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"funds\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("funds"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("funds")); err != nil { - return err - } - - if t.Funds == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Funds); err != nil { - return xerrors.Errorf("failed to write cid field t.Funds: %w", err) - } - } - - // t.Debt (cid.Cid) (struct) - if len("debt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"debt\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("debt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("debt")); err != nil { - return err - } - - if t.Debt == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Debt); err != nil { - return xerrors.Errorf("failed to write cid field t.Debt: %w", err) - } - } - - // t.SectorStatus (cid.Cid) (struct) - if len("sector_status") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sector_status\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("sector_status")); err != nil { - return err - } - - if t.SectorStatus == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { - return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) - } - } - - // t.Info (cid.Cid) (struct) - if len("info") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"info\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("info")); err != nil { - return err - } - - if t.Info == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) - } - } - - // t.PreCommits (cid.Cid) (struct) - if len("pre_commits") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"pre_commits\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("pre_commits")); err != nil { - return err - } - - if t.PreCommits == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.PreCommits); err != nil { - return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) - } - } - - // t.Sectors (cid.Cid) (struct) - if len("sectors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sectors\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("sectors")); err != nil { - return err - } - - if t.Sectors == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Sectors); err != nil { - return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) - } - } - - return nil -} - -func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Miner (address.Address) (struct) - case "miner": - - { - - if err := t.Miner.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Miner: %w", err) - } - - } - // t.Funds (cid.Cid) (struct) - case "funds": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Funds: %w", err) - } - - t.Funds = &c - } - - } - // t.Debt (cid.Cid) (struct) - case "debt": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Debt: %w", err) - } - - t.Debt = &c - } - - } - // t.SectorStatus (cid.Cid) (struct) - case "sector_status": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) - } - - t.SectorStatus = &c - } - - } - // t.Info (cid.Cid) (struct) - case "info": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) - } - - t.Info = &c - } - - } - // t.PreCommits (cid.Cid) (struct) - case "pre_commits": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) - } - - t.PreCommits = &c - } - - } - // t.Sectors (cid.Cid) (struct) - case "sectors": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) - } - - t.Sectors = &c - } - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *Info) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Info (cid.Cid) (struct) - if len("info") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"info\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("info")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) - } - - // t.Change (core.ChangeType) (uint8) - if len("change") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"change\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("change")); err != nil { - return err - } - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { - return err - } - return nil -} - -func (t *Info) UnmarshalCBOR(r io.Reader) (err error) { - *t = Info{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("Info: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Info (cid.Cid) (struct) - case "info": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) - } - - t.Info = c - - } - // t.Change (core.ChangeType) (uint8) - case "change": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint8 field") - } - if extra > math.MaxUint8 { - return fmt.Errorf("integer in input was too large for uint8 field") - } - t.Change = core.ChangeType(extra) - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/pkg/transform/cbor/miner/state.go b/pkg/transform/cbor/miner/state.go index 6b895634b..377058df5 100644 --- a/pkg/transform/cbor/miner/state.go +++ b/pkg/transform/cbor/miner/state.go @@ -1,251 +1,33 @@ package miner import ( - "bytes" "context" - "sort" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - "github.com/ipfs/go-cid" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lotus/blockstore" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" - "gorm.io/driver/sqlite" - "gorm.io/gorm" - - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/extract/actors" ) -var DB *gorm.DB - -func init() { - var err error - DB, err = gorm.Open(sqlite.Open("test.db"), &gorm.Config{}) +func HandleChanges(ctx context.Context, bs blockstore.Blockstore, miners map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { + store := adt2.WrapBlockStore(ctx, bs) + minerHamt, err := adt.MakeEmptyMap(store, 5 /*TODO*/) if err != nil { - panic(err) - } - if err := DB.AutoMigrate(&StateChangeModel{}); err != nil { - panic(err) - } -} - -type StateChangeModel struct { - gorm.Model - Miner string - Funds string - Debt string - SectorStatus string - Info string - PreCommits string - Sectors string -} - -func (sc *StateChange) ToStorage(db *gorm.DB) error { - var ( - funds = "" - debt = "" - status = "" - info = "" - precommit = "" - sector = "" - ) - if sc.Funds != nil { - funds = sc.Funds.String() - } - if sc.Debt != nil { - debt = sc.Debt.String() - } - if sc.SectorStatus != nil { - status = sc.SectorStatus.String() - } - if sc.Info != nil { - info = sc.Info.String() - } - if sc.PreCommits != nil { - precommit = sc.PreCommits.String() - } - if sc.Sectors != nil { - sector = sc.Sectors.String() - } - tx := db.Create(&StateChangeModel{ - Miner: sc.Miner.String(), - Funds: funds, - Debt: debt, - SectorStatus: status, - Info: info, - PreCommits: precommit, - Sectors: sector, - }) - if tx.Error != nil { - return tx.Error + return cid.Undef, err } - return nil -} - -type StateChange struct { - // Miner is the address of the miner that changes. - Miner address.Address `cborgen:"miner"` - // Funds is the funds that changed for this miner or empty. - Funds *cid.Cid `cborgen:"funds"` - // Debt is the debt that changed for this miner or empty. - Debt *cid.Cid `cborgen:"debt"` - // SectorStatus is the sectors whose status changed for this miner or empty. - SectorStatus *cid.Cid `cborgen:"sector_status"` - // Info is the cid of the miner change info that changed for this miner or empty. - Info *cid.Cid `cborgen:"info"` - // PreCommits is an AMT of the pre commits that changed for this miner or empty. - PreCommits *cid.Cid `cborgen:"pre_commits"` - // Sectors is an AMT of the sectors that changed for this miner or empty. - Sectors *cid.Cid `cborgen:"sectors"` -} - -func HandleChanges(ctx context.Context, store adt.Store, actorHamt *adt.Map, miners map[address.Address]*minerdiff.StateDiff) error { for addr, change := range miners { - msc, err := ChangeHandler(ctx, store, addr, change) - if err != nil { - return err - } - if msc == nil { - continue - } - if err := msc.ToStorage(DB); err != nil { - return err - } - if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { - return err - } - } - return nil -} - -func ChangeHandler(ctx context.Context, store adt.Store, miner address.Address, change *minerdiff.StateDiff) (*StateChange, error) { - hasChangedState := false - out := &StateChange{} - if change.FundsChange != nil { - hasChangedState = true - c, err := FundsHandler(ctx, store, change.FundsChange) - if err != nil { - return nil, err - } - out.Funds = &c - } - if change.DebtChange != nil { - hasChangedState = true - c, err := DebtHandler(ctx, store, change.DebtChange) + msc, err := change.MarshalStateChange(ctx, bs) if err != nil { - return nil, err - } - out.Debt = &c - } - if change.SectorStatusChanges != nil { - hasChangedState = true - c, err := SectorStatusHandler(ctx, store, change.SectorStatusChanges) - if err != nil { - return nil, err - } - out.SectorStatus = &c - } - if change.InfoChange != nil { - hasChangedState = true - c, err := InfoHandler(ctx, store, change.InfoChange) - if err != nil { - return nil, err - } - out.Info = &c - } - if change.SectorChanges != nil && len(change.SectorChanges) > 0 { - hasChangedState = true - c, err := SectorHandler(ctx, store, change.SectorChanges) - if err != nil { - return nil, err - } - out.Sectors = &c - } - if change.PreCommitChanges != nil && len(change.PreCommitChanges) > 0 { - hasChangedState = true - c, err := PreCommitHandler(ctx, store, change.PreCommitChanges) - if err != nil { - return nil, err - } - out.PreCommits = &c - } - if hasChangedState { - out.Miner = miner - return out, nil - } - return nil, nil -} - -func FundsHandler(ctx context.Context, store adt.Store, funds *minerdiff.FundsChange) (cid.Cid, error) { - return store.Put(ctx, funds) -} - -func DebtHandler(ctx context.Context, store adt.Store, debt *minerdiff.DebtChange) (cid.Cid, error) { - return store.Put(ctx, debt) -} - -func SectorStatusHandler(ctx context.Context, store adt.Store, sectors *minerdiff.SectorStatusChange) (cid.Cid, error) { - return store.Put(ctx, sectors) -} - -func PreCommitHandler(ctx context.Context, store adt.Store, list minerdiff.PreCommitChangeList) (cid.Cid, error) { - // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. - sort.Slice(list, func(i, j int) bool { - cmp := bytes.Compare(list[i].SectorNumber, list[j].SectorNumber) - switch cmp { - case 1: - return true - case -1: - return false - default: - panic("precommit with same ID changed twice in one epoch, not possible") - } - }) - arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for _, l := range list { - if err := arr.AppendContinuous(l); err != nil { return cid.Undef, err } - } - return arr.Root() -} -func SectorHandler(ctx context.Context, store adt.Store, list minerdiff.SectorChangeList) (cid.Cid, error) { - // HACK: this isn't ideal, but we need deterministic ordering and lack a native IPLD ordered set. - sort.Slice(list, func(i, j int) bool { - return list[i].SectorNumber < list[j].SectorNumber - }) - arr, err := adt.MakeEmptyArray(store, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for _, l := range list { - if err := arr.AppendContinuous(l); err != nil { + if err := minerHamt.Put(abi.AddrKey(addr), msc); err != nil { return cid.Undef, err } } - return arr.Root() -} - -type Info struct { - Info cid.Cid `cborgen:"info"` - Change core.ChangeType `cborgen:"change"` -} - -func InfoHandler(ctx context.Context, store adt.Store, info *minerdiff.InfoChange) (cid.Cid, error) { - // ensure the miner info CID is the same as the CID found on chain. - mInfoCid, err := store.Put(ctx, info.Info) - if err != nil { - return cid.Undef, err - } - infoChange := &Info{ - Info: mInfoCid, - Change: info.Change, - } - return store.Put(ctx, infoChange) + return minerHamt.Root() } diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 4a8dd5951..1edad2cd6 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -4,32 +4,35 @@ import ( "context" "io" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" "github.com/ipld/go-car/util" + v1car "github.com/ipld/go-car" + "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/pkg/transform/cbor/miner" - - v1car "github.com/ipld/go-car" + "github.com/filecoin-project/lily/pkg/transform/cbor/verifreg" ) type ActorIPLDContainer struct { + // TODO this needs to be versioned CurrentTipSet *types.TipSet ExecutedTipSet *types.TipSet - MinerActors cid.Cid // HAMT[Address]MinerStateChange + MinerActors cid.Cid // HAMT[Address]MinerStateChange + VerifregActor *cid.Cid // VerifregStateChange or empty } func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { bs := blockstore.NewMemorySync() store := adt2.WrapBlockStore(ctx, bs) - actorStates, err := ProcessActors(ctx, store, changes) + actorStates, err := ProcessActors(ctx, bs, changes) if err != nil { return err } + actorStatesRoot, err := store.Put(ctx, actorStates) if err != nil { return err @@ -56,21 +59,22 @@ func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io return nil } -func ProcessActors(ctx context.Context, store adt.Store, changes *procesor.ActorStateChanges) (*ActorIPLDContainer, error) { - m, err := adt.MakeEmptyMap(store, 5 /*TODO*/) - if err != nil { - return nil, err - } - if err := miner.HandleChanges(ctx, store, m, changes.MinerActors); err != nil { - return nil, err +func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *procesor.ActorStateChanges) (*ActorIPLDContainer, error) { + out := &ActorIPLDContainer{ + CurrentTipSet: changes.Current, + ExecutedTipSet: changes.Executed, } - minerActorsRoot, err := m.Root() + minerHamtRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) if err != nil { return nil, err } - return &ActorIPLDContainer{ - CurrentTipSet: changes.Current, - ExecutedTipSet: changes.Executed, - MinerActors: minerActorsRoot, - }, nil + out.MinerActors = minerHamtRoot + if changes.VerifregActor != nil { + verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) + if err != nil { + return nil, err + } + out.VerifregActor = &verifregRoot + } + return out, nil } diff --git a/pkg/transform/cbor/verifreg/state.go b/pkg/transform/cbor/verifreg/state.go new file mode 100644 index 000000000..0bc8640b1 --- /dev/null +++ b/pkg/transform/cbor/verifreg/state.go @@ -0,0 +1,16 @@ +package verifreg + +import ( + "context" + + "github.com/filecoin-project/lotus/blockstore" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func HandleChanges(ctx context.Context, bs blockstore.Blockstore, verifreg actors.ActorDiffResult) (cid.Cid, error) { + store := adt2.WrapBlockStore(ctx, bs) + return store.Put(ctx, verifreg) +} diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go new file mode 100644 index 000000000..ab865910b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/router.go @@ -0,0 +1 @@ +package miner diff --git a/pkg/transform/timescale/actors/miner/v0/debt.go b/pkg/transform/timescale/actors/miner/v0/debt.go new file mode 100644 index 000000000..61453a8be --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/debt.go @@ -0,0 +1,23 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +type Debt struct{} + +func (Debt) Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { + return &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: addr.String(), + FeeDebt: change.DebtChange.FeeDebt.String(), + }, nil +} diff --git a/pkg/transform/timescale/miner/v9/funds.go b/pkg/transform/timescale/actors/miner/v0/funds.go similarity index 85% rename from pkg/transform/timescale/miner/v9/funds.go rename to pkg/transform/timescale/actors/miner/v0/funds.go index d08eb8d5e..36574d00c 100644 --- a/pkg/transform/timescale/miner/v9/funds.go +++ b/pkg/transform/timescale/actors/miner/v0/funds.go @@ -1,4 +1,4 @@ -package v9 +package v0 import ( "context" @@ -9,10 +9,10 @@ import ( "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" ) -func HandleMinerFundsChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.FundsChange) (model.Persistable, error) { +func HandleMinerFundsChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *v0.FundsChange) (model.Persistable, error) { return &minermodel.MinerLockedFund{ Height: int64(current.Height()), MinerID: addr.String(), diff --git a/pkg/transform/timescale/actors/miner/v0/info.go b/pkg/transform/timescale/actors/miner/v0/info.go new file mode 100644 index 000000000..10a8c80b5 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/info.go @@ -0,0 +1,87 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + "github.com/libp2p/go-libp2p/core/peer" + maddr "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +func HandleMinerInfo(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { + info := change.InfoChange + var out model.Persistable + var err error + if err := core.StateReadDeferred(ctx, info.Info, func(in *miner0.MinerInfo) error { + out, err = MinerInfoAsModel(ctx, current, executed, addr, *in) + if err != nil { + return err + } + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner0.MinerInfo) (model.Persistable, error) { + return GenericMinerInfoAsModel(ctx, current, executed, addr, info) +} + +func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner0.MinerInfo) (model.Persistable, error) { + var newWorker string + var newWorkerEpoch int64 + if pendingWorkerKey := info.PendingWorkerKey; pendingWorkerKey != nil { + if pendingWorkerKey.NewWorker != address.Undef { + newWorker = pendingWorkerKey.NewWorker.String() + } + newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt) + } + + var newCtrlAddresses []string + for _, addr := range info.ControlAddresses { + newCtrlAddresses = append(newCtrlAddresses, addr.String()) + } + + // best effort to decode, we have no control over what miners put in this field, its just bytes. + var newMultiAddrs []string + for _, addr := range info.Multiaddrs { + newMaddr, err := maddr.NewMultiaddrBytes(addr) + if err == nil { + newMultiAddrs = append(newMultiAddrs, newMaddr.String()) + } else { + //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) + } + } + mi := &minermodel.MinerInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + OwnerID: info.Owner.String(), + WorkerID: info.Worker.String(), + NewWorker: newWorker, + WorkerChangeEpoch: newWorkerEpoch, + ConsensusFaultedElapsed: -1, + ControlAddresses: newCtrlAddresses, + MultiAddresses: newMultiAddrs, + SectorSize: uint64(info.SectorSize), + } + + if info.PeerId != nil { + newPeerID, err := peer.IDFromBytes(info.PeerId) + if err != nil { + //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) + } else { + mi.PeerID = newPeerID.String() + } + } + + return mi, nil +} diff --git a/pkg/transform/timescale/actors/miner/v0/precommits.go b/pkg/transform/timescale/actors/miner/v0/precommits.go new file mode 100644 index 000000000..7ff8498b6 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/precommits.go @@ -0,0 +1,59 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +type PreCommits struct{} + +func (PreCommits) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { + var precommits []*miner0.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + if err := core.StateReadDeferred(ctx, change.Current, func(precommit *miner0.SectorPreCommitOnChainInfo) error { + precommits = append(precommits, precommit) + return nil + }); err != nil { + return nil, err + } + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner0.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go new file mode 100644 index 000000000..6c978179f --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/processor.go @@ -0,0 +1,70 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + minerState := new(v0.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + if err != nil { + return err + } + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + // TODO + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v0.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v0/sector_deals.go new file mode 100644 index 000000000..6ff4d42d3 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/sector_deals.go @@ -0,0 +1,87 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, sectors v0.SectorChangeList) (model.Persistable, error) { + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner0.SectorOnChainInfo) error { + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + return nil + }); err != nil { + return nil, err + } + case core.ChangeTypeModify: + previousSector := new(miner0.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner0.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range compareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} + +func compareDealIDs(cur, pre []abi.DealID) []abi.DealID { + var diff []abi.DealID + + // Loop two times, first to find cur dealIDs not in pre, + // second loop to find pre dealIDs not in cur + for i := 0; i < 2; i++ { + for _, s1 := range cur { + found := false + for _, s2 := range pre { + if s1 == s2 { + found = true + break + } + } + // DealID not found. We add it to return slice + if !found { + diff = append(diff, s1) + } + } + // Swap the slices, only if it was the first loop + if i == 0 { + cur, pre = pre, cur + } + } + + return diff +} diff --git a/pkg/transform/timescale/actors/miner/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v0/sector_events.go new file mode 100644 index 000000000..ba00fa676 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/sector_events.go @@ -0,0 +1,149 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, precommits v0.PreCommitChangeList, sectors v0.SectorChangeList, sectorstatus *v0.SectorStatusChange) (model.Persistable, error) { + out := minermodel.MinerSectorEventList{} + height := int64(current.Height()) + minerAddr := addr.String() + stateRoot := current.ParentState().String() + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner0.SectorOnChainInfo) error { + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + return nil + }); err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner0.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner0.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + // snapping didn't exist at this epoch + /* + if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorSnapped, + }) + } + + */ + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v0/sectors.go b/pkg/transform/timescale/actors/miner/v0/sectors.go new file mode 100644 index 000000000..9795c8bde --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/sectors.go @@ -0,0 +1,55 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" +) + +func HandleMinerSectorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes v0.SectorChangeList) (model.Persistable, error) { + var sectors []*miner0.SectorOnChainInfo + for _, change := range changes { + // only care about modified and added sectors + if change.Change == core.ChangeTypeRemove { + continue + } + // change.Current is the newly added sector, or its state after modification. + if err := core.StateReadDeferred(ctx, change.Current, func(sector *miner0.SectorOnChainInfo) error { + sectors = append(sectors, sector) + return nil + }); err != nil { + return nil, err + } + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner0.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v2/processor.go b/pkg/transform/timescale/actors/miner/v2/processor.go new file mode 100644 index 000000000..9e8abb873 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/processor.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" +) + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + minerState := new(v2.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + if err != nil { + return err + } + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + // TODO + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v2.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v2.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} diff --git a/pkg/transform/timescale/miner/v9/debt.go b/pkg/transform/timescale/actors/miner/v9/debt.go similarity index 53% rename from pkg/transform/timescale/miner/v9/debt.go rename to pkg/transform/timescale/actors/miner/v9/debt.go index f4533ef26..14971abce 100644 --- a/pkg/transform/timescale/miner/v9/debt.go +++ b/pkg/transform/timescale/actors/miner/v9/debt.go @@ -6,17 +6,21 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerDebtChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.DebtChange) (model.Persistable, error) { +type Debt struct{} + +func (Debt) Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { + if change.DebtChange == nil { + return nil, nil + } return &minermodel.MinerFeeDebt{ Height: int64(current.Height()), StateRoot: current.ParentState().String(), MinerID: addr.String(), - FeeDebt: changes.FeeDebt.String(), + FeeDebt: change.DebtChange.FeeDebt.String(), }, nil } diff --git a/pkg/transform/timescale/actors/miner/v9/funds.go b/pkg/transform/timescale/actors/miner/v9/funds.go new file mode 100644 index 000000000..c9d75cd9b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v9/funds.go @@ -0,0 +1,29 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" +) + +type Fund struct{} + +func (Fund) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { + if change.FundsChange == nil { + return nil, nil + } + funds := change.FundsChange + return &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: funds.VestingFunds.String(), + InitialPledge: funds.InitialPledgeRequirement.String(), + PreCommitDeposits: funds.PreCommitDeposit.String(), + }, nil +} diff --git a/pkg/transform/timescale/miner/v9/info.go b/pkg/transform/timescale/actors/miner/v9/info.go similarity index 88% rename from pkg/transform/timescale/miner/v9/info.go rename to pkg/transform/timescale/actors/miner/v9/info.go index 2afe008ba..4a36d0336 100644 --- a/pkg/transform/timescale/miner/v9/info.go +++ b/pkg/transform/timescale/actors/miner/v9/info.go @@ -9,14 +9,19 @@ import ( "github.com/libp2p/go-libp2p/core/peer" maddr "github.com/multiformats/go-multiaddr" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerInfo(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, info *minerdiff.InfoChange) (model.Persistable, error) { +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { + if change.InfoChange == nil { + return nil, nil + } + info := change.InfoChange var out model.Persistable var err error if err := core.StateReadDeferred(ctx, info.Info, func(in *miner9.MinerInfo) error { @@ -68,7 +73,7 @@ func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSe WorkerID: info.Worker.String(), NewWorker: newWorker, WorkerChangeEpoch: newWorkerEpoch, - ConsensusFaultedElapsed: int64(info.ConsensusFaultElapsed), + ConsensusFaultedElapsed: -1, ControlAddresses: newCtrlAddresses, MultiAddresses: newMultiAddrs, SectorSize: uint64(info.SectorSize), diff --git a/pkg/transform/timescale/miner/v9/precommits.go b/pkg/transform/timescale/actors/miner/v9/precommits.go similarity index 85% rename from pkg/transform/timescale/miner/v9/precommits.go rename to pkg/transform/timescale/actors/miner/v9/precommits.go index 8c5155b96..53eb47f4d 100644 --- a/pkg/transform/timescale/miner/v9/precommits.go +++ b/pkg/transform/timescale/actors/miner/v9/precommits.go @@ -7,16 +7,17 @@ import ( miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerPreCommitChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes minerdiff.PreCommitChangeList) (model.Persistable, error) { +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { var precommits []*miner9.SectorPreCommitOnChainInfo - for _, change := range changes { + for _, change := range change.PreCommitChanges { // only care about precommits added if change.Change != core.ChangeTypeAdd { continue diff --git a/pkg/transform/timescale/actors/miner/v9/processor.go b/pkg/transform/timescale/actors/miner/v9/processor.go new file mode 100644 index 000000000..2a7f2d02d --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v9/processor.go @@ -0,0 +1,78 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" +) + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + minerState := new(v9.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + if err != nil { + return err + } + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Debt{}, + Fund{}, + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v9.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/miner/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v9/sector_deals.go similarity index 87% rename from pkg/transform/timescale/miner/v9/sector_deals.go rename to pkg/transform/timescale/actors/miner/v9/sector_deals.go index 119dbd402..5dbf9ff4d 100644 --- a/pkg/transform/timescale/miner/v9/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_deals.go @@ -9,14 +9,16 @@ import ( miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, sectors minerdiff.SectorChangeList) (model.Persistable, error) { +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) minerAddr := addr.String() diff --git a/pkg/transform/timescale/miner/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v9/sector_events.go similarity index 86% rename from pkg/transform/timescale/miner/v9/sector_events.go rename to pkg/transform/timescale/actors/miner/v9/sector_events.go index 44a1bbaa3..c624b58ae 100644 --- a/pkg/transform/timescale/miner/v9/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_events.go @@ -9,18 +9,24 @@ import ( miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, precommits minerdiff.PreCommitChangeList, sectors minerdiff.SectorChangeList, sectorstatus *minerdiff.SectorStatusChange) (model.Persistable, error) { - out := minermodel.MinerSectorEventList{} - height := int64(current.Height()) - minerAddr := addr.String() - stateRoot := current.ParentState().String() +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) for _, precommit := range precommits { // only care about new precommits if precommit.Change != core.ChangeTypeAdd { diff --git a/pkg/transform/timescale/miner/v9/sectors.go b/pkg/transform/timescale/actors/miner/v9/sectors.go similarity index 80% rename from pkg/transform/timescale/miner/v9/sectors.go rename to pkg/transform/timescale/actors/miner/v9/sectors.go index 0d92f5ade..f98fd5a83 100644 --- a/pkg/transform/timescale/miner/v9/sectors.go +++ b/pkg/transform/timescale/actors/miner/v9/sectors.go @@ -7,22 +7,25 @@ import ( miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) -func HandleMinerSectorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes minerdiff.SectorChangeList) (model.Persistable, error) { +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { var sectors []*miner9.SectorOnChainInfo - for _, change := range changes { + changes := change.SectorChanges + for _, sector := range changes { // only care about modified and added sectors - if change.Change == core.ChangeTypeRemove { + if sector.Change == core.ChangeTypeRemove { continue } // change.Current is the newly added sector, or its state after modification. - if err := core.StateReadDeferred(ctx, change.Current, func(sector *miner9.SectorOnChainInfo) error { + if err := core.StateReadDeferred(ctx, sector.Current, func(sector *miner9.SectorOnChainInfo) error { sectors = append(sectors, sector) return nil }); err != nil { diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/actors/processor.go new file mode 100644 index 000000000..52db021bf --- /dev/null +++ b/pkg/transform/timescale/actors/processor.go @@ -0,0 +1,71 @@ +package actors + +import ( + "context" + "fmt" + "io" + + "github.com/filecoin-project/go-state-types/abi" + actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v9" +) + +type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version + +func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVersionGetter) error { + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + adtStore := store.WrapBlockStore(ctx, bs) + + var actorIPLDContainer cbor.ActorIPLDContainer + if err := adtStore.Get(ctx, header.Roots[0], &actorIPLDContainer); err != nil { + return err + } + + current := actorIPLDContainer.CurrentTipSet + executed := actorIPLDContainer.ExecutedTipSet + av, err := core.ActorVersionForTipSet(ctx, current, nvg) + if err != nil { + return err + } + mapHandler := MakeMinerProcessor(av) + minerModels, err := mapHandler(ctx, bs, current, executed, actorIPLDContainer.MinerActors) + if err != nil { + return err + } + + return strg.PersistBatch(ctx, minerModels...) +} + +type MinerHandler = func(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) + +func MakeMinerProcessor(av actorstypes.Version) MinerHandler { + switch av { + case actorstypes.Version0: + return v0.MinerHandler + case actorstypes.Version2: + return v2.MinerHandler + case actorstypes.Version9: + return v9.MinerHandler + } + panic("developer error") +} diff --git a/pkg/transform/timescale/actors/types.go b/pkg/transform/timescale/actors/types.go new file mode 100644 index 000000000..1256222bc --- /dev/null +++ b/pkg/transform/timescale/actors/types.go @@ -0,0 +1 @@ +package actors diff --git a/pkg/transform/timescale/miner/router.go b/pkg/transform/timescale/miner/router.go deleted file mode 100644 index 9153e0b5e..000000000 --- a/pkg/transform/timescale/miner/router.go +++ /dev/null @@ -1,151 +0,0 @@ -package miner - -import ( - "context" - - "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/ipfs/go-cid" - typegen "github.com/whyrusleeping/cbor-gen" - - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" - cborminer "github.com/filecoin-project/lily/pkg/transform/cbor/miner" -) - -func DecodeMinerStateDiff(ctx context.Context, store adt.Store, change cborminer.StateChange) (*minerdiff.StateDiff, error) { - out := &minerdiff.StateDiff{ - PreCommitChanges: minerdiff.PreCommitChangeList{}, - SectorChanges: minerdiff.SectorChangeList{}, - SectorStatusChanges: &minerdiff.SectorStatusChange{ - Removed: bitfield.New(), - Recovering: bitfield.New(), - Faulted: bitfield.New(), - Recovered: bitfield.New(), - }, - } - if c := change.Info; c != nil { - info, err := DecodeMinerInfoChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.InfoChange = info - } - - if c := change.Funds; c != nil { - funds, err := DecodeMinerFundsChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.FundsChange = funds - } - - if c := change.Debt; c != nil { - debt, err := DecodeMinerDebtChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.DebtChange = debt - } - - if c := change.SectorStatus; c != nil { - sectorStatus, err := DecodeMinerSectorStatusChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.SectorStatusChanges = sectorStatus - } - - if c := change.Sectors; c != nil { - sectors, err := DecodeMinerSectorChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.SectorChanges = sectors - } - - if c := change.PreCommits; c != nil { - precommits, err := DecodeMinerPreCommitChanges(ctx, store, *c) - if err != nil { - return nil, err - } - out.PreCommitChanges = precommits - } - return out, nil -} - -func DecodeMinerInfoChanges(ctx context.Context, store adt.Store, info cid.Cid) (*minerdiff.InfoChange, error) { - var infoContainer cborminer.Info - if err := store.Get(ctx, info, &infoContainer); err != nil { - return nil, err - } - minfo := new(typegen.Deferred) - if err := store.Get(ctx, infoContainer.Info, minfo); err != nil { - return nil, err - } - return &minerdiff.InfoChange{ - Info: minfo, - Change: infoContainer.Change, - }, nil -} - -func DecodeMinerFundsChanges(ctx context.Context, store adt.Store, funds cid.Cid) (*minerdiff.FundsChange, error) { - minerFundsChange := new(minerdiff.FundsChange) - if err := store.Get(ctx, funds, minerFundsChange); err != nil { - return nil, err - } - return minerFundsChange, nil -} - -func DecodeMinerDebtChanges(ctx context.Context, store adt.Store, debt cid.Cid) (*minerdiff.DebtChange, error) { - var minerDebtChange *minerdiff.DebtChange - if err := store.Get(ctx, debt, minerDebtChange); err != nil { - return nil, err - } - return minerDebtChange, nil -} - -func DecodeMinerSectorStatusChanges(ctx context.Context, store adt.Store, sectorStatus cid.Cid) (*minerdiff.SectorStatusChange, error) { - minerSectorStatusChange := new(minerdiff.SectorStatusChange) - if err := store.Get(ctx, sectorStatus, minerSectorStatusChange); err != nil { - return nil, err - } - return minerSectorStatusChange, nil -} - -func DecodeMinerSectorChanges(ctx context.Context, store adt.Store, sectors cid.Cid) (minerdiff.SectorChangeList, error) { - sectorArr, err := adt.AsArray(store, sectors, 5) - if err != nil { - return nil, err - } - sectorChangeList := make(minerdiff.SectorChangeList, 0, sectorArr.Length()) - sectorChange := new(minerdiff.SectorChange) - if err := sectorArr.ForEach(sectorChange, func(sectorNumber int64) error { - // make a copy - val := new(minerdiff.SectorChange) - *val = *sectorChange - sectorChangeList = append(sectorChangeList, val) - return nil - }); err != nil { - return nil, err - } - return sectorChangeList, nil -} - -func DecodeMinerPreCommitChanges(ctx context.Context, store adt.Store, precommits cid.Cid) (minerdiff.PreCommitChangeList, error) { - precommitArr, err := adt.AsArray(store, precommits, 5) - if err != nil { - return nil, err - } - precommitChangeList := make(minerdiff.PreCommitChangeList, 0, precommitArr.Length()) - precommitChange := new(minerdiff.PreCommitChange) - if err := precommitArr.ForEach(precommitChange, func(sectorNumber int64) error { - // make a copy - val := new(minerdiff.PreCommitChange) - *val = *precommitChange - precommitChangeList = append(precommitChangeList, val) - return nil - }); err != nil { - return nil, err - } - return precommitChangeList, nil -} diff --git a/pkg/transform/timescale/miner/v8/sectors.go b/pkg/transform/timescale/miner/v8/sectors.go deleted file mode 100644 index 688b3064d..000000000 --- a/pkg/transform/timescale/miner/v8/sectors.go +++ /dev/null @@ -1,23 +0,0 @@ -package v8 - -import ( - "context" - - miner8 "github.com/filecoin-project/go-state-types/builtin/v8/miner" - - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" -) - -func HandleV8MinerSectorChanges(ctx context.Context, changes []minerdiff.SectorChange) ([]*miner8.SectorOnChainInfo, error) { - var sectors []*miner8.SectorOnChainInfo - for _, change := range changes { - if err := core.StateReadDeferred(ctx, *change.Current, func(sector *miner8.SectorOnChainInfo) error { - sectors = append(sectors, sector) - return nil - }); err != nil { - return nil, err - } - } - return sectors, nil -} diff --git a/pkg/transform/timescale/miner/v9/processor.go b/pkg/transform/timescale/miner/v9/processor.go deleted file mode 100644 index be77a6d15..000000000 --- a/pkg/transform/timescale/miner/v9/processor.go +++ /dev/null @@ -1,73 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" -) - -func ProcessMinerStateChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiff) (model.PersistableList, error) { - var ( - infoModel model.Persistable - fundsModel model.Persistable - debtModel model.Persistable - err error - ) - if changes.InfoChange != nil { - infoModel, err = HandleMinerInfo(ctx, store, current, executed, addr, changes.InfoChange) - if err != nil { - return nil, err - } - } - - if changes.FundsChange != nil { - fundsModel, err = HandleMinerFundsChange(ctx, store, current, executed, addr, changes.FundsChange) - if err != nil { - return nil, err - } - } - - if changes.DebtChange != nil { - debtModel, err = HandleMinerDebtChange(ctx, store, current, executed, addr, changes.DebtChange) - if err != nil { - return nil, err - } - } - - sectorModel, err := HandleMinerSectorChanges(ctx, store, current, executed, addr, changes.SectorChanges) - if err != nil { - return nil, err - } - - precommitModel, err := HandleMinerPreCommitChanges(ctx, store, current, executed, addr, changes.PreCommitChanges) - if err != nil { - return nil, err - } - - sectorEventModel, err := HandleMinerSectorEvents(ctx, store, current, executed, addr, changes.PreCommitChanges, changes.SectorChanges, changes.SectorStatusChanges) - if err != nil { - return nil, err - } - - sectorDealsModel, err := HandleMinerSectorDeals(ctx, store, current, executed, addr, changes.SectorChanges) - if err != nil { - return nil, err - } - return batchIfNotNil(infoModel, fundsModel, debtModel, sectorModel, precommitModel, sectorEventModel, sectorDealsModel), nil -} - -func batchIfNotNil(models ...model.Persistable) model.PersistableList { - out := make(model.PersistableList, 0, len(models)) - for _, m := range models { - if m == nil { - continue - } - out = append(out, m) - } - return out -} diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go deleted file mode 100644 index 0b8ff94fd..000000000 --- a/pkg/transform/timescale/processor.go +++ /dev/null @@ -1,60 +0,0 @@ -package timescale - -import ( - "context" - "fmt" - "io" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" - v1car "github.com/ipld/go-car" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/transform/cbor" - cborminer "github.com/filecoin-project/lily/pkg/transform/cbor/miner" - "github.com/filecoin-project/lily/pkg/transform/timescale/miner" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/miner/v9" -) - -func Process(ctx context.Context, r io.Reader, strg model.Storage) error { - bs := blockstore.NewMemorySync() - header, err := v1car.LoadCar(ctx, bs, r) - if err != nil { - return err - } - if len(header.Roots) != 1 { - return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) - } - - store := store.WrapBlockStore(ctx, bs) - - var actorIPLDContainer cbor.ActorIPLDContainer - if err := store.Get(ctx, header.Roots[0], &actorIPLDContainer); err != nil { - return err - } - - minerMap, err := adt.AsMap(store, actorIPLDContainer.MinerActors, 5) - if err != nil { - return err - } - - var minerState cborminer.StateChange - var minerModels model.PersistableList - if err := minerMap.ForEach(&minerState, func(key string) error { - stateChange, err := miner.DecodeMinerStateDiff(ctx, store, minerState) - if err != nil { - return err - } - minerModel, err := v9.ProcessMinerStateChanges(ctx, store, actorIPLDContainer.CurrentTipSet, actorIPLDContainer.ExecutedTipSet, minerState.Miner, stateChange) - if err != nil { - return err - } - minerModels = append(minerModels, minerModel...) - return nil - }); err != nil { - return err - } - - return strg.PersistBatch(ctx, minerModels...) -} From 39e755a5dababbf19bcc7e34d5badeff3e0e780d Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 10 Jan 2023 15:56:32 -0800 Subject: [PATCH 16/48] add init extract --- pkg/extract/actors/initdiff/v0/addresses.go | 61 ++++++++++++++++ pkg/extract/actors/initdiff/v0/load.go | 24 +++++++ pkg/extract/actors/initdiff/v0/state.go | 54 ++++++++++++++ pkg/extract/actors/initdiff/v2/addresses.go | 7 ++ pkg/extract/actors/initdiff/v2/state.go | 7 ++ pkg/extract/actors/initdiff/v3/addresses.go | 7 ++ pkg/extract/actors/initdiff/v3/state.go | 7 ++ pkg/extract/actors/initdiff/v4/addresses.go | 7 ++ pkg/extract/actors/initdiff/v4/state.go | 7 ++ pkg/extract/actors/initdiff/v5/addresses.go | 7 ++ pkg/extract/actors/initdiff/v5/state.go | 7 ++ pkg/extract/actors/initdiff/v6/addresses.go | 7 ++ pkg/extract/actors/initdiff/v6/state.go | 7 ++ pkg/extract/actors/initdiff/v7/addresses.go | 7 ++ pkg/extract/actors/initdiff/v7/state.go | 7 ++ pkg/extract/actors/initdiff/v8/addresses.go | 7 ++ pkg/extract/actors/initdiff/v8/state.go | 7 ++ pkg/extract/actors/initdiff/v9/addresses.go | 7 ++ pkg/extract/actors/initdiff/v9/state.go | 7 ++ pkg/extract/actors/initdiff/version.go | 71 +++++++++++++++++++ pkg/extract/actors/minerdiff/v0/load.go | 1 + pkg/extract/actors/minerdiff/v0/precommits.go | 2 +- pkg/extract/actors/minerdiff/v0/sectors.go | 2 +- pkg/extract/procesor/process.go | 29 +++++++- pkg/gen/main.go | 3 +- 25 files changed, 353 insertions(+), 6 deletions(-) create mode 100644 pkg/extract/actors/initdiff/v0/addresses.go create mode 100644 pkg/extract/actors/initdiff/v0/load.go create mode 100644 pkg/extract/actors/initdiff/v0/state.go create mode 100644 pkg/extract/actors/initdiff/v2/addresses.go create mode 100644 pkg/extract/actors/initdiff/v2/state.go create mode 100644 pkg/extract/actors/initdiff/v3/addresses.go create mode 100644 pkg/extract/actors/initdiff/v3/state.go create mode 100644 pkg/extract/actors/initdiff/v4/addresses.go create mode 100644 pkg/extract/actors/initdiff/v4/state.go create mode 100644 pkg/extract/actors/initdiff/v5/addresses.go create mode 100644 pkg/extract/actors/initdiff/v5/state.go create mode 100644 pkg/extract/actors/initdiff/v6/addresses.go create mode 100644 pkg/extract/actors/initdiff/v6/state.go create mode 100644 pkg/extract/actors/initdiff/v7/addresses.go create mode 100644 pkg/extract/actors/initdiff/v7/state.go create mode 100644 pkg/extract/actors/initdiff/v8/addresses.go create mode 100644 pkg/extract/actors/initdiff/v8/state.go create mode 100644 pkg/extract/actors/initdiff/v9/addresses.go create mode 100644 pkg/extract/actors/initdiff/v9/state.go create mode 100644 pkg/extract/actors/initdiff/version.go diff --git a/pkg/extract/actors/initdiff/v0/addresses.go b/pkg/extract/actors/initdiff/v0/addresses.go new file mode 100644 index 000000000..7f1abf78f --- /dev/null +++ b/pkg/extract/actors/initdiff/v0/addresses.go @@ -0,0 +1,61 @@ +package v0 + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/init") + +var _ actors.ActorStateChange = (*AddressChangeList)(nil) + +type AddressChange struct { + Address []byte + Current *typegen.Deferred // actorID + Previous *typegen.Deferred // actorID + Change core.ChangeType +} + +type AddressChangeList []*AddressChange + +const KindInitAddresses = "init_addresses" + +func (a AddressChangeList) Kind() actors.ActorStateKind { + return KindInitAddresses +} + +type Addresses struct{} + +func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindInitAddresses, zap.Inline(act), "duration", time.Since(start)) + }() + return AddressesDiff(ctx, api, act) +} + +func AddressesDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, InitStateLoader, InitAddressesMapLoader) + if err != nil { + return nil, err + } + out := make(AddressChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &AddressChange{ + Address: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/initdiff/v0/load.go b/pkg/extract/actors/initdiff/v0/load.go new file mode 100644 index 000000000..f063aabf5 --- /dev/null +++ b/pkg/extract/actors/initdiff/v0/load.go @@ -0,0 +1,24 @@ +package v0 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/init" +) + +var InitStateLoader = func(strore adt.Store, act *types.Actor) (interface{}, error) { + return init.Load(strore, act) +} + +var InitAddressesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + initState := m.(init.State) + addressesMap, err := initState.AddressMap() + if err != nil { + return nil, nil, err + } + return addressesMap, &adt.MapOpts{ + Bitwidth: initState.AddressMapBitWidth(), + HashFunc: initState.AddressMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/initdiff/v0/state.go b/pkg/extract/actors/initdiff/v0/state.go new file mode 100644 index 000000000..e27ead520 --- /dev/null +++ b/pkg/extract/actors/initdiff/v0/state.go @@ -0,0 +1,54 @@ +package v0 + +import ( + "context" + "time" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiffResult struct { + AddressesChanges AddressChangeList +} + +func (s *StateDiffResult) Kind() string { + return "init" +} + +func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + //TODO implement me + panic("implement me") +} + +type StateChange struct { + Addresses cid.Cid +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindInitAddresses: + stateDiff.AddressesChanges = stateChange.(AddressChangeList) + } + } + log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} diff --git a/pkg/extract/actors/initdiff/v2/addresses.go b/pkg/extract/actors/initdiff/v2/addresses.go new file mode 100644 index 000000000..2bf020b19 --- /dev/null +++ b/pkg/extract/actors/initdiff/v2/addresses.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v2/state.go b/pkg/extract/actors/initdiff/v2/state.go new file mode 100644 index 000000000..b0076ae9d --- /dev/null +++ b/pkg/extract/actors/initdiff/v2/state.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v3/addresses.go b/pkg/extract/actors/initdiff/v3/addresses.go new file mode 100644 index 000000000..a1369dc92 --- /dev/null +++ b/pkg/extract/actors/initdiff/v3/addresses.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v3/state.go b/pkg/extract/actors/initdiff/v3/state.go new file mode 100644 index 000000000..1b76abdf1 --- /dev/null +++ b/pkg/extract/actors/initdiff/v3/state.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v4/addresses.go b/pkg/extract/actors/initdiff/v4/addresses.go new file mode 100644 index 000000000..04a790c2c --- /dev/null +++ b/pkg/extract/actors/initdiff/v4/addresses.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v4/state.go b/pkg/extract/actors/initdiff/v4/state.go new file mode 100644 index 000000000..a5afc65b0 --- /dev/null +++ b/pkg/extract/actors/initdiff/v4/state.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v5/addresses.go b/pkg/extract/actors/initdiff/v5/addresses.go new file mode 100644 index 000000000..504767884 --- /dev/null +++ b/pkg/extract/actors/initdiff/v5/addresses.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v5/state.go b/pkg/extract/actors/initdiff/v5/state.go new file mode 100644 index 000000000..507a44a1b --- /dev/null +++ b/pkg/extract/actors/initdiff/v5/state.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v6/addresses.go b/pkg/extract/actors/initdiff/v6/addresses.go new file mode 100644 index 000000000..01a52c23a --- /dev/null +++ b/pkg/extract/actors/initdiff/v6/addresses.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v6/state.go b/pkg/extract/actors/initdiff/v6/state.go new file mode 100644 index 000000000..ac102b082 --- /dev/null +++ b/pkg/extract/actors/initdiff/v6/state.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v7/addresses.go b/pkg/extract/actors/initdiff/v7/addresses.go new file mode 100644 index 000000000..fdff66564 --- /dev/null +++ b/pkg/extract/actors/initdiff/v7/addresses.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v7/state.go b/pkg/extract/actors/initdiff/v7/state.go new file mode 100644 index 000000000..4d520263a --- /dev/null +++ b/pkg/extract/actors/initdiff/v7/state.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v8/addresses.go b/pkg/extract/actors/initdiff/v8/addresses.go new file mode 100644 index 000000000..8bcb05bfc --- /dev/null +++ b/pkg/extract/actors/initdiff/v8/addresses.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v8/state.go b/pkg/extract/actors/initdiff/v8/state.go new file mode 100644 index 000000000..d720d448f --- /dev/null +++ b/pkg/extract/actors/initdiff/v8/state.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v9/addresses.go b/pkg/extract/actors/initdiff/v9/addresses.go new file mode 100644 index 000000000..fdeb6c47c --- /dev/null +++ b/pkg/extract/actors/initdiff/v9/addresses.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type AddressesChange = v0.AddressChange +type AddressesChangeList = v0.AddressChangeList +type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v9/state.go b/pkg/extract/actors/initdiff/v9/state.go new file mode 100644 index 000000000..55056ae74 --- /dev/null +++ b/pkg/extract/actors/initdiff/v9/state.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + +type StateDiff = v0.StateDiff +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/version.go b/pkg/extract/actors/initdiff/version.go new file mode 100644 index 000000000..63f61d12d --- /dev/null +++ b/pkg/extract/actors/initdiff/version.go @@ -0,0 +1,71 @@ +package initdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v2" + v3 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v3" + v4 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v4" + v5 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v5" + v6 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v6" + v7 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v7" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v8" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version0: + return &v0.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v0.Addresses{}, + }}, nil + case actortypes.Version2: + return &v2.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v2.Addresses{}, + }}, nil + case actortypes.Version3: + return &v3.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v3.Addresses{}, + }}, nil + case actortypes.Version4: + return &v4.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v4.Addresses{}, + }}, nil + case actortypes.Version5: + return &v5.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v5.Addresses{}, + }}, nil + case actortypes.Version6: + return &v6.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v6.Addresses{}, + }}, nil + case actortypes.Version7: + return &v7.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v7.Addresses{}, + }}, nil + case actortypes.Version8: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v8.Addresses{}, + }}, nil + case actortypes.Version9: + return &v9.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Addresses{}, + }}, nil + case actortypes.Version10: + panic("NYI") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/minerdiff/v0/load.go b/pkg/extract/actors/minerdiff/v0/load.go index 974fbed48..7eefb1990 100644 --- a/pkg/extract/actors/minerdiff/v0/load.go +++ b/pkg/extract/actors/minerdiff/v0/load.go @@ -10,6 +10,7 @@ import ( var MinerStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { return miner.Load(store, act) } + var MinerPreCommitMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { minerState := m.(miner.State) perCommitMap, err := minerState.PrecommitsMap() diff --git a/pkg/extract/actors/minerdiff/v0/precommits.go b/pkg/extract/actors/minerdiff/v0/precommits.go index 01867a4f2..610f6fb7b 100644 --- a/pkg/extract/actors/minerdiff/v0/precommits.go +++ b/pkg/extract/actors/minerdiff/v0/precommits.go @@ -48,7 +48,7 @@ func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) } func (p *PreCommitChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { - precommitMap, err := adt.AsMap(store, root, 5) + precommitMap, err := adt.AsMap(store, root, bw) if err != nil { return err } diff --git a/pkg/extract/actors/minerdiff/v0/sectors.go b/pkg/extract/actors/minerdiff/v0/sectors.go index 342a42d77..10bc1ad22 100644 --- a/pkg/extract/actors/minerdiff/v0/sectors.go +++ b/pkg/extract/actors/minerdiff/v0/sectors.go @@ -47,7 +47,7 @@ func (s SectorChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { } func (s *SectorChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { - sectorMap, err := adt.AsMap(store, root, 5) + sectorMap, err := adt.AsMap(store, root, bw) if err != nil { return err } diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 4c735f141..096ff3360 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -14,12 +14,14 @@ import ( "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" + "github.com/filecoin-project/lily/chain/actors/builtin/init" "github.com/filecoin-project/lily/chain/actors/builtin/market" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/chain/actors/builtin/power" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" "github.com/filecoin-project/lily/pkg/extract/statetree" @@ -29,6 +31,7 @@ import ( var log = logging.Logger("lily/extract/processor") var ( + InitCodes = cid.NewSet() MinerCodes = cid.NewSet() PowerCodes = cid.NewSet() MarketCodes = cid.NewSet() @@ -48,6 +51,9 @@ func init() { for _, c := range verifreg.AllCodes() { VerifregCodes.Add(c) } + for _, c := range init.AllCodes() { + InitCodes.Add(c) + } } type ActorStateChanges struct { @@ -56,6 +62,7 @@ type ActorStateChanges struct { Actors map[address.Address]statetree.ActorDiff MinerActors map[address.Address]actors.ActorDiffResult VerifregActor actors.ActorDiffResult + InitActor actors.ActorDiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { @@ -129,7 +136,6 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current } } if VerifregCodes.Has(change.Current.Code) { - return nil start := time.Now() // construct the state differ required by this actor version actorDiff, err := verifregdiff.StateDiffFor(actorVersion) @@ -147,6 +153,22 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } + if InitCodes.Has(change.Current.Code) { + start := time.Now() + actorDiff, err := initdiff.StateDiffFor(actorVersion) + if err != nil { + return err + } + initChanges, err := actorDiff.State(ctx, api, act) + if err != nil { + return err + } + log.Infow("Extracted Init", "addresses", addr, "duration", time.Since(start)) + results <- &StateDiffResult{ + ActorDiff: initChanges, + Address: addr, + } + } return nil }) } @@ -158,11 +180,14 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current }() for stateDiff := range results { switch stateDiff.ActorDiff.Kind() { - case "miner": asc.MinerActors[stateDiff.Address] = stateDiff.ActorDiff case "verifreg": asc.VerifregActor = stateDiff.ActorDiff + case "init": + asc.InitActor = stateDiff.ActorDiff + default: + panic(stateDiff.ActorDiff.Kind()) } } return asc, nil diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 30d37815e..3f9f29906 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -4,7 +4,6 @@ import ( cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - "github.com/filecoin-project/lily/pkg/transform/cbor/miner/v9" ) const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" @@ -24,7 +23,7 @@ func main() { v0.FundsChange{}, v0.DebtChange{}, v0.InfoChange{}, - v9.StateChange{}, + v0.StateChange{}, ); err != nil { panic(err) } From 11bc7c58f9ec6e9822dabe3f005ffab587e4c50d Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 10 Jan 2023 16:51:40 -0800 Subject: [PATCH 17/48] add power diff --- pkg/extract/actors/powerdiff/v0/claims.go | 59 ++++++++++++++++ pkg/extract/actors/powerdiff/v0/load.go | 24 +++++++ pkg/extract/actors/powerdiff/v0/state.go | 56 +++++++++++++++ pkg/extract/actors/powerdiff/v2/claims.go | 7 ++ pkg/extract/actors/powerdiff/v2/state.go | 7 ++ pkg/extract/actors/powerdiff/v3/claims.go | 7 ++ pkg/extract/actors/powerdiff/v3/state.go | 7 ++ pkg/extract/actors/powerdiff/v4/claims.go | 7 ++ pkg/extract/actors/powerdiff/v4/state.go | 7 ++ pkg/extract/actors/powerdiff/v5/claims.go | 7 ++ pkg/extract/actors/powerdiff/v5/state.go | 7 ++ pkg/extract/actors/powerdiff/v6/claims.go | 7 ++ pkg/extract/actors/powerdiff/v6/state.go | 7 ++ pkg/extract/actors/powerdiff/v7/claims.go | 7 ++ pkg/extract/actors/powerdiff/v7/state.go | 7 ++ pkg/extract/actors/powerdiff/v8/claims.go | 7 ++ pkg/extract/actors/powerdiff/v8/state.go | 7 ++ pkg/extract/actors/powerdiff/v9/claims.go | 7 ++ pkg/extract/actors/powerdiff/v9/state.go | 7 ++ pkg/extract/actors/powerdiff/version.go | 71 ++++++++++++++++++++ pkg/extract/actors/verifregdiff/v0/load.go | 12 ---- pkg/extract/actors/verifregdiff/v9/claims.go | 4 +- pkg/extract/actors/verifregdiff/v9/load.go | 18 +++++ pkg/extract/procesor/process.go | 22 +++++- 24 files changed, 363 insertions(+), 15 deletions(-) create mode 100644 pkg/extract/actors/powerdiff/v0/claims.go create mode 100644 pkg/extract/actors/powerdiff/v0/load.go create mode 100644 pkg/extract/actors/powerdiff/v0/state.go create mode 100644 pkg/extract/actors/powerdiff/v2/claims.go create mode 100644 pkg/extract/actors/powerdiff/v2/state.go create mode 100644 pkg/extract/actors/powerdiff/v3/claims.go create mode 100644 pkg/extract/actors/powerdiff/v3/state.go create mode 100644 pkg/extract/actors/powerdiff/v4/claims.go create mode 100644 pkg/extract/actors/powerdiff/v4/state.go create mode 100644 pkg/extract/actors/powerdiff/v5/claims.go create mode 100644 pkg/extract/actors/powerdiff/v5/state.go create mode 100644 pkg/extract/actors/powerdiff/v6/claims.go create mode 100644 pkg/extract/actors/powerdiff/v6/state.go create mode 100644 pkg/extract/actors/powerdiff/v7/claims.go create mode 100644 pkg/extract/actors/powerdiff/v7/state.go create mode 100644 pkg/extract/actors/powerdiff/v8/claims.go create mode 100644 pkg/extract/actors/powerdiff/v8/state.go create mode 100644 pkg/extract/actors/powerdiff/v9/claims.go create mode 100644 pkg/extract/actors/powerdiff/v9/state.go create mode 100644 pkg/extract/actors/powerdiff/version.go create mode 100644 pkg/extract/actors/verifregdiff/v9/load.go diff --git a/pkg/extract/actors/powerdiff/v0/claims.go b/pkg/extract/actors/powerdiff/v0/claims.go new file mode 100644 index 000000000..a2e8cb4d3 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v0/claims.go @@ -0,0 +1,59 @@ +package v0 + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/power") + +type ClaimsChange struct { + Miner []byte + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType +} + +type ClaimsChangeList []*ClaimsChange + +const KindPowerClaims = "power_claims" + +func (c ClaimsChangeList) Kind() actors.ActorStateKind { + return KindPowerClaims +} + +type Claims struct{} + +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindPowerClaims, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffClaims(ctx, api, act) +} + +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, PowerStateLoader, PowerClaimsMapLoader) + if err != nil { + return nil, err + } + out := make(ClaimsChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &ClaimsChange{ + Miner: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/powerdiff/v0/load.go b/pkg/extract/actors/powerdiff/v0/load.go new file mode 100644 index 000000000..1ac9026af --- /dev/null +++ b/pkg/extract/actors/powerdiff/v0/load.go @@ -0,0 +1,24 @@ +package v0 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/power" +) + +var PowerStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return power.Load(store, act) +} + +var PowerClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + powerState := m.(power.State) + claimsMap, err := powerState.ClaimsMap() + if err != nil { + return nil, nil, err + } + return claimsMap, &adt.MapOpts{ + Bitwidth: powerState.ClaimsMapBitWidth(), + HashFunc: powerState.ClaimsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v0/state.go new file mode 100644 index 000000000..bed352755 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v0/state.go @@ -0,0 +1,56 @@ +package v0 + +import ( + "context" + "time" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiffResult struct { + ClaimsChanges ClaimsChangeList +} + +func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + //TODO implement me + panic("implement me") +} + +func (s *StateDiffResult) Kind() string { + return "power" +} + +type StateChange struct { + Claims cid.Cid +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindPowerClaims: + stateDiff.ClaimsChanges = stateChange.(ClaimsChangeList) + default: + panic(stateChange.Kind()) + } + } + log.Infow("Extracted Power State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} diff --git a/pkg/extract/actors/powerdiff/v2/claims.go b/pkg/extract/actors/powerdiff/v2/claims.go new file mode 100644 index 000000000..bf063933b --- /dev/null +++ b/pkg/extract/actors/powerdiff/v2/claims.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v2/state.go b/pkg/extract/actors/powerdiff/v2/state.go new file mode 100644 index 000000000..08bb44e09 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v2/state.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v3/claims.go b/pkg/extract/actors/powerdiff/v3/claims.go new file mode 100644 index 000000000..29bff2844 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v3/claims.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v3/state.go b/pkg/extract/actors/powerdiff/v3/state.go new file mode 100644 index 000000000..f79f6693e --- /dev/null +++ b/pkg/extract/actors/powerdiff/v3/state.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v4/claims.go b/pkg/extract/actors/powerdiff/v4/claims.go new file mode 100644 index 000000000..8cf17b525 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v4/claims.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v4/state.go b/pkg/extract/actors/powerdiff/v4/state.go new file mode 100644 index 000000000..42701a660 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v4/state.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v5/claims.go b/pkg/extract/actors/powerdiff/v5/claims.go new file mode 100644 index 000000000..4fc36881f --- /dev/null +++ b/pkg/extract/actors/powerdiff/v5/claims.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v5/state.go b/pkg/extract/actors/powerdiff/v5/state.go new file mode 100644 index 000000000..702bc2916 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v5/state.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v6/claims.go b/pkg/extract/actors/powerdiff/v6/claims.go new file mode 100644 index 000000000..776615258 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v6/claims.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v6/state.go b/pkg/extract/actors/powerdiff/v6/state.go new file mode 100644 index 000000000..43c9922df --- /dev/null +++ b/pkg/extract/actors/powerdiff/v6/state.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v7/claims.go b/pkg/extract/actors/powerdiff/v7/claims.go new file mode 100644 index 000000000..e9b8d03c7 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v7/claims.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v7/state.go b/pkg/extract/actors/powerdiff/v7/state.go new file mode 100644 index 000000000..0b4811439 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v7/state.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v8/claims.go b/pkg/extract/actors/powerdiff/v8/claims.go new file mode 100644 index 000000000..b8e9708f0 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v8/claims.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v8/state.go b/pkg/extract/actors/powerdiff/v8/state.go new file mode 100644 index 000000000..21c051563 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v8/state.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v9/claims.go b/pkg/extract/actors/powerdiff/v9/claims.go new file mode 100644 index 000000000..35da6a1d1 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v9/claims.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type ClaimsChange = v0.ClaimsChange +type ClaimsChangeList = v0.ClaimsChangeList +type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v9/state.go b/pkg/extract/actors/powerdiff/v9/state.go new file mode 100644 index 000000000..1267b3707 --- /dev/null +++ b/pkg/extract/actors/powerdiff/v9/state.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/version.go b/pkg/extract/actors/powerdiff/version.go new file mode 100644 index 000000000..1d74a7696 --- /dev/null +++ b/pkg/extract/actors/powerdiff/version.go @@ -0,0 +1,71 @@ +package powerdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v2" + v3 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v3" + v4 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v4" + v5 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v5" + v6 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v6" + v7 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v7" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v8" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version0: + return &v0.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v0.Claims{}, + }}, nil + case actortypes.Version2: + return &v2.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v2.Claims{}, + }}, nil + case actortypes.Version3: + return &v3.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v3.Claims{}, + }}, nil + case actortypes.Version4: + return &v4.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v4.Claims{}, + }}, nil + case actortypes.Version5: + return &v5.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v5.Claims{}, + }}, nil + case actortypes.Version6: + return &v6.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v6.Claims{}, + }}, nil + case actortypes.Version7: + return &v7.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v7.Claims{}, + }}, nil + case actortypes.Version8: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v8.Claims{}, + }}, nil + case actortypes.Version9: + return &v9.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Claims{}, + }}, nil + case actortypes.Version10: + panic("NYI") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/actors/verifregdiff/v0/load.go b/pkg/extract/actors/verifregdiff/v0/load.go index e0bc4bed6..2722c6ae6 100644 --- a/pkg/extract/actors/verifregdiff/v0/load.go +++ b/pkg/extract/actors/verifregdiff/v0/load.go @@ -35,15 +35,3 @@ var VerifiregClientsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, erro HashFunc: verifregState.VerifiedClientsMapHashFunction(), }, nil } - -var VerifiiregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { - verifregState := m.(verifreg.State) - claimsMap, err := verifregState.ClaimsMap() - if err != nil { - return nil, nil, err - } - return claimsMap, &adt.MapOpts{ - Bitwidth: verifregState.ClaimsMapBitWidth(), - HashFunc: verifregState.ClaimsMapHashFunction(), - }, nil -} diff --git a/pkg/extract/actors/verifregdiff/v9/claims.go b/pkg/extract/actors/verifregdiff/v9/claims.go index 4b370da0c..44233f633 100644 --- a/pkg/extract/actors/verifregdiff/v9/claims.go +++ b/pkg/extract/actors/verifregdiff/v9/claims.go @@ -10,7 +10,7 @@ import ( "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" - "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" @@ -47,7 +47,7 @@ func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorC } func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, v0.VerifiiregClaimsMapLoader) + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifiiregClaimsMapLoader) if err != nil { return nil, err } diff --git a/pkg/extract/actors/verifregdiff/v9/load.go b/pkg/extract/actors/verifregdiff/v9/load.go new file mode 100644 index 000000000..b5ddea1f5 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v9/load.go @@ -0,0 +1,18 @@ +package v9 + +import ( + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" +) + +var VerifiiregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + claimsMap, err := verifregState.ClaimsMap() + if err != nil { + return nil, nil, err + } + return claimsMap, &adt.MapOpts{ + Bitwidth: verifregState.ClaimsMapBitWidth(), + HashFunc: verifregState.ClaimsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 096ff3360..07ec7c14a 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -23,6 +23,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" "github.com/filecoin-project/lily/pkg/extract/statetree" "github.com/filecoin-project/lily/tasks" @@ -63,6 +64,7 @@ type ActorStateChanges struct { MinerActors map[address.Address]actors.ActorDiffResult VerifregActor actors.ActorDiffResult InitActor actors.ActorDiffResult + PowerActor actors.ActorDiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { @@ -163,12 +165,28 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current if err != nil { return err } - log.Infow("Extracted Init", "addresses", addr, "duration", time.Since(start)) + log.Infow("Extracted Init", "address", addr, "duration", time.Since(start)) results <- &StateDiffResult{ ActorDiff: initChanges, Address: addr, } } + if PowerCodes.Has(change.Current.Code) { + start := time.Now() + actorDiff, err := powerdiff.StateDiffFor(actorVersion) + if err != nil { + return err + } + powerChanges, err := actorDiff.State(ctx, api, act) + if err != nil { + return err + } + log.Infow("Extracted Power", "address", addr, "duration", time.Since(start)) + results <- &StateDiffResult{ + ActorDiff: powerChanges, + Address: addr, + } + } return nil }) } @@ -186,6 +204,8 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current asc.VerifregActor = stateDiff.ActorDiff case "init": asc.InitActor = stateDiff.ActorDiff + case "power": + asc.PowerActor = stateDiff.ActorDiff default: panic(stateDiff.ActorDiff.Kind()) } From bd16e3f311be917ef24dae36998bd8d8fee51e8e Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 10 Jan 2023 18:04:40 -0800 Subject: [PATCH 18/48] add market diff --- pkg/extract/actors/actordiff/actor.go | 83 +++++++++++++++++++ pkg/extract/actors/actordiff/state.go | 54 ++++++++++++ pkg/extract/actors/marketdiff/v0/deals.go | 56 +++++++++++++ pkg/extract/actors/marketdiff/v0/proposals.go | 59 +++++++++++++ pkg/extract/actors/marketdiff/v0/state.go | 60 ++++++++++++++ pkg/extract/actors/marketdiff/v2/deals.go | 7 ++ pkg/extract/actors/marketdiff/v2/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v2/state.go | 7 ++ pkg/extract/actors/marketdiff/v3/deals.go | 7 ++ pkg/extract/actors/marketdiff/v3/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v3/state.go | 7 ++ pkg/extract/actors/marketdiff/v4/deals.go | 7 ++ pkg/extract/actors/marketdiff/v4/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v4/state.go | 7 ++ pkg/extract/actors/marketdiff/v5/deals.go | 7 ++ pkg/extract/actors/marketdiff/v5/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v5/state.go | 7 ++ pkg/extract/actors/marketdiff/v6/deals.go | 7 ++ pkg/extract/actors/marketdiff/v6/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v6/state.go | 7 ++ pkg/extract/actors/marketdiff/v7/deals.go | 7 ++ pkg/extract/actors/marketdiff/v7/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v7/state.go | 7 ++ pkg/extract/actors/marketdiff/v8/deals.go | 7 ++ pkg/extract/actors/marketdiff/v8/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v8/state.go | 7 ++ pkg/extract/actors/marketdiff/v9/deals.go | 7 ++ pkg/extract/actors/marketdiff/v9/proposals.go | 7 ++ pkg/extract/actors/marketdiff/v9/state.go | 7 ++ pkg/extract/actors/marketdiff/version.go | 80 ++++++++++++++++++ pkg/extract/procesor/process.go | 43 ++++++++-- 31 files changed, 598 insertions(+), 5 deletions(-) create mode 100644 pkg/extract/actors/actordiff/actor.go create mode 100644 pkg/extract/actors/actordiff/state.go create mode 100644 pkg/extract/actors/marketdiff/v0/deals.go create mode 100644 pkg/extract/actors/marketdiff/v0/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v0/state.go create mode 100644 pkg/extract/actors/marketdiff/v2/deals.go create mode 100644 pkg/extract/actors/marketdiff/v2/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v2/state.go create mode 100644 pkg/extract/actors/marketdiff/v3/deals.go create mode 100644 pkg/extract/actors/marketdiff/v3/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v3/state.go create mode 100644 pkg/extract/actors/marketdiff/v4/deals.go create mode 100644 pkg/extract/actors/marketdiff/v4/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v4/state.go create mode 100644 pkg/extract/actors/marketdiff/v5/deals.go create mode 100644 pkg/extract/actors/marketdiff/v5/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v5/state.go create mode 100644 pkg/extract/actors/marketdiff/v6/deals.go create mode 100644 pkg/extract/actors/marketdiff/v6/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v6/state.go create mode 100644 pkg/extract/actors/marketdiff/v7/deals.go create mode 100644 pkg/extract/actors/marketdiff/v7/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v7/state.go create mode 100644 pkg/extract/actors/marketdiff/v8/deals.go create mode 100644 pkg/extract/actors/marketdiff/v8/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v8/state.go create mode 100644 pkg/extract/actors/marketdiff/v9/deals.go create mode 100644 pkg/extract/actors/marketdiff/v9/proposals.go create mode 100644 pkg/extract/actors/marketdiff/v9/state.go create mode 100644 pkg/extract/actors/marketdiff/version.go diff --git a/pkg/extract/actors/actordiff/actor.go b/pkg/extract/actors/actordiff/actor.go new file mode 100644 index 000000000..bd5e99863 --- /dev/null +++ b/pkg/extract/actors/actordiff/actor.go @@ -0,0 +1,83 @@ +package actordiff + +import ( + "context" + "fmt" + "time" + + "github.com/filecoin-project/lotus/chain/types" + logging "github.com/ipfs/go-log/v2" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/actor") + +type ActorChange struct { + Actor *types.Actor + Current []byte + Previous []byte + Change core.ChangeType +} + +const KindActorChange = "actor_change" + +func (a *ActorChange) Kind() actors.ActorStateKind { + return KindActorChange +} + +type Actor struct{} + +func (Actor) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindActorChange, zap.Inline(act), "duration", time.Since(start)) + }() + return ActorDiff(ctx, api, act) +} + +func ActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + switch act.Type { + case core.ChangeTypeAdd: + currentState, err := api.ChainReadObj(ctx, act.Current.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Current, + Current: currentState, + Previous: nil, + Change: act.Type, + }, nil + case core.ChangeTypeRemove: + executedState, err := api.ChainReadObj(ctx, act.Executed.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Executed, + Current: nil, + Previous: executedState, + Change: act.Type, + }, nil + case core.ChangeTypeModify: + currentState, err := api.ChainReadObj(ctx, act.Current.Head) + if err != nil { + return nil, err + } + executedState, err := api.ChainReadObj(ctx, act.Executed.Head) + if err != nil { + return nil, err + } + return &ActorChange{ + Actor: act.Current, + Current: currentState, + Previous: executedState, + Change: act.Type, + }, nil + } + return nil, fmt.Errorf("unknown actor change type %s", act.Type) +} diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/actordiff/state.go new file mode 100644 index 000000000..35f190b28 --- /dev/null +++ b/pkg/extract/actors/actordiff/state.go @@ -0,0 +1,54 @@ +package actordiff + +import ( + "context" + "time" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiffResult struct { + ActorStateChanges *ActorChange +} + +func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + //TODO implement me + panic("implement me") +} + +func (s *StateDiffResult) Kind() string { + return "actor" +} + +type StateChange struct { + ActorStates cid.Cid // Hamt[address]StateChange +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindActorChange: + stateDiff.ActorStateChanges = stateChange.(*ActorChange) + } + } + log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} diff --git a/pkg/extract/actors/marketdiff/v0/deals.go b/pkg/extract/actors/marketdiff/v0/deals.go new file mode 100644 index 000000000..58d223738 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v0/deals.go @@ -0,0 +1,56 @@ +package v0 + +import ( + "context" + "time" + + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type DealChange struct { + DealID uint64 + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType +} + +type DealChangeList []*DealChange + +const KindMarketDeal = "market_deal" + +func (p DealChangeList) Kind() actors.ActorStateKind { + return KindMarketDeal +} + +type Deals struct{} + +func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMarketDeal, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffDeals(ctx, api, act) +} + +func DiffDeals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + arrayChanges, err := generic.DiffActorArray(ctx, api, act, nil, nil) + if err != nil { + return nil, err + } + out := make(DealChangeList, len(arrayChanges)) + for i, change := range arrayChanges { + out[i] = &DealChange{ + DealID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/marketdiff/v0/proposals.go b/pkg/extract/actors/marketdiff/v0/proposals.go new file mode 100644 index 000000000..10d53e0c8 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v0/proposals.go @@ -0,0 +1,59 @@ +package v0 + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/market") + +type ProposalChange struct { + DealID uint64 + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType +} + +type ProposalChangeList []*ProposalChange + +const KindMarketProposal = "market_proposal" + +func (p ProposalChangeList) Kind() actors.ActorStateKind { + return KindMarketProposal +} + +type Proposals struct{} + +func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindMarketProposal, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffProposals(ctx, api, act) +} + +func DiffProposals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + arrayChanges, err := generic.DiffActorArray(ctx, api, act, nil, nil) + if err != nil { + return nil, err + } + out := make(ProposalChangeList, len(arrayChanges)) + for i, change := range arrayChanges { + out[i] = &ProposalChange{ + DealID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/marketdiff/v0/state.go b/pkg/extract/actors/marketdiff/v0/state.go new file mode 100644 index 000000000..c81d92516 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v0/state.go @@ -0,0 +1,60 @@ +package v0 + +import ( + "context" + "time" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiffResult struct { + DealStateChanges DealChangeList + DealProposalChanges ProposalChangeList +} + +func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + //TODO implement me + panic("implement me") +} + +func (s *StateDiffResult) Kind() string { + return "market" +} + +type StateChange struct { + Deals cid.Cid + Proposals cid.Cid +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindMarketDeal: + stateDiff.DealStateChanges = stateChange.(DealChangeList) + case KindMarketProposal: + stateDiff.DealProposalChanges = stateChange.(ProposalChangeList) + default: + panic(stateChange.Kind()) + } + } + log.Infow("Extracted Market State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} diff --git a/pkg/extract/actors/marketdiff/v2/deals.go b/pkg/extract/actors/marketdiff/v2/deals.go new file mode 100644 index 000000000..d22d33dfb --- /dev/null +++ b/pkg/extract/actors/marketdiff/v2/deals.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v2/proposals.go b/pkg/extract/actors/marketdiff/v2/proposals.go new file mode 100644 index 000000000..a973d6369 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v2/proposals.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v2/state.go b/pkg/extract/actors/marketdiff/v2/state.go new file mode 100644 index 000000000..bd66d0927 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v2/state.go @@ -0,0 +1,7 @@ +package v2 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v3/deals.go b/pkg/extract/actors/marketdiff/v3/deals.go new file mode 100644 index 000000000..713acc401 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v3/deals.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v3/proposals.go b/pkg/extract/actors/marketdiff/v3/proposals.go new file mode 100644 index 000000000..1c07caf57 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v3/proposals.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v3/state.go b/pkg/extract/actors/marketdiff/v3/state.go new file mode 100644 index 000000000..c03e05cba --- /dev/null +++ b/pkg/extract/actors/marketdiff/v3/state.go @@ -0,0 +1,7 @@ +package v3 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v4/deals.go b/pkg/extract/actors/marketdiff/v4/deals.go new file mode 100644 index 000000000..eacc1bb7c --- /dev/null +++ b/pkg/extract/actors/marketdiff/v4/deals.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v4/proposals.go b/pkg/extract/actors/marketdiff/v4/proposals.go new file mode 100644 index 000000000..b0d1bb3aa --- /dev/null +++ b/pkg/extract/actors/marketdiff/v4/proposals.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v4/state.go b/pkg/extract/actors/marketdiff/v4/state.go new file mode 100644 index 000000000..f9171ada2 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v4/state.go @@ -0,0 +1,7 @@ +package v4 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v5/deals.go b/pkg/extract/actors/marketdiff/v5/deals.go new file mode 100644 index 000000000..b8d8e5010 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v5/deals.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v5/proposals.go b/pkg/extract/actors/marketdiff/v5/proposals.go new file mode 100644 index 000000000..035a9c6c3 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v5/proposals.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v5/state.go b/pkg/extract/actors/marketdiff/v5/state.go new file mode 100644 index 000000000..25c60e343 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v5/state.go @@ -0,0 +1,7 @@ +package v5 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v6/deals.go b/pkg/extract/actors/marketdiff/v6/deals.go new file mode 100644 index 000000000..52d05f04e --- /dev/null +++ b/pkg/extract/actors/marketdiff/v6/deals.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v6/proposals.go b/pkg/extract/actors/marketdiff/v6/proposals.go new file mode 100644 index 000000000..ba831d475 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v6/proposals.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v6/state.go b/pkg/extract/actors/marketdiff/v6/state.go new file mode 100644 index 000000000..ba9540cb3 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v6/state.go @@ -0,0 +1,7 @@ +package v6 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v7/deals.go b/pkg/extract/actors/marketdiff/v7/deals.go new file mode 100644 index 000000000..c3af174c8 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v7/deals.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v7/proposals.go b/pkg/extract/actors/marketdiff/v7/proposals.go new file mode 100644 index 000000000..312694afd --- /dev/null +++ b/pkg/extract/actors/marketdiff/v7/proposals.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v7/state.go b/pkg/extract/actors/marketdiff/v7/state.go new file mode 100644 index 000000000..710e10d11 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v7/state.go @@ -0,0 +1,7 @@ +package v7 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v8/deals.go b/pkg/extract/actors/marketdiff/v8/deals.go new file mode 100644 index 000000000..64257a047 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v8/deals.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v8/proposals.go b/pkg/extract/actors/marketdiff/v8/proposals.go new file mode 100644 index 000000000..f884016a9 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v8/proposals.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v8/state.go b/pkg/extract/actors/marketdiff/v8/state.go new file mode 100644 index 000000000..174d1bac5 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v8/state.go @@ -0,0 +1,7 @@ +package v8 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v9/deals.go b/pkg/extract/actors/marketdiff/v9/deals.go new file mode 100644 index 000000000..5f0685444 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v9/deals.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type DealChange = v0.DealChange +type DealChangeList = v0.DealChangeList +type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v9/proposals.go b/pkg/extract/actors/marketdiff/v9/proposals.go new file mode 100644 index 000000000..b7e27f8f0 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v9/proposals.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type ProposalChange = v0.ProposalChange +type ProposalChangeList = v0.ProposalChangeList +type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v9/state.go b/pkg/extract/actors/marketdiff/v9/state.go new file mode 100644 index 000000000..fa3914096 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v9/state.go @@ -0,0 +1,7 @@ +package v9 + +import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + +type StateDiffResult = v0.StateDiffResult +type StateChange = v0.StateChange +type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/version.go b/pkg/extract/actors/marketdiff/version.go new file mode 100644 index 000000000..98e5123ab --- /dev/null +++ b/pkg/extract/actors/marketdiff/version.go @@ -0,0 +1,80 @@ +package marketdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + v2 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" + v3 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" + v4 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" + v5 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" + v6 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" + v7 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" + v8 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version0: + return &v0.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v0.Deals{}, + v0.Proposals{}, + }}, nil + case actortypes.Version2: + return &v2.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v2.Deals{}, + v2.Proposals{}, + }}, nil + case actortypes.Version3: + return &v3.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v3.Deals{}, + v3.Proposals{}, + }}, nil + case actortypes.Version4: + return &v4.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v4.Deals{}, + v4.Proposals{}, + }}, nil + case actortypes.Version5: + return &v5.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v5.Deals{}, + v5.Proposals{}, + }}, nil + case actortypes.Version6: + return &v6.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v6.Deals{}, + v6.Proposals{}, + }}, nil + case actortypes.Version7: + return &v7.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v7.Deals{}, + v7.Proposals{}, + }}, nil + case actortypes.Version8: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v8.Deals{}, + v8.Proposals{}, + }}, nil + case actortypes.Version9: + return &v8.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Deals{}, + v9.Proposals{}, + }}, nil + case actortypes.Version10: + panic("NYI") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 07ec7c14a..6f99ae129 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -21,7 +21,9 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" + "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" @@ -60,18 +62,18 @@ func init() { type ActorStateChanges struct { Current *types.TipSet Executed *types.TipSet - Actors map[address.Address]statetree.ActorDiff + ActorStates map[address.Address]actors.ActorDiffResult MinerActors map[address.Address]actors.ActorDiffResult VerifregActor actors.ActorDiffResult InitActor actors.ActorDiffResult PowerActor actors.ActorDiffResult + MarketActor actors.ActorDiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ attribute.Int64("current", int64(a.Current.Height())), attribute.Int64("executed", int64(a.Executed.Height())), - attribute.Int("actor_change", len(a.Actors)), attribute.Int("miner_changes", len(a.MinerActors)), } } @@ -98,8 +100,8 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current asc := &ActorStateChanges{ Current: current, Executed: executed, - Actors: actorChanges, MinerActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries + ActorStates: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } actorVersion, err := core.ActorVersionForTipSet(ctx, current, nvg) @@ -119,6 +121,17 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Type: change.ChangeType, } grp.Go(func() error { + actorDiff := &actordiff.StateDiff{ + DiffMethods: []actors.ActorStateDiff{actordiff.Actor{}}, + } + actorStateChanges, err := actorDiff.State(grpCtx, api, act) + if err != nil { + return err + } + results <- &StateDiffResult{ + ActorDiff: actorStateChanges, + Address: addr, + } if MinerCodes.Has(change.Current.Code) { start := time.Now() // construct the state differ required by this actor version @@ -161,7 +174,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current if err != nil { return err } - initChanges, err := actorDiff.State(ctx, api, act) + initChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { return err } @@ -177,7 +190,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current if err != nil { return err } - powerChanges, err := actorDiff.State(ctx, api, act) + powerChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { return err } @@ -187,6 +200,22 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } + if MarketCodes.Has(change.Current.Code) { + start := time.Now() + actorDiff, err := marketdiff.StateDiffFor(actorVersion) + if err != nil { + return err + } + marketChanges, err := actorDiff.State(grpCtx, api, act) + if err != nil { + return err + } + log.Infow("Extracted Market", "address", addr, "duration", time.Since(start)) + results <- &StateDiffResult{ + ActorDiff: marketChanges, + Address: addr, + } + } return nil }) } @@ -198,6 +227,8 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current }() for stateDiff := range results { switch stateDiff.ActorDiff.Kind() { + case "actor": + asc.ActorStates[stateDiff.Address] = stateDiff.ActorDiff case "miner": asc.MinerActors[stateDiff.Address] = stateDiff.ActorDiff case "verifreg": @@ -206,6 +237,8 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current asc.InitActor = stateDiff.ActorDiff case "power": asc.PowerActor = stateDiff.ActorDiff + case "market": + asc.MarketActor = stateDiff.ActorDiff default: panic(stateDiff.ActorDiff.Kind()) } From cfa9ae556ac5c0807947542c53c7e169454cf88e Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 12 Jan 2023 09:09:45 -0800 Subject: [PATCH 19/48] WIP --- pkg/extract/actors/actordiff/actor.go | 40 +- pkg/extract/actors/actordiff/cbor_gen.go | 332 +++++++++ pkg/extract/actors/actordiff/state.go | 20 +- pkg/extract/actors/initdiff/v0/addresses.go | 28 +- pkg/extract/actors/initdiff/v0/cbor_gen.go | 303 ++++++++ pkg/extract/actors/initdiff/v0/load.go | 6 +- pkg/extract/actors/initdiff/v0/state.go | 16 +- pkg/extract/actors/minerdiff/v0/cbor_gen.go | 3 +- .../actors/verifregdiff/v0/cbor_gen.go | 523 ++++++++++++++ pkg/extract/actors/verifregdiff/v0/clients.go | 28 +- pkg/extract/actors/verifregdiff/v0/state.go | 25 +- .../actors/verifregdiff/v0/verifiers.go | 28 +- .../actors/verifregdiff/v9/allocations.go | 10 +- .../actors/verifregdiff/v9/cbor_gen.go | 644 ++++++++++++++++++ pkg/extract/actors/verifregdiff/v9/claims.go | 10 +- pkg/extract/actors/verifregdiff/v9/state.go | 17 +- pkg/extract/procesor/process.go | 4 +- pkg/gen/main.go | 77 ++- pkg/transform/cbor/actor/state.go | 33 + pkg/transform/cbor/init_/state.go | 20 + pkg/transform/cbor/router.go | 25 +- 21 files changed, 2117 insertions(+), 75 deletions(-) create mode 100644 pkg/extract/actors/actordiff/cbor_gen.go create mode 100644 pkg/extract/actors/initdiff/v0/cbor_gen.go create mode 100644 pkg/extract/actors/verifregdiff/v0/cbor_gen.go create mode 100644 pkg/extract/actors/verifregdiff/v9/cbor_gen.go create mode 100644 pkg/transform/cbor/actor/state.go create mode 100644 pkg/transform/cbor/init_/state.go diff --git a/pkg/extract/actors/actordiff/actor.go b/pkg/extract/actors/actordiff/actor.go index bd5e99863..9801b640e 100644 --- a/pkg/extract/actors/actordiff/actor.go +++ b/pkg/extract/actors/actordiff/actor.go @@ -1,11 +1,15 @@ package actordiff import ( + "bytes" "context" "fmt" "time" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + block "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.uber.org/zap" @@ -17,10 +21,38 @@ import ( var log = logging.Logger("extract/actors/actor") type ActorChange struct { - Actor *types.Actor - Current []byte - Previous []byte - Change core.ChangeType + Actor *types.Actor `cborgen:"actor"` + Current []byte `cborgen:"current_state"` + Previous []byte `cborgen:"previous_state"` + Change core.ChangeType `cborgen:"change"` +} + +func (a *ActorChange) Serialize() ([]byte, error) { + buf := new(bytes.Buffer) + if err := a.MarshalCBOR(buf); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (a *ActorChange) ToStorageBlock() (block.Block, error) { + data, err := a.Serialize() + if err != nil { + return nil, err + } + c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) + if err != nil { + return nil, err + } + return block.NewBlockWithCid(data, c) +} + +func DecodeActorChange(b []byte) (*ActorChange, error) { + var ac ActorChange + if err := ac.UnmarshalCBOR(bytes.NewReader(b)); err != nil { + return nil, err + } + return &ac, nil } const KindActorChange = "actor_change" diff --git a/pkg/extract/actors/actordiff/cbor_gen.go b/pkg/extract/actors/actordiff/cbor_gen.go new file mode 100644 index 000000000..0ffe1f633 --- /dev/null +++ b/pkg/extract/actors/actordiff/cbor_gen.go @@ -0,0 +1,332 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package actordiff + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ActorChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Actor (types.Actor) (struct) + if len("actor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actor")); err != nil { + return err + } + + if err := t.Actor.MarshalCBOR(cw); err != nil { + return err + } + + // t.Current ([]uint8) (slice) + if len("current_state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_state")); err != nil { + return err + } + + if len(t.Current) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Current was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Current))); err != nil { + return err + } + + if _, err := cw.Write(t.Current[:]); err != nil { + return err + } + + // t.Previous ([]uint8) (slice) + if len("previous_state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_state")); err != nil { + return err + } + + if len(t.Previous) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Previous was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Previous))); err != nil { + return err + } + + if _, err := cw.Write(t.Previous[:]); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ActorChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Actor (types.Actor) (struct) + case "actor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Actor = new(types.Actor) + if err := t.Actor.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Actor pointer: %w", err) + } + } + + } + // t.Current ([]uint8) (slice) + case "current_state": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Current: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Current = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Current[:]); err != nil { + return err + } + // t.Previous ([]uint8) (slice) + case "previous_state": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Previous: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Previous = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Previous[:]); err != nil { + return err + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.ActorState (cid.Cid) (struct) + if len("actors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ActorState); err != nil { + return xerrors.Errorf("failed to write cid field t.ActorState: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.ActorState (cid.Cid) (struct) + case "actors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ActorState: %w", err) + } + + t.ActorState = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/actordiff/state.go index 35f190b28..95a05584e 100644 --- a/pkg/extract/actors/actordiff/state.go +++ b/pkg/extract/actors/actordiff/state.go @@ -17,8 +17,22 @@ type StateDiffResult struct { } func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - //TODO implement me - panic("implement me") + out := &StateChange{} + + if actorChanges := s.ActorStateChanges; actorChanges != nil { + blk, err := actorChanges.ToStorageBlock() + if err != nil { + return nil, err + } + if err := bs.Put(ctx, blk); err != nil { + return nil, err + } + c := blk.Cid() + out.ActorState = c + } else { + return nil, nil + } + return out, nil } func (s *StateDiffResult) Kind() string { @@ -26,7 +40,7 @@ func (s *StateDiffResult) Kind() string { } type StateChange struct { - ActorStates cid.Cid // Hamt[address]StateChange + ActorState cid.Cid `cborgen:"actors"` } type StateDiff struct { diff --git a/pkg/extract/actors/initdiff/v0/addresses.go b/pkg/extract/actors/initdiff/v0/addresses.go index 7f1abf78f..7c4c05fe9 100644 --- a/pkg/extract/actors/initdiff/v0/addresses.go +++ b/pkg/extract/actors/initdiff/v0/addresses.go @@ -4,10 +4,13 @@ import ( "context" "time" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -19,10 +22,14 @@ var log = logging.Logger("extract/actors/init") var _ actors.ActorStateChange = (*AddressChangeList)(nil) type AddressChange struct { - Address []byte - Current *typegen.Deferred // actorID - Previous *typegen.Deferred // actorID - Change core.ChangeType + Address []byte `cborgen:"address"` + Current *typegen.Deferred `cborgen:"current_actorID"` + Previous *typegen.Deferred `cborgen:"previous_actorID"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *AddressChange) Key() string { + return core.StringKey(t.Address).Key() } type AddressChangeList []*AddressChange @@ -33,6 +40,19 @@ func (a AddressChangeList) Kind() actors.ActorStateKind { return KindInitAddresses } +func (a AddressChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, a := range a { + if err := node.Put(a, a); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Addresses struct{} func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/initdiff/v0/cbor_gen.go b/pkg/extract/actors/initdiff/v0/cbor_gen.go new file mode 100644 index 000000000..93611d92b --- /dev/null +++ b/pkg/extract/actors/initdiff/v0/cbor_gen.go @@ -0,0 +1,303 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v0 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *AddressChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Address ([]uint8) (slice) + if len("address") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"address\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("address"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("address")); err != nil { + return err + } + + if len(t.Address) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Address was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Address))); err != nil { + return err + } + + if _, err := cw.Write(t.Address[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current_actorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current_actorID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current_actorID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current_actorID")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous_actorID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous_actorID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous_actorID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous_actorID")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AddressChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AddressChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AddressChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Address ([]uint8) (slice) + case "address": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Address: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Address = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Address[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current_actorID": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous_actorID": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Addresses (cid.Cid) (struct) + if len("addresses") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"addresses\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("addresses"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("addresses")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Addresses); err != nil { + return xerrors.Errorf("failed to write cid field t.Addresses: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Addresses (cid.Cid) (struct) + case "addresses": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Addresses: %w", err) + } + + t.Addresses = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/initdiff/v0/load.go b/pkg/extract/actors/initdiff/v0/load.go index f063aabf5..6ecae75fb 100644 --- a/pkg/extract/actors/initdiff/v0/load.go +++ b/pkg/extract/actors/initdiff/v0/load.go @@ -4,15 +4,15 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/chain/actors/builtin/init" + init_ "github.com/filecoin-project/lily/chain/actors/builtin/init" ) var InitStateLoader = func(strore adt.Store, act *types.Actor) (interface{}, error) { - return init.Load(strore, act) + return init_.Load(strore, act) } var InitAddressesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { - initState := m.(init.State) + initState := m.(init_.State) addressesMap, err := initState.AddressMap() if err != nil { return nil, nil, err diff --git a/pkg/extract/actors/initdiff/v0/state.go b/pkg/extract/actors/initdiff/v0/state.go index e27ead520..d74d255f6 100644 --- a/pkg/extract/actors/initdiff/v0/state.go +++ b/pkg/extract/actors/initdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -21,12 +22,21 @@ func (s *StateDiffResult) Kind() string { } func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - //TODO implement me - panic("implement me") + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if addresses := s.AddressesChanges; addresses != nil { + root, err := addresses.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Addresses = root + } + return out, nil } type StateChange struct { - Addresses cid.Cid + Addresses cid.Cid `cborgen:"addresses"` } type StateDiff struct { diff --git a/pkg/extract/actors/minerdiff/v0/cbor_gen.go b/pkg/extract/actors/minerdiff/v0/cbor_gen.go index 6db935a72..e0b48e4b0 100644 --- a/pkg/extract/actors/minerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/v0/cbor_gen.go @@ -8,11 +8,10 @@ import ( "math" "sort" + core "github.com/filecoin-project/lily/pkg/core" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - - core "github.com/filecoin-project/lily/pkg/core" ) var _ = xerrors.Errorf diff --git a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go new file mode 100644 index 000000000..a2b46c898 --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go @@ -0,0 +1,523 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v0 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Verifiers (cid.Cid) (struct) + if len("verifiers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifiers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifiers")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } + + // t.Clients (cid.Cid) (struct) + if len("clients") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"clients\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("clients"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("clients")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Clients); err != nil { + return xerrors.Errorf("failed to write cid field t.Clients: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifiers (cid.Cid) (struct) + case "verifiers": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = c + + } + // t.Clients (cid.Cid) (struct) + case "clients": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Clients: %w", err) + } + + t.Clients = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClientsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Client ([]uint8) (slice) + if len("client") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"client\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("client"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("client")); err != nil { + return err + } + + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { + return err + } + + if _, err := cw.Write(t.Client[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClientsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClientsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClientsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Client ([]uint8) (slice) + case "client": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Client: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Client = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VerifiersChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Verifier ([]uint8) (slice) + if len("verifier") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifier\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifier"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifier")); err != nil { + return err + } + + if len(t.Verifier) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Verifier was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Verifier))); err != nil { + return err + } + + if _, err := cw.Write(t.Verifier[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *VerifiersChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = VerifiersChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VerifiersChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifier ([]uint8) (slice) + case "verifier": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Verifier: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Verifier = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Verifier[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/verifregdiff/v0/clients.go b/pkg/extract/actors/verifregdiff/v0/clients.go index 22982e52f..fc0c5e75e 100644 --- a/pkg/extract/actors/verifregdiff/v0/clients.go +++ b/pkg/extract/actors/verifregdiff/v0/clients.go @@ -4,10 +4,13 @@ import ( "context" "time" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -19,10 +22,14 @@ var log = logging.Logger("lily/extract/actors/verifreg/v0") // TODO add cbor gen tags type ClientsChange struct { - Client []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Client []byte `cborgen:"client"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *ClientsChange) Key() string { + return core.StringKey(t.Client).Key() } type ClientsChangeList []*ClientsChange @@ -33,6 +40,19 @@ func (v ClientsChangeList) Kind() actors.ActorStateKind { return KindVerifregClients } +func (v ClientsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range v { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Clients struct{} func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/verifregdiff/v0/state.go b/pkg/extract/actors/verifregdiff/v0/state.go index 004e48a06..f20528fda 100644 --- a/pkg/extract/actors/verifregdiff/v0/state.go +++ b/pkg/extract/actors/verifregdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" @@ -22,12 +23,30 @@ func (s *StateDiffResult) Kind() string { } type StateChange struct { - Verifiers cid.Cid - Clients cid.Cid + Verifiers cid.Cid `cborgen:"verifiers"` + Clients cid.Cid `cborgen:"clients"` } func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) { - panic("TODO") + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if clients := sd.ClientChanges; clients != nil { + root, err := clients.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Clients = root + } + + if verifiers := sd.VerifierChanges; verifiers != nil { + root, err := verifiers.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Verifiers = root + } + return out, nil } type StateDiff struct { diff --git a/pkg/extract/actors/verifregdiff/v0/verifiers.go b/pkg/extract/actors/verifregdiff/v0/verifiers.go index 574ab3e3a..423c5b147 100644 --- a/pkg/extract/actors/verifregdiff/v0/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v0/verifiers.go @@ -4,9 +4,12 @@ import ( "context" "time" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -15,10 +18,14 @@ import ( // TODO add cbor gen tags type VerifiersChange struct { - Verifier []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Verifier []byte `cborgen:"verifier"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *VerifiersChange) Key() string { + return core.StringKey(t.Verifier).Key() } type VerifiersChangeList []*VerifiersChange @@ -29,6 +36,19 @@ func (v VerifiersChangeList) Kind() actors.ActorStateKind { return KindVerifregVerifiers } +func (v VerifiersChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range v { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Verifiers struct{} func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/verifregdiff/v9/allocations.go b/pkg/extract/actors/verifregdiff/v9/allocations.go index 0a5009efb..5b802bf21 100644 --- a/pkg/extract/actors/verifregdiff/v9/allocations.go +++ b/pkg/extract/actors/verifregdiff/v9/allocations.go @@ -14,11 +14,11 @@ import ( // TODO add cbor gen tags type AllocationsChange struct { - Provider []byte - ClaimID []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Provider []byte `cborgen:"provider"` + ClaimID []byte `cborgen:"claimID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` } type AllocationsChangeList []*AllocationsChange diff --git a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go new file mode 100644 index 000000000..4c116dddd --- /dev/null +++ b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go @@ -0,0 +1,644 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v9 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Verifiers (cid.Cid) (struct) + if len("verifiers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifiers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifiers")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } + + // t.Claims (cid.Cid) (struct) + if len("claims") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claims\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claims")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } + + // t.Allocations (cid.Cid) (struct) + if len("allocations") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"allocations\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allocations"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("allocations")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Allocations); err != nil { + return xerrors.Errorf("failed to write cid field t.Allocations: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifiers (cid.Cid) (struct) + case "verifiers": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = c + + } + // t.Claims (cid.Cid) (struct) + case "claims": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } + + t.Claims = c + + } + // t.Allocations (cid.Cid) (struct) + case "allocations": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allocations: %w", err) + } + + t.Allocations = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Provider ([]uint8) (slice) + if len("provider") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"provider\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("provider"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("provider")); err != nil { + return err + } + + if len(t.Provider) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Provider was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { + return err + } + + if _, err := cw.Write(t.Provider[:]); err != nil { + return err + } + + // t.ClaimID ([]uint8) (slice) + if len("claimID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claimID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claimID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claimID")); err != nil { + return err + } + + if len(t.ClaimID) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.ClaimID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ClaimID))); err != nil { + return err + } + + if _, err := cw.Write(t.ClaimID[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClaimsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClaimsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Provider ([]uint8) (slice) + case "provider": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Provider: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Provider = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Provider[:]); err != nil { + return err + } + // t.ClaimID ([]uint8) (slice) + case "claimID": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.ClaimID: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.ClaimID = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.ClaimID[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *AllocationsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Provider ([]uint8) (slice) + if len("provider") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"provider\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("provider"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("provider")); err != nil { + return err + } + + if len(t.Provider) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Provider was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { + return err + } + + if _, err := cw.Write(t.Provider[:]); err != nil { + return err + } + + // t.ClaimID ([]uint8) (slice) + if len("claimID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claimID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claimID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claimID")); err != nil { + return err + } + + if len(t.ClaimID) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.ClaimID was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.ClaimID))); err != nil { + return err + } + + if _, err := cw.Write(t.ClaimID[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AllocationsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AllocationsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AllocationsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Provider ([]uint8) (slice) + case "provider": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Provider: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Provider = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Provider[:]); err != nil { + return err + } + // t.ClaimID ([]uint8) (slice) + case "claimID": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.ClaimID: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.ClaimID = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.ClaimID[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/verifregdiff/v9/claims.go b/pkg/extract/actors/verifregdiff/v9/claims.go index 44233f633..a644c51f9 100644 --- a/pkg/extract/actors/verifregdiff/v9/claims.go +++ b/pkg/extract/actors/verifregdiff/v9/claims.go @@ -21,11 +21,11 @@ import ( // TODO add cbor gen tags type ClaimsChange struct { - Provider []byte - ClaimID []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Provider []byte `cborgen:"provider"` + ClaimID []byte `cborgen:"claimID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` } type ClaimsChangeList []*ClaimsChange diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index 6fe65bf3a..5a924fdab 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -5,11 +5,12 @@ import ( "time" "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" v8 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v8" "github.com/filecoin-project/lily/tasks" ) @@ -22,13 +23,19 @@ type StateDiffResult struct { AllocationsChanges AllocationsChangeList } +func (s *StateDiffResult) Kind() string { + return "verifreg" +} + func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { //TODO implement me panic("implement me") } -func (s *StateDiffResult) Kind() string { - return "verifreg" +type StateChange struct { + Verifiers cid.Cid `cborgen:"verifiers"` + Claims cid.Cid `cborgen:"claims"` + Allocations cid.Cid `cborgen:"allocations"` } type StateDiff struct { @@ -48,13 +55,13 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors } switch stateChange.Kind() { case v0.KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(v0.VerifiersChangeList) + stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) case KindVerifregClaims: stateDiff.ClaimChanges = stateChange.(ClaimsChangeList) case KindVerifregAllocations: stateDiff.AllocationsChanges = stateChange.(AllocationsChangeList) } } - log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) + log.Infow("Extracted Verified Registry State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 6f99ae129..741ce2710 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -14,7 +14,7 @@ import ( "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" - "github.com/filecoin-project/lily/chain/actors/builtin/init" + init_ "github.com/filecoin-project/lily/chain/actors/builtin/init" "github.com/filecoin-project/lily/chain/actors/builtin/market" "github.com/filecoin-project/lily/chain/actors/builtin/miner" "github.com/filecoin-project/lily/chain/actors/builtin/power" @@ -54,7 +54,7 @@ func init() { for _, c := range verifreg.AllCodes() { VerifregCodes.Add(c) } - for _, c := range init.AllCodes() { + for _, c := range init_.AllCodes() { InitCodes.Add(c) } } diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 3f9f29906..9f6a099de 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -3,44 +3,69 @@ package main import ( cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + minerv0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" ) -const minerDiffPath = "pkg/extract/actors/minerdiff/cbor_gen.go" -const minerDiffPkg = "minerdiff" +const actorDiffPath = "pkg/extract/actors/actordiff/cbor_gen.go" +const actorDiffPkg = "actordiff" -const minerTransformPath = "pkg/transform/cbor/miner/cbor_gen.go" -const minerTransformPkg = "miner" +const minerDiffPath = "pkg/extract/actors/minerdiff/v0/cbor_gen.go" +const minerDiffPkg = "v0" -const actorTransformPath = "pkg/transform/cbor/cbor_gen.go" -const actorTransformPkg = "cbor" +const initDiffPath = "pkg/extract/actors/initdiff/v0/cbor_gen.go" +const initDiffPkg = "v0" + +const verifDiffPathV0 = "pkg/extract/actors/verifregdiff/v0/cbor_gen.go" +const verifDiffPkgV0 = "v0" + +const verifDiffPathV9 = "pkg/extract/actors/verifregdiff/v9/cbor_gen.go" +const verifDiffPkgV9 = "v9" func main() { + if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, + actordiff.ActorChange{}, + actordiff.StateChange{}, + ); err != nil { + panic(err) + } + if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, - v0.SectorStatusChange{}, - v0.PreCommitChange{}, - v0.SectorChange{}, - v0.FundsChange{}, - v0.DebtChange{}, - v0.InfoChange{}, - v0.StateChange{}, + minerv0.SectorStatusChange{}, + minerv0.PreCommitChange{}, + minerv0.SectorChange{}, + minerv0.FundsChange{}, + minerv0.DebtChange{}, + minerv0.InfoChange{}, + minerv0.StateChange{}, ); err != nil { panic(err) } - /* - if err := cbg.WriteMapEncodersToFile(minerTransformPath, minerTransformPkg, - v9.StateChange{}, - ); err != nil { - panic(err) - } + if err := cbg.WriteMapEncodersToFile(initDiffPath, initDiffPkg, + initv0.AddressChange{}, + initv0.StateChange{}, + ); err != nil { + panic(err) + } - if err := cbg.WriteMapEncodersToFile(actorTransformPath, actorTransformPkg, - cbor.ActorIPLDContainer{}, - ); err != nil { - panic(err) - } + if err := cbg.WriteMapEncodersToFile(verifDiffPathV0, verifDiffPkgV0, + verifV0.StateChange{}, + verifV0.ClientsChange{}, + verifV0.VerifiersChange{}, + ); err != nil { + panic(err) + } - */ + if err := cbg.WriteMapEncodersToFile(verifDiffPathV9, verifDiffPkgV9, + verifV9.StateChange{}, + verifV9.ClaimsChange{}, + verifV9.AllocationsChange{}, + ); err != nil { + panic(err) + } } diff --git a/pkg/transform/cbor/actor/state.go b/pkg/transform/cbor/actor/state.go new file mode 100644 index 000000000..09577869f --- /dev/null +++ b/pkg/transform/cbor/actor/state.go @@ -0,0 +1,33 @@ +package actor + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + store2 "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func HandleChanges(ctx context.Context, bs blockstore.Blockstore, actors map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { + store := store2.WrapBlockStore(ctx, bs) + actorHamt, err := adt.MakeEmptyMap(store, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for addr, change := range actors { + msc, err := change.MarshalStateChange(ctx, bs) + if err != nil { + return cid.Undef, err + } + + if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { + return cid.Undef, err + } + } + return actorHamt.Root() +} diff --git a/pkg/transform/cbor/init_/state.go b/pkg/transform/cbor/init_/state.go new file mode 100644 index 000000000..22d695bcd --- /dev/null +++ b/pkg/transform/cbor/init_/state.go @@ -0,0 +1,20 @@ +package init_ + +import ( + "context" + + "github.com/filecoin-project/lotus/blockstore" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func HandleChanges(ctx context.Context, bs blockstore.Blockstore, addresses actors.ActorDiffResult) (cid.Cid, error) { + store := adt2.WrapBlockStore(ctx, bs) + isc, err := addresses.MarshalStateChange(ctx, bs) + if err != nil { + return cid.Undef, err + } + return store.Put(ctx, isc) +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 1edad2cd6..c66c5caf5 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -13,6 +13,8 @@ import ( v1car "github.com/ipld/go-car" "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actor" + "github.com/filecoin-project/lily/pkg/transform/cbor/init_" "github.com/filecoin-project/lily/pkg/transform/cbor/miner" "github.com/filecoin-project/lily/pkg/transform/cbor/verifreg" ) @@ -23,6 +25,8 @@ type ActorIPLDContainer struct { ExecutedTipSet *types.TipSet MinerActors cid.Cid // HAMT[Address]MinerStateChange VerifregActor *cid.Cid // VerifregStateChange or empty + ActorStates cid.Cid // HAMT[Address]ActorStateChange + InitActor cid.Cid // HAMT[Address]AddressChanges. } func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { @@ -64,11 +68,12 @@ func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *proce CurrentTipSet: changes.Current, ExecutedTipSet: changes.Executed, } - minerHamtRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) + minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) if err != nil { return nil, err } - out.MinerActors = minerHamtRoot + out.MinerActors = minerRoot + if changes.VerifregActor != nil { verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) if err != nil { @@ -76,5 +81,21 @@ func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *proce } out.VerifregActor = &verifregRoot } + + if changes.ActorStates != nil { + actorsRoot, err := actor.HandleChanges(ctx, bs, changes.ActorStates) + if err != nil { + return nil, err + } + out.ActorStates = actorsRoot + } + + if changes.InitActor != nil { + initRoot, err := init_.HandleChanges(ctx, bs, changes.InitActor) + if err != nil { + return nil, err + } + out.InitActor = initRoot + } return out, nil } From 1575b14c354d8d69685fdefa881811e414a77406 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 12 Jan 2023 15:19:37 -0800 Subject: [PATCH 20/48] can still produce car files --- chain/actors/builtin/market/actor.go.template | 4 +- chain/actors/builtin/market/diff.go | 8 +- chain/actors/builtin/market/market.go | 4 +- chain/actors/builtin/market/state.go.template | 4 +- chain/actors/builtin/market/v0.go | 4 +- chain/actors/builtin/market/v2.go | 4 +- chain/actors/builtin/market/v3.go | 4 +- chain/actors/builtin/market/v4.go | 4 +- chain/actors/builtin/market/v5.go | 4 +- chain/actors/builtin/market/v6.go | 4 +- chain/actors/builtin/market/v7.go | 4 +- chain/actors/builtin/market/v8.go | 4 +- chain/actors/builtin/market/v9.go | 4 +- pkg/extract/actors/actordiff/state.go | 2 +- pkg/extract/actors/initdiff/v0/load.go | 4 +- pkg/extract/actors/marketdiff/v0/cbor_gen.go | 493 ++++++++++++++++++ pkg/extract/actors/marketdiff/v0/deals.go | 31 +- pkg/extract/actors/marketdiff/v0/load.go | 30 ++ pkg/extract/actors/marketdiff/v0/proposals.go | 31 +- pkg/extract/actors/marketdiff/v0/state.go | 30 +- pkg/extract/actors/powerdiff/v0/cbor_gen.go | 303 +++++++++++ pkg/extract/actors/powerdiff/v0/claims.go | 27 +- pkg/extract/actors/powerdiff/v0/state.go | 21 +- .../actors/verifregdiff/v9/allocations.go | 11 +- .../actors/verifregdiff/v9/cbor_gen.go | 78 ++- pkg/extract/actors/verifregdiff/v9/claims.go | 57 +- pkg/extract/actors/verifregdiff/v9/state.go | 48 +- pkg/extract/actors/verifregdiff/version.go | 3 +- pkg/gen/main.go | 38 +- pkg/transform/cbor/market/state.go | 21 + pkg/transform/cbor/power/state.go | 20 + pkg/transform/cbor/router.go | 30 +- pkg/transform/cbor/verifreg/state.go | 6 +- 33 files changed, 1228 insertions(+), 112 deletions(-) create mode 100644 pkg/extract/actors/marketdiff/v0/cbor_gen.go create mode 100644 pkg/extract/actors/marketdiff/v0/load.go create mode 100644 pkg/extract/actors/powerdiff/v0/cbor_gen.go create mode 100644 pkg/transform/cbor/market/state.go create mode 100644 pkg/transform/cbor/power/state.go diff --git a/chain/actors/builtin/market/actor.go.template b/chain/actors/builtin/market/actor.go.template index 36053de4d..46c94fe50 100644 --- a/chain/actors/builtin/market/actor.go.template +++ b/chain/actors/builtin/market/actor.go.template @@ -80,7 +80,7 @@ type DealStates interface { ForEach(cb func(id abi.DealID, ds DealState) error) error Get(id abi.DealID) (*DealState, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*DealState, error) } @@ -88,7 +88,7 @@ type DealProposals interface { ForEach(cb func(id abi.DealID, dp markettypes.DealProposal) error) error Get(id abi.DealID) (*markettypes.DealProposal, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*markettypes.DealProposal, error) } diff --git a/chain/actors/builtin/market/diff.go b/chain/actors/builtin/market/diff.go index 283e29d09..a53e4aa06 100644 --- a/chain/actors/builtin/market/diff.go +++ b/chain/actors/builtin/market/diff.go @@ -34,13 +34,13 @@ func DiffDealProposals(ctx context.Context, store adt.Store, pre, cur State) (*D diffContainer := NewMarketProposalsDiffContainer(preP, curP) if requiresLegacyDiffing(pre, cur, preOpts, curOpts) { log.Warn("actor AMT opts differ, running slower generic array diff", "preCID", pre.Code(), "curCID", cur.Code()) - if err := diff.CompareArray(preP.array(), curP.array(), diffContainer); err != nil { + if err := diff.CompareArray(preP.AsArray(), curP.AsArray(), diffContainer); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) } return diffContainer.Results, nil } - changes, err := diff.Amt(ctx, preP.array(), curP.array(), store, store, amt.UseTreeBitWidth(uint(preOpts))) + changes, err := diff.Amt(ctx, preP.AsArray(), curP.AsArray(), store, store, amt.UseTreeBitWidth(uint(preOpts))) if err != nil { return nil, err } @@ -116,13 +116,13 @@ func DiffDealStates(ctx context.Context, store adt.Store, pre, cur State) (*Deal diffContainer := NewMarketStatesDiffContainer(preS, curS) if requiresLegacyDiffing(pre, cur, preOpts, curOpts) { log.Warn("actor AMT opts differ, running slower generic array diff", "preCID", pre.Code(), "curCID", cur.Code()) - if err := diff.CompareArray(preS.array(), curS.array(), diffContainer); err != nil { + if err := diff.CompareArray(preS.AsArray(), curS.AsArray(), diffContainer); err != nil { return nil, fmt.Errorf("diffing deal states: %w", err) } return diffContainer.Results, nil } - changes, err := diff.Amt(ctx, preS.array(), curS.array(), store, store, amt.UseTreeBitWidth(uint(preOpts))) + changes, err := diff.Amt(ctx, preS.AsArray(), curS.AsArray(), store, store, amt.UseTreeBitWidth(uint(preOpts))) if err != nil { return nil, err } diff --git a/chain/actors/builtin/market/market.go b/chain/actors/builtin/market/market.go index 7c765b1e1..ccb3ef7e3 100644 --- a/chain/actors/builtin/market/market.go +++ b/chain/actors/builtin/market/market.go @@ -105,7 +105,7 @@ type DealStates interface { ForEach(cb func(id abi.DealID, ds DealState) error) error Get(id abi.DealID) (*DealState, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*DealState, error) } @@ -113,7 +113,7 @@ type DealProposals interface { ForEach(cb func(id abi.DealID, dp markettypes.DealProposal) error) error Get(id abi.DealID) (*markettypes.DealProposal, bool, error) - array() adt.Array + AsArray() adt.Array decode(*cbg.Deferred) (*markettypes.DealProposal, error) } diff --git a/chain/actors/builtin/market/state.go.template b/chain/actors/builtin/market/state.go.template index 0ddee8698..5dce97966 100644 --- a/chain/actors/builtin/market/state.go.template +++ b/chain/actors/builtin/market/state.go.template @@ -134,7 +134,7 @@ func (s *dealStates{{.v}}) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates{{.v}}) array() adt.Array { +func (s *dealStates{{.v}}) AsArray() adt.Array { return s.Array } @@ -199,7 +199,7 @@ func (s *dealProposals{{.v}}) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals{{.v}}) array() adt.Array { +func (s *dealProposals{{.v}}) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v0.go b/chain/actors/builtin/market/v0.go index 20bf51c9a..63db494e8 100644 --- a/chain/actors/builtin/market/v0.go +++ b/chain/actors/builtin/market/v0.go @@ -121,7 +121,7 @@ func (s *dealStates0) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates0) array() adt.Array { +func (s *dealStates0) AsArray() adt.Array { return s.Array } @@ -184,7 +184,7 @@ func (s *dealProposals0) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals0) array() adt.Array { +func (s *dealProposals0) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v2.go b/chain/actors/builtin/market/v2.go index 81bb6f87d..5dffdf854 100644 --- a/chain/actors/builtin/market/v2.go +++ b/chain/actors/builtin/market/v2.go @@ -121,7 +121,7 @@ func (s *dealStates2) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates2) array() adt.Array { +func (s *dealStates2) AsArray() adt.Array { return s.Array } @@ -184,7 +184,7 @@ func (s *dealProposals2) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals2) array() adt.Array { +func (s *dealProposals2) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v3.go b/chain/actors/builtin/market/v3.go index dfba355e4..1b6292c96 100644 --- a/chain/actors/builtin/market/v3.go +++ b/chain/actors/builtin/market/v3.go @@ -116,7 +116,7 @@ func (s *dealStates3) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates3) array() adt.Array { +func (s *dealStates3) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals3) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals3) array() adt.Array { +func (s *dealProposals3) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v4.go b/chain/actors/builtin/market/v4.go index c1002dcc6..119e219da 100644 --- a/chain/actors/builtin/market/v4.go +++ b/chain/actors/builtin/market/v4.go @@ -116,7 +116,7 @@ func (s *dealStates4) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates4) array() adt.Array { +func (s *dealStates4) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals4) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals4) array() adt.Array { +func (s *dealProposals4) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v5.go b/chain/actors/builtin/market/v5.go index c282dc842..af231d062 100644 --- a/chain/actors/builtin/market/v5.go +++ b/chain/actors/builtin/market/v5.go @@ -116,7 +116,7 @@ func (s *dealStates5) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates5) array() adt.Array { +func (s *dealStates5) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals5) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals5) array() adt.Array { +func (s *dealProposals5) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v6.go b/chain/actors/builtin/market/v6.go index 60355dc59..b4c8d104e 100644 --- a/chain/actors/builtin/market/v6.go +++ b/chain/actors/builtin/market/v6.go @@ -116,7 +116,7 @@ func (s *dealStates6) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates6) array() adt.Array { +func (s *dealStates6) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals6) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals6) array() adt.Array { +func (s *dealProposals6) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v7.go b/chain/actors/builtin/market/v7.go index 24f78821e..22a266cd4 100644 --- a/chain/actors/builtin/market/v7.go +++ b/chain/actors/builtin/market/v7.go @@ -116,7 +116,7 @@ func (s *dealStates7) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates7) array() adt.Array { +func (s *dealStates7) AsArray() adt.Array { return s.Array } @@ -179,7 +179,7 @@ func (s *dealProposals7) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals7) array() adt.Array { +func (s *dealProposals7) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v8.go b/chain/actors/builtin/market/v8.go index 42be7995c..b47be1e64 100644 --- a/chain/actors/builtin/market/v8.go +++ b/chain/actors/builtin/market/v8.go @@ -117,7 +117,7 @@ func (s *dealStates8) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates8) array() adt.Array { +func (s *dealStates8) AsArray() adt.Array { return s.Array } @@ -180,7 +180,7 @@ func (s *dealProposals8) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals8) array() adt.Array { +func (s *dealProposals8) AsArray() adt.Array { return s.Array } diff --git a/chain/actors/builtin/market/v9.go b/chain/actors/builtin/market/v9.go index 36586b69e..b613f09bc 100644 --- a/chain/actors/builtin/market/v9.go +++ b/chain/actors/builtin/market/v9.go @@ -117,7 +117,7 @@ func (s *dealStates9) decode(val *cbg.Deferred) (*DealState, error) { return &ds, nil } -func (s *dealStates9) array() adt.Array { +func (s *dealStates9) AsArray() adt.Array { return s.Array } @@ -175,7 +175,7 @@ func (s *dealProposals9) decode(val *cbg.Deferred) (*DealProposal, error) { return &dp, nil } -func (s *dealProposals9) array() adt.Array { +func (s *dealProposals9) AsArray() adt.Array { return s.Array } diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/actordiff/state.go index 95a05584e..811ba250d 100644 --- a/pkg/extract/actors/actordiff/state.go +++ b/pkg/extract/actors/actordiff/state.go @@ -63,6 +63,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.ActorStateChanges = stateChange.(*ActorChange) } } - log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) + log.Infow("Extracted Raw Actor State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/initdiff/v0/load.go b/pkg/extract/actors/initdiff/v0/load.go index 6ecae75fb..ce98e79d4 100644 --- a/pkg/extract/actors/initdiff/v0/load.go +++ b/pkg/extract/actors/initdiff/v0/load.go @@ -7,8 +7,8 @@ import ( init_ "github.com/filecoin-project/lily/chain/actors/builtin/init" ) -var InitStateLoader = func(strore adt.Store, act *types.Actor) (interface{}, error) { - return init_.Load(strore, act) +var InitStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return init_.Load(store, act) } var InitAddressesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { diff --git a/pkg/extract/actors/marketdiff/v0/cbor_gen.go b/pkg/extract/actors/marketdiff/v0/cbor_gen.go new file mode 100644 index 000000000..5343e19c1 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v0/cbor_gen.go @@ -0,0 +1,493 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v0 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Deals (cid.Cid) (struct) + if len("deals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"deals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("deals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("deals")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Deals); err != nil { + return xerrors.Errorf("failed to write cid field t.Deals: %w", err) + } + + // t.Proposals (cid.Cid) (struct) + if len("proposals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"proposals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("proposals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("proposals")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Proposals); err != nil { + return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Deals (cid.Cid) (struct) + case "deals": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Deals: %w", err) + } + + t.Deals = c + + } + // t.Proposals (cid.Cid) (struct) + case "proposals": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Proposals: %w", err) + } + + t.Proposals = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ProposalChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.DealID (uint64) (uint64) + if len("dealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"dealID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("dealID")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ProposalChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ProposalChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ProposalChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DealID (uint64) (uint64) + case "dealID": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *DealChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.DealID (uint64) (uint64) + if len("dealID") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"dealID\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("dealID"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("dealID")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.DealID)); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *DealChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = DealChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("DealChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DealID (uint64) (uint64) + case "dealID": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.DealID = uint64(extra) + + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/marketdiff/v0/deals.go b/pkg/extract/actors/marketdiff/v0/deals.go index 58d223738..0540a9a89 100644 --- a/pkg/extract/actors/marketdiff/v0/deals.go +++ b/pkg/extract/actors/marketdiff/v0/deals.go @@ -4,9 +4,13 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -14,10 +18,14 @@ import ( ) type DealChange struct { - DealID uint64 - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + DealID uint64 `cborgen:"dealID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *DealChange) Key() string { + return abi.UIntKey(t.DealID).Key() } type DealChangeList []*DealChange @@ -28,6 +36,19 @@ func (p DealChangeList) Kind() actors.ActorStateKind { return KindMarketDeal } +func (p DealChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Deals struct{} func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { @@ -39,7 +60,7 @@ func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorCh } func DiffDeals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - arrayChanges, err := generic.DiffActorArray(ctx, api, act, nil, nil) + arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketDealsArrayLoader) if err != nil { return nil, err } diff --git a/pkg/extract/actors/marketdiff/v0/load.go b/pkg/extract/actors/marketdiff/v0/load.go new file mode 100644 index 000000000..07dbefdc9 --- /dev/null +++ b/pkg/extract/actors/marketdiff/v0/load.go @@ -0,0 +1,30 @@ +package v0 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/market" +) + +var MarketStateLoader = func(store adt.Store, act *types.Actor) (interface{}, error) { + return market.Load(store, act) +} + +var MarketDealsArrayLoader = func(m interface{}) (adt.Array, int, error) { + marketState := m.(market.State) + dealsArray, err := marketState.States() + if err != nil { + return nil, -1, err + } + return dealsArray.AsArray(), marketState.DealStatesAmtBitwidth(), nil +} + +var MarketProposlasArrayLoader = func(m interface{}) (adt.Array, int, error) { + marketState := m.(market.State) + proposalsArray, err := marketState.Proposals() + if err != nil { + return nil, -1, err + } + return proposalsArray.AsArray(), marketState.DealProposalsAmtBitwidth(), nil +} diff --git a/pkg/extract/actors/marketdiff/v0/proposals.go b/pkg/extract/actors/marketdiff/v0/proposals.go index 10d53e0c8..191b978f1 100644 --- a/pkg/extract/actors/marketdiff/v0/proposals.go +++ b/pkg/extract/actors/marketdiff/v0/proposals.go @@ -4,10 +4,14 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -17,10 +21,14 @@ import ( var log = logging.Logger("extract/actors/market") type ProposalChange struct { - DealID uint64 - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + DealID uint64 `cborgen:"dealID"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *ProposalChange) Key() string { + return abi.UIntKey(t.DealID).Key() } type ProposalChangeList []*ProposalChange @@ -31,6 +39,19 @@ func (p ProposalChangeList) Kind() actors.ActorStateKind { return KindMarketProposal } +func (p ProposalChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Proposals struct{} func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { @@ -42,7 +63,7 @@ func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act } func DiffProposals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - arrayChanges, err := generic.DiffActorArray(ctx, api, act, nil, nil) + arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketProposlasArrayLoader) if err != nil { return nil, err } diff --git a/pkg/extract/actors/marketdiff/v0/state.go b/pkg/extract/actors/marketdiff/v0/state.go index c81d92516..4e270c0e7 100644 --- a/pkg/extract/actors/marketdiff/v0/state.go +++ b/pkg/extract/actors/marketdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -17,18 +18,35 @@ type StateDiffResult struct { DealProposalChanges ProposalChangeList } -func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - //TODO implement me - panic("implement me") +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if deals := sd.DealStateChanges; deals != nil { + root, err := deals.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Deals = root + } + + if proposals := sd.DealProposalChanges; proposals != nil { + root, err := proposals.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Proposals = root + } + return out, nil } -func (s *StateDiffResult) Kind() string { +func (sd *StateDiffResult) Kind() string { return "market" } type StateChange struct { - Deals cid.Cid - Proposals cid.Cid + Deals cid.Cid `cborgen:"deals"` + Proposals cid.Cid `cborgen:"proposals"` } type StateDiff struct { diff --git a/pkg/extract/actors/powerdiff/v0/cbor_gen.go b/pkg/extract/actors/powerdiff/v0/cbor_gen.go new file mode 100644 index 000000000..1d1a4d7fa --- /dev/null +++ b/pkg/extract/actors/powerdiff/v0/cbor_gen.go @@ -0,0 +1,303 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v0 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Claims (cid.Cid) (struct) + if len("claims") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claims\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claims")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Claims (cid.Cid) (struct) + case "claims": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } + + t.Claims = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Miner ([]uint8) (slice) + if len("miner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("miner")); err != nil { + return err + } + + if len(t.Miner) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Miner was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Miner))); err != nil { + return err + } + + if _, err := cw.Write(t.Miner[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = ClaimsChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ClaimsChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Miner ([]uint8) (slice) + case "miner": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Miner: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Miner = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Miner[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/powerdiff/v0/claims.go b/pkg/extract/actors/powerdiff/v0/claims.go index a2e8cb4d3..4bcabb75a 100644 --- a/pkg/extract/actors/powerdiff/v0/claims.go +++ b/pkg/extract/actors/powerdiff/v0/claims.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -17,10 +19,14 @@ import ( var log = logging.Logger("extract/actors/power") type ClaimsChange struct { - Miner []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Miner []byte `cborgen:"miner"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +func (t *ClaimsChange) Key() string { + return core.StringKey(t.Miner).Key() } type ClaimsChangeList []*ClaimsChange @@ -31,6 +37,19 @@ func (c ClaimsChangeList) Kind() actors.ActorStateKind { return KindPowerClaims } +func (p ClaimsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range p { + if err := node.Put(l, l); err != nil { + return cid.Undef, err + } + } + return node.Root() +} + type Claims struct{} func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v0/state.go index bed352755..14c590cdd 100644 --- a/pkg/extract/actors/powerdiff/v0/state.go +++ b/pkg/extract/actors/powerdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -16,17 +17,27 @@ type StateDiffResult struct { ClaimsChanges ClaimsChangeList } -func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - //TODO implement me - panic("implement me") +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if claims := sd.ClaimsChanges; claims != nil { + root, err := claims.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Claims = root + } + + return out, nil } -func (s *StateDiffResult) Kind() string { +func (sd *StateDiffResult) Kind() string { return "power" } type StateChange struct { - Claims cid.Cid + Claims cid.Cid `cborgen:"claims"` } type StateDiff struct { diff --git a/pkg/extract/actors/verifregdiff/v9/allocations.go b/pkg/extract/actors/verifregdiff/v9/allocations.go index 5b802bf21..e3f4c5c82 100644 --- a/pkg/extract/actors/verifregdiff/v9/allocations.go +++ b/pkg/extract/actors/verifregdiff/v9/allocations.go @@ -4,9 +4,12 @@ import ( "context" "time" + "github.com/filecoin-project/go-address" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -21,14 +24,18 @@ type AllocationsChange struct { Change core.ChangeType `cborgen:"change"` } -type AllocationsChangeList []*AllocationsChange +type AllocationsChangeMap map[address.Address][]*AllocationsChange const KindVerifregAllocations = "verifreg_allocations" -func (c AllocationsChangeList) Kind() actors.ActorStateKind { +func (c AllocationsChangeMap) Kind() actors.ActorStateKind { return KindVerifregAllocations } +func (c AllocationsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + panic("NYI") +} + type Allocations struct{} func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go index 4c116dddd..a5b4c64e6 100644 --- a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go @@ -43,8 +43,14 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := cbg.WriteCid(cw, t.Verifiers); err != nil { - return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + if t.Verifiers == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } } // t.Claims (cid.Cid) (struct) @@ -59,8 +65,14 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := cbg.WriteCid(cw, t.Claims); err != nil { - return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + if t.Claims == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } } // t.Allocations (cid.Cid) (struct) @@ -75,8 +87,14 @@ func (t *StateChange) MarshalCBOR(w io.Writer) error { return err } - if err := cbg.WriteCid(cw, t.Allocations); err != nil { - return xerrors.Errorf("failed to write cid field t.Allocations: %w", err) + if t.Allocations == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Allocations); err != nil { + return xerrors.Errorf("failed to write cid field t.Allocations: %w", err) + } } return nil @@ -125,12 +143,22 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { { - c, err := cbg.ReadCid(cr) + b, err := cr.ReadByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } - t.Verifiers = c + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = &c + } } // t.Claims (cid.Cid) (struct) @@ -138,12 +166,22 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { { - c, err := cbg.ReadCid(cr) + b, err := cr.ReadByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } - t.Claims = c + t.Claims = &c + } } // t.Allocations (cid.Cid) (struct) @@ -151,12 +189,22 @@ func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { { - c, err := cbg.ReadCid(cr) + b, err := cr.ReadByte() if err != nil { - return xerrors.Errorf("failed to read cid field t.Allocations: %w", err) + return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allocations: %w", err) + } - t.Allocations = c + t.Allocations = &c + } } diff --git a/pkg/extract/actors/verifregdiff/v9/claims.go b/pkg/extract/actors/verifregdiff/v9/claims.go index a644c51f9..14d9a0c4c 100644 --- a/pkg/extract/actors/verifregdiff/v9/claims.go +++ b/pkg/extract/actors/verifregdiff/v9/claims.go @@ -6,6 +6,8 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -28,14 +30,41 @@ type ClaimsChange struct { Change core.ChangeType `cborgen:"change"` } -type ClaimsChangeList []*ClaimsChange +type ClaimsChangeMap map[address.Address][]*ClaimsChange const KindVerifregClaims = "verifreg_claims" -func (c ClaimsChangeList) Kind() actors.ActorStateKind { +func (c ClaimsChangeMap) Kind() actors.ActorStateKind { return KindVerifregClaims } +// returns a HAMT[provider]HAMT[claimID]ClaimsChange +func (c ClaimsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for provider, changes := range c { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.ClaimID), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(provider), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() +} + type Claims struct{} func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { @@ -52,27 +81,27 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan return nil, err } // map change is keyed on provider address with value adt.Map - out := make(ClaimsChangeList, 0, len(mapChange)) + out := make(ClaimsChangeMap) for _, change := range mapChange { - subMapChanges, err := diffSubMap(ctx, api, act, change.Key) + providerID, err := abi.ParseUIntKey(string(change.Key)) if err != nil { return nil, err } - out = append(out, subMapChanges...) + providerAddress, err := address.NewIDAddress(providerID) + if err != nil { + return nil, err + } + subMapChanges, err := diffSubMap(ctx, api, act, providerAddress, change.Key) + if err != nil { + return nil, err + } + out[providerAddress] = subMapChanges } return out, nil } -func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerKey []byte) ([]*ClaimsChange, error) { +func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { - providerID, err := abi.ParseUIntKey(string(providerKey)) - if err != nil { - return nil, nil, err - } - providerAddress, err := address.NewIDAddress(providerID) - if err != nil { - return nil, nil, err - } verifregState := i.(verifreg.State) providerClaimMap, err := verifregState.ClaimMapForProvider(providerAddress) if err != nil { diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index 5a924fdab..cafd30fe8 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" @@ -19,23 +20,48 @@ var log = logging.Logger("extract/actors/verifreg") type StateDiffResult struct { VerifierChanges v8.VerifiersChangeList - ClaimChanges ClaimsChangeList - AllocationsChanges AllocationsChangeList + ClaimChanges ClaimsChangeMap + AllocationsChanges AllocationsChangeMap } -func (s *StateDiffResult) Kind() string { +func (sd *StateDiffResult) Kind() string { return "verifreg" } -func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - //TODO implement me - panic("implement me") +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + out := &StateChange{} + adtStore := store.WrapBlockStore(ctx, bs) + + if verifiers := sd.VerifierChanges; verifiers != nil { + root, err := verifiers.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Verifiers = &root + } + + if claims := sd.ClaimChanges; claims != nil { + root, err := claims.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Claims = &root + } + + if allocations := sd.AllocationsChanges; allocations != nil { + root, err := allocations.ToAdtMap(adtStore, 5) + if err != nil { + return nil, err + } + out.Allocations = &root + } + return out, nil } type StateChange struct { - Verifiers cid.Cid `cborgen:"verifiers"` - Claims cid.Cid `cborgen:"claims"` - Allocations cid.Cid `cborgen:"allocations"` + Verifiers *cid.Cid `cborgen:"verifiers"` + Claims *cid.Cid `cborgen:"claims"` + Allocations *cid.Cid `cborgen:"allocations"` } type StateDiff struct { @@ -57,9 +83,9 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors case v0.KindVerifregVerifiers: stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) case KindVerifregClaims: - stateDiff.ClaimChanges = stateChange.(ClaimsChangeList) + stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) case KindVerifregAllocations: - stateDiff.AllocationsChanges = stateChange.(AllocationsChangeList) + stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) } } log.Infow("Extracted Verified Registry State Diff", "address", act.Address, "duration", time.Since(start)) diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go index 1f9c19e8b..fcaea63e1 100644 --- a/pkg/extract/actors/verifregdiff/version.go +++ b/pkg/extract/actors/verifregdiff/version.go @@ -72,7 +72,8 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { DiffMethods: []actors.ActorStateDiff{ v9.Verifiers{}, v9.Claims{}, - v9.Allocations{}, + // TODO + //v9.Allocations{}, }}, nil case actortypes.Version10: panic("Not yet implemented") diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 9f6a099de..9c721940c 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -5,7 +5,9 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - minerv0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" ) @@ -25,6 +27,12 @@ const verifDiffPkgV0 = "v0" const verifDiffPathV9 = "pkg/extract/actors/verifregdiff/v9/cbor_gen.go" const verifDiffPkgV9 = "v9" +const marketDiffPath = "pkg/extract/actors/marketdiff/v0/cbor_gen.go" +const marketDiffPkg = "v0" + +const powerDiffPath = "pkg/extract/actors/powerdiff/v0/cbor_gen.go" +const powerDiffPkg = "v0" + func main() { if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, actordiff.ActorChange{}, @@ -34,13 +42,13 @@ func main() { } if err := cbg.WriteMapEncodersToFile(minerDiffPath, minerDiffPkg, - minerv0.SectorStatusChange{}, - minerv0.PreCommitChange{}, - minerv0.SectorChange{}, - minerv0.FundsChange{}, - minerv0.DebtChange{}, - minerv0.InfoChange{}, - minerv0.StateChange{}, + minerV0.SectorStatusChange{}, + minerV0.PreCommitChange{}, + minerV0.SectorChange{}, + minerV0.FundsChange{}, + minerV0.DebtChange{}, + minerV0.InfoChange{}, + minerV0.StateChange{}, ); err != nil { panic(err) } @@ -68,4 +76,18 @@ func main() { panic(err) } + if err := cbg.WriteMapEncodersToFile(marketDiffPath, marketDiffPkg, + marketV0.StateChange{}, + marketV0.ProposalChange{}, + marketV0.DealChange{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(powerDiffPath, powerDiffPkg, + powerV0.StateChange{}, + powerV0.ClaimsChange{}, + ); err != nil { + panic(err) + } } diff --git a/pkg/transform/cbor/market/state.go b/pkg/transform/cbor/market/state.go new file mode 100644 index 000000000..d30235281 --- /dev/null +++ b/pkg/transform/cbor/market/state.go @@ -0,0 +1,21 @@ +package market + +import ( + "context" + + "github.com/filecoin-project/lotus/blockstore" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func HandleChange(ctx context.Context, bs blockstore.Blockstore, market actors.ActorDiffResult) (cid.Cid, error) { + store := adt2.WrapBlockStore(ctx, bs) + vsc, err := market.MarshalStateChange(ctx, bs) + if err != nil { + return cid.Undef, err + } + return store.Put(ctx, vsc) + +} diff --git a/pkg/transform/cbor/power/state.go b/pkg/transform/cbor/power/state.go new file mode 100644 index 000000000..f58e41525 --- /dev/null +++ b/pkg/transform/cbor/power/state.go @@ -0,0 +1,20 @@ +package power + +import ( + "context" + + "github.com/filecoin-project/lotus/blockstore" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func HandleChange(ctx context.Context, bs blockstore.Blockstore, power actors.ActorDiffResult) (cid.Cid, error) { + store := adt2.WrapBlockStore(ctx, bs) + vsc, err := power.MarshalStateChange(ctx, bs) + if err != nil { + return cid.Undef, err + } + return store.Put(ctx, vsc) +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index c66c5caf5..e6fd08fd9 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -15,7 +15,9 @@ import ( "github.com/filecoin-project/lily/pkg/extract/procesor" "github.com/filecoin-project/lily/pkg/transform/cbor/actor" "github.com/filecoin-project/lily/pkg/transform/cbor/init_" + "github.com/filecoin-project/lily/pkg/transform/cbor/market" "github.com/filecoin-project/lily/pkg/transform/cbor/miner" + "github.com/filecoin-project/lily/pkg/transform/cbor/power" "github.com/filecoin-project/lily/pkg/transform/cbor/verifreg" ) @@ -27,6 +29,8 @@ type ActorIPLDContainer struct { VerifregActor *cid.Cid // VerifregStateChange or empty ActorStates cid.Cid // HAMT[Address]ActorStateChange InitActor cid.Cid // HAMT[Address]AddressChanges. + MarketActor cid.Cid // MarketStateChange or empty + PowerActor cid.Cid // PowerStateChange or empty } func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { @@ -68,11 +72,13 @@ func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *proce CurrentTipSet: changes.Current, ExecutedTipSet: changes.Executed, } - minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) - if err != nil { - return nil, err + if changes.MinerActors != nil { + minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) + if err != nil { + return nil, err + } + out.MinerActors = minerRoot } - out.MinerActors = minerRoot if changes.VerifregActor != nil { verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) @@ -97,5 +103,21 @@ func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *proce } out.InitActor = initRoot } + + if changes.MarketActor != nil { + marketRoot, err := market.HandleChange(ctx, bs, changes.MarketActor) + if err != nil { + return nil, err + } + out.MarketActor = marketRoot + } + + if changes.PowerActor != nil { + powerRoot, err := power.HandleChange(ctx, bs, changes.PowerActor) + if err != nil { + return nil, err + } + out.PowerActor = powerRoot + } return out, nil } diff --git a/pkg/transform/cbor/verifreg/state.go b/pkg/transform/cbor/verifreg/state.go index 0bc8640b1..3ee098d9a 100644 --- a/pkg/transform/cbor/verifreg/state.go +++ b/pkg/transform/cbor/verifreg/state.go @@ -12,5 +12,9 @@ import ( func HandleChanges(ctx context.Context, bs blockstore.Blockstore, verifreg actors.ActorDiffResult) (cid.Cid, error) { store := adt2.WrapBlockStore(ctx, bs) - return store.Put(ctx, verifreg) + vsc, err := verifreg.MarshalStateChange(ctx, bs) + if err != nil { + return cid.Undef, err + } + return store.Put(ctx, vsc) } From 8da1603ba20dace8c8e6b58446bb859c2049978d Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 12 Jan 2023 18:35:22 -0800 Subject: [PATCH 21/48] working --- go.mod | 6 +- go.sum | 12 +- pkg/core/versions.go | 54 +++++++ pkg/extract/actors/actordiff/state.go | 1 + pkg/extract/actors/minerdiff/v0/state.go | 2 +- pkg/extract/actors/minerdiff/v2/state.go | 2 + pkg/extract/actors/minerdiff/v3/state.go | 2 + pkg/extract/actors/minerdiff/v4/state.go | 2 + pkg/extract/actors/minerdiff/v5/state.go | 2 + pkg/extract/actors/minerdiff/v6/state.go | 2 + pkg/extract/actors/minerdiff/v7/state.go | 2 + pkg/extract/actors/minerdiff/v8/state.go | 2 + pkg/extract/actors/minerdiff/v9/state.go | 2 + pkg/extract/procesor/process.go | 42 +---- pkg/transform/cbor/router.go | 19 +++ .../timescale/actors/miner/router.go | 77 +++++++++ .../timescale/actors/miner/util/deals.go | 31 ++++ .../timescale/actors/miner/util/info.go | 80 +++++++++ .../timescale/actors/miner/v0/debt.go | 23 --- .../timescale/actors/miner/v0/funds.go | 24 --- .../timescale/actors/miner/v0/info.go | 127 ++++++++------- .../timescale/actors/miner/v0/precommits.go | 28 ++-- .../timescale/actors/miner/v0/processor.go | 68 +++++--- .../timescale/actors/miner/v0/sector_deals.go | 69 +++----- .../actors/miner/v0/sector_events.go | 52 +++--- .../timescale/actors/miner/v0/sectors.go | 32 ++-- .../timescale/actors/miner/v0/state.go | 63 ++++++++ .../timescale/actors/miner/v2/info.go | 90 +++++++++++ .../timescale/actors/miner/v2/precommits.go | 65 ++++++++ .../timescale/actors/miner/v2/processor.go | 71 +++++--- .../timescale/actors/miner/v2/sector_deals.go | 62 +++++++ .../actors/miner/v2/sector_events.go | 143 ++++++++++++++++ .../timescale/actors/miner/v2/sectors.go | 59 +++++++ .../timescale/actors/miner/v2/state.go | 70 ++++++++ .../timescale/actors/miner/v3/info.go | 90 +++++++++++ .../timescale/actors/miner/v3/precommits.go | 65 ++++++++ .../timescale/actors/miner/v3/processor.go | 88 ++++++++++ .../timescale/actors/miner/v3/sector_deals.go | 62 +++++++ .../actors/miner/v3/sector_events.go | 143 ++++++++++++++++ .../timescale/actors/miner/v3/sectors.go | 59 +++++++ .../timescale/actors/miner/v3/state.go | 70 ++++++++ .../timescale/actors/miner/v4/info.go | 90 +++++++++++ .../timescale/actors/miner/v4/precommits.go | 65 ++++++++ .../timescale/actors/miner/v4/processor.go | 88 ++++++++++ .../timescale/actors/miner/v4/sector_deals.go | 62 +++++++ .../actors/miner/v4/sector_events.go | 143 ++++++++++++++++ .../timescale/actors/miner/v4/sectors.go | 59 +++++++ .../timescale/actors/miner/v4/state.go | 70 ++++++++ .../timescale/actors/miner/v5/info.go | 90 +++++++++++ .../timescale/actors/miner/v5/precommits.go | 65 ++++++++ .../timescale/actors/miner/v5/processor.go | 88 ++++++++++ .../timescale/actors/miner/v5/sector_deals.go | 62 +++++++ .../actors/miner/v5/sector_events.go | 143 ++++++++++++++++ .../timescale/actors/miner/v5/sectors.go | 59 +++++++ .../timescale/actors/miner/v5/state.go | 70 ++++++++ .../timescale/actors/miner/v6/info.go | 90 +++++++++++ .../timescale/actors/miner/v6/precommits.go | 65 ++++++++ .../timescale/actors/miner/v6/processor.go | 88 ++++++++++ .../timescale/actors/miner/v6/sector_deals.go | 62 +++++++ .../actors/miner/v6/sector_events.go | 143 ++++++++++++++++ .../timescale/actors/miner/v6/sectors.go | 59 +++++++ .../timescale/actors/miner/v6/state.go | 70 ++++++++ .../timescale/actors/miner/v7/info.go | 90 +++++++++++ .../timescale/actors/miner/v7/precommits.go | 65 ++++++++ .../timescale/actors/miner/v7/processor.go | 88 ++++++++++ .../timescale/actors/miner/v7/sector_deals.go | 62 +++++++ .../actors/miner/v7/sector_events.go | 152 ++++++++++++++++++ .../timescale/actors/miner/v7/sectors.go | 64 ++++++++ .../timescale/actors/miner/v7/state.go | 70 ++++++++ .../timescale/actors/miner/v8/info.go | 90 +++++++++++ .../timescale/actors/miner/v8/precommits.go | 65 ++++++++ .../timescale/actors/miner/v8/processor.go | 88 ++++++++++ .../timescale/actors/miner/v8/sector_deals.go | 62 +++++++ .../actors/miner/v8/sector_events.go | 152 ++++++++++++++++++ .../timescale/actors/miner/v8/sectors.go | 64 ++++++++ .../timescale/actors/miner/v8/state.go | 70 ++++++++ .../timescale/actors/miner/v9/debt.go | 26 --- .../timescale/actors/miner/v9/funds.go | 29 ---- .../timescale/actors/miner/v9/info.go | 122 +++++++------- .../timescale/actors/miner/v9/precommits.go | 20 +-- .../timescale/actors/miner/v9/processor.go | 64 ++++---- .../timescale/actors/miner/v9/sector_deals.go | 65 +++----- .../actors/miner/v9/sector_events.go | 23 +-- .../timescale/actors/miner/v9/sectors.go | 21 +-- .../timescale/actors/miner/v9/state.go | 70 ++++++++ pkg/transform/timescale/actors/processor.go | 82 ++++++++-- pkg/transform/timescale/actors/raw/state.go | 33 ++++ .../timescale/actors/reward/router.go | 49 ++++++ .../timescale/actors/reward/v0/state.go | 39 +++++ .../timescale/actors/reward/v2/state.go | 39 +++++ .../timescale/actors/reward/v3/state.go | 39 +++++ .../timescale/actors/reward/v4/state.go | 39 +++++ .../timescale/actors/reward/v5/state.go | 39 +++++ .../timescale/actors/reward/v6/state.go | 39 +++++ .../timescale/actors/reward/v7/state.go | 39 +++++ .../timescale/actors/reward/v8/state.go | 39 +++++ .../timescale/actors/reward/v9/state.go | 39 +++++ 97 files changed, 5282 insertions(+), 549 deletions(-) create mode 100644 pkg/transform/timescale/actors/miner/util/deals.go create mode 100644 pkg/transform/timescale/actors/miner/util/info.go delete mode 100644 pkg/transform/timescale/actors/miner/v0/debt.go delete mode 100644 pkg/transform/timescale/actors/miner/v0/funds.go create mode 100644 pkg/transform/timescale/actors/miner/v0/state.go create mode 100644 pkg/transform/timescale/actors/miner/v2/info.go create mode 100644 pkg/transform/timescale/actors/miner/v2/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v2/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v2/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v2/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v2/state.go create mode 100644 pkg/transform/timescale/actors/miner/v3/info.go create mode 100644 pkg/transform/timescale/actors/miner/v3/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v3/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v3/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v3/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v3/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v3/state.go create mode 100644 pkg/transform/timescale/actors/miner/v4/info.go create mode 100644 pkg/transform/timescale/actors/miner/v4/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v4/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v4/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v4/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v4/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v4/state.go create mode 100644 pkg/transform/timescale/actors/miner/v5/info.go create mode 100644 pkg/transform/timescale/actors/miner/v5/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v5/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v5/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v5/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v5/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v5/state.go create mode 100644 pkg/transform/timescale/actors/miner/v6/info.go create mode 100644 pkg/transform/timescale/actors/miner/v6/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v6/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v6/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v6/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v6/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v6/state.go create mode 100644 pkg/transform/timescale/actors/miner/v7/info.go create mode 100644 pkg/transform/timescale/actors/miner/v7/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v7/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v7/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v7/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v7/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v7/state.go create mode 100644 pkg/transform/timescale/actors/miner/v8/info.go create mode 100644 pkg/transform/timescale/actors/miner/v8/precommits.go create mode 100644 pkg/transform/timescale/actors/miner/v8/processor.go create mode 100644 pkg/transform/timescale/actors/miner/v8/sector_deals.go create mode 100644 pkg/transform/timescale/actors/miner/v8/sector_events.go create mode 100644 pkg/transform/timescale/actors/miner/v8/sectors.go create mode 100644 pkg/transform/timescale/actors/miner/v8/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v9/debt.go delete mode 100644 pkg/transform/timescale/actors/miner/v9/funds.go create mode 100644 pkg/transform/timescale/actors/miner/v9/state.go create mode 100644 pkg/transform/timescale/actors/raw/state.go create mode 100644 pkg/transform/timescale/actors/reward/router.go create mode 100644 pkg/transform/timescale/actors/reward/v0/state.go create mode 100644 pkg/transform/timescale/actors/reward/v2/state.go create mode 100644 pkg/transform/timescale/actors/reward/v3/state.go create mode 100644 pkg/transform/timescale/actors/reward/v4/state.go create mode 100644 pkg/transform/timescale/actors/reward/v5/state.go create mode 100644 pkg/transform/timescale/actors/reward/v6/state.go create mode 100644 pkg/transform/timescale/actors/reward/v7/state.go create mode 100644 pkg/transform/timescale/actors/reward/v8/state.go create mode 100644 pkg/transform/timescale/actors/reward/v9/state.go diff --git a/go.mod b/go.mod index 44fed0f42..2acefecca 100644 --- a/go.mod +++ b/go.mod @@ -69,11 +69,10 @@ require ( github.com/ipfs/go-ipld-format v0.4.0 github.com/jedib0t/go-pretty/v6 v6.2.7 github.com/libp2p/go-libp2p v0.22.0 + github.com/libp2p/go-libp2p-peer v0.2.0 github.com/multiformats/go-varint v0.0.6 go.opentelemetry.io/otel/trace v1.7.0 go.uber.org/atomic v1.10.0 - gorm.io/driver/sqlite v1.4.3 - gorm.io/gorm v1.24.2 ) require ( @@ -221,7 +220,6 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jessevdk/go-flags v1.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect - github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect @@ -234,6 +232,7 @@ require ( github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-connmgr v0.4.0 // indirect + github.com/libp2p/go-libp2p-crypto v0.1.0 // indirect github.com/libp2p/go-libp2p-gostream v0.4.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.18.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect @@ -262,7 +261,6 @@ require ( github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect diff --git a/go.sum b/go.sum index ef5e87299..f1561827e 100644 --- a/go.sum +++ b/go.sum @@ -864,9 +864,6 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= -github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= -github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= -github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1021,6 +1018,7 @@ github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQR github.com/libp2p/go-libp2p-core v0.13.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.20.0 h1:PGKM74+T+O/FaZNARNW32i90RMBHCcgd/hkum2UQ5eY= github.com/libp2p/go-libp2p-core v0.20.0/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= +github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= @@ -1049,6 +1047,7 @@ github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhU github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ= github.com/libp2p/go-libp2p-noise v0.5.0 h1:gwJZ/3iH3MRnBrLIyr/YLdCOnmqfJMptlsFFUIc3j0Y= github.com/libp2p/go-libp2p-noise v0.5.0/go.mod h1:CYYpJ+T4pox1d1J/hhUVxuMf6b2s3c41hFUgS8/yFQw= +github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= @@ -1252,8 +1251,6 @@ github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/mattn/go-xmlrpc v0.0.3/go.mod h1:mqc2dz7tP5x5BKlCahN/n+hs7OSZKJkS9JsHNBRlrxA= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -2342,11 +2339,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/sqlite v1.4.3 h1:HBBcZSDnWi5BW3B3rwvVTc510KGkBkexlOg0QrmLUuU= -gorm.io/driver/sqlite v1.4.3/go.mod h1:0Aq3iPO+v9ZKbcdiz8gLWRw5VOPcBOPUQJFLq5e2ecI= -gorm.io/gorm v1.24.0/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= -gorm.io/gorm v1.24.2 h1:9wR6CFD+G8nOusLdvkZelOEhpJVwwHzpQOUM+REd6U0= -gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/pkg/core/versions.go b/pkg/core/versions.go index 1408db07e..94d853daa 100644 --- a/pkg/core/versions.go +++ b/pkg/core/versions.go @@ -6,10 +6,64 @@ import ( "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" ) func ActorVersionForTipSet(ctx context.Context, ts *types.TipSet, ntwkVersionGetter func(ctx context.Context, epoch abi.ChainEpoch) network.Version) (actorstypes.Version, error) { ntwkVersion := ntwkVersionGetter(ctx, ts.Height()) return actorstypes.VersionForNetwork(ntwkVersion) } + +var ( + AccountCodes = cid.NewSet() + CronCodes = cid.NewSet() + DataCapCodes = cid.NewSet() + InitCodes = cid.NewSet() + MarketCodes = cid.NewSet() + MinerCodes = cid.NewSet() + MultisigCodes = cid.NewSet() + PaychCodes = cid.NewSet() + PowerCodes = cid.NewSet() + RewardCodes = cid.NewSet() + SystemCodes = cid.NewSet() + VerifregCodes = cid.NewSet() +) + +func init() { + for _, a := range []string{actors.AccountKey, actors.CronKey, actors.DatacapKey, actors.InitKey, actors.MarketKey, actors.MinerKey, actors.MultisigKey, actors.PaychKey, actors.PowerKey, actors.RewardKey, actors.SystemKey, actors.VerifregKey} { + for _, v := range []int{0, 2, 3, 4, 5, 6, 7, 8, 9} { + code, ok := actors.GetActorCodeID(actorstypes.Version(v), a) + if !ok { + continue + } + switch a { + case actors.AccountKey: + AccountCodes.Add(code) + case actors.CronKey: + CronCodes.Add(code) + case actors.DatacapKey: + DataCapCodes.Add(code) + case actors.InitKey: + InitCodes.Add(code) + case actors.MarketKey: + MarketCodes.Add(code) + case actors.MinerKey: + MinerCodes.Add(code) + case actors.MultisigKey: + MultisigCodes.Add(code) + case actors.PaychKey: + PaychCodes.Add(code) + case actors.PowerKey: + PowerCodes.Add(code) + case actors.RewardKey: + RewardCodes.Add(code) + case actors.SystemKey: + SystemCodes.Add(code) + case actors.VerifregKey: + VerifregCodes.Add(code) + } + } + } +} diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/actordiff/state.go index 811ba250d..9eedc6e4d 100644 --- a/pkg/extract/actors/actordiff/state.go +++ b/pkg/extract/actors/actordiff/state.go @@ -17,6 +17,7 @@ type StateDiffResult struct { } func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { + return s.ActorStateChanges, nil out := &StateChange{} if actorChanges := s.ActorStateChanges; actorChanges != nil { diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v0/state.go index 45bc0654d..7db40d07d 100644 --- a/pkg/extract/actors/minerdiff/v0/state.go +++ b/pkg/extract/actors/minerdiff/v0/state.go @@ -98,7 +98,7 @@ type StateChange struct { Sectors cid.Cid `cborgen:"sectors"` } -func (sc *StateChange) StateChangeAsStateDiffResult(ctx context.Context, bs blockstore.Blockstore) (*StateDiffResult, error) { +func DecodeStateDiffResultFromStateChange(ctx context.Context, bs blockstore.Blockstore, sc *StateChange) (*StateDiffResult, error) { out := &StateDiffResult{} adtStore := store.WrapBlockStore(ctx, bs) diff --git a/pkg/extract/actors/minerdiff/v2/state.go b/pkg/extract/actors/minerdiff/v2/state.go index 69c74dac0..9fe4be164 100644 --- a/pkg/extract/actors/minerdiff/v2/state.go +++ b/pkg/extract/actors/minerdiff/v2/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v3/state.go b/pkg/extract/actors/minerdiff/v3/state.go index d3dd5a70b..75f47ca25 100644 --- a/pkg/extract/actors/minerdiff/v3/state.go +++ b/pkg/extract/actors/minerdiff/v3/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v4/state.go b/pkg/extract/actors/minerdiff/v4/state.go index ea583d7fe..784c40dcd 100644 --- a/pkg/extract/actors/minerdiff/v4/state.go +++ b/pkg/extract/actors/minerdiff/v4/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v5/state.go b/pkg/extract/actors/minerdiff/v5/state.go index 95b671e65..f5636c4ad 100644 --- a/pkg/extract/actors/minerdiff/v5/state.go +++ b/pkg/extract/actors/minerdiff/v5/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v6/state.go b/pkg/extract/actors/minerdiff/v6/state.go index c3545513b..3d7175e60 100644 --- a/pkg/extract/actors/minerdiff/v6/state.go +++ b/pkg/extract/actors/minerdiff/v6/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v7/state.go b/pkg/extract/actors/minerdiff/v7/state.go index dac14967c..9b7332d07 100644 --- a/pkg/extract/actors/minerdiff/v7/state.go +++ b/pkg/extract/actors/minerdiff/v7/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v8/state.go b/pkg/extract/actors/minerdiff/v8/state.go index d2a72308d..1314b2693 100644 --- a/pkg/extract/actors/minerdiff/v8/state.go +++ b/pkg/extract/actors/minerdiff/v8/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v9/state.go b/pkg/extract/actors/minerdiff/v9/state.go index 287cffc2c..71b9f53b8 100644 --- a/pkg/extract/actors/minerdiff/v9/state.go +++ b/pkg/extract/actors/minerdiff/v9/state.go @@ -5,3 +5,5 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff + +var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/procesor/process.go b/pkg/extract/procesor/process.go index 741ce2710..51d74f906 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/procesor/process.go @@ -8,17 +8,11 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" - init_ "github.com/filecoin-project/lily/chain/actors/builtin/init" - "github.com/filecoin-project/lily/chain/actors/builtin/market" - "github.com/filecoin-project/lily/chain/actors/builtin/miner" - "github.com/filecoin-project/lily/chain/actors/builtin/power" - "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" @@ -33,32 +27,6 @@ import ( var log = logging.Logger("lily/extract/processor") -var ( - InitCodes = cid.NewSet() - MinerCodes = cid.NewSet() - PowerCodes = cid.NewSet() - MarketCodes = cid.NewSet() - VerifregCodes = cid.NewSet() -) - -func init() { - for _, c := range miner.AllCodes() { - MinerCodes.Add(c) - } - for _, c := range power.AllCodes() { - PowerCodes.Add(c) - } - for _, c := range market.AllCodes() { - MarketCodes.Add(c) - } - for _, c := range verifreg.AllCodes() { - VerifregCodes.Add(c) - } - for _, c := range init_.AllCodes() { - InitCodes.Add(c) - } -} - type ActorStateChanges struct { Current *types.TipSet Executed *types.TipSet @@ -132,7 +100,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current ActorDiff: actorStateChanges, Address: addr, } - if MinerCodes.Has(change.Current.Code) { + if core.MinerCodes.Has(change.Current.Code) { start := time.Now() // construct the state differ required by this actor version actorDiff, err := minerdiff.StateDiffFor(actorVersion) @@ -150,7 +118,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } - if VerifregCodes.Has(change.Current.Code) { + if core.VerifregCodes.Has(change.Current.Code) { start := time.Now() // construct the state differ required by this actor version actorDiff, err := verifregdiff.StateDiffFor(actorVersion) @@ -168,7 +136,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } - if InitCodes.Has(change.Current.Code) { + if core.InitCodes.Has(change.Current.Code) { start := time.Now() actorDiff, err := initdiff.StateDiffFor(actorVersion) if err != nil { @@ -184,7 +152,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } - if PowerCodes.Has(change.Current.Code) { + if core.PowerCodes.Has(change.Current.Code) { start := time.Now() actorDiff, err := powerdiff.StateDiffFor(actorVersion) if err != nil { @@ -200,7 +168,7 @@ func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current Address: addr, } } - if MarketCodes.Has(change.Current.Code) { + if core.MarketCodes.Has(change.Current.Code) { start := time.Now() actorDiff, err := marketdiff.StateDiffFor(actorVersion) if err != nil { diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index e6fd08fd9..7953a625d 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -8,7 +8,10 @@ import ( "github.com/filecoin-project/lotus/chain/types" adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car/util" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" v1car "github.com/ipld/go-car" @@ -21,6 +24,8 @@ import ( "github.com/filecoin-project/lily/pkg/transform/cbor/verifreg" ) +var log = logging.Logger("lily/transform/cbor") + type ActorIPLDContainer struct { // TODO this needs to be versioned CurrentTipSet *types.TipSet @@ -33,6 +38,19 @@ type ActorIPLDContainer struct { PowerActor cid.Cid // PowerStateChange or empty } +func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("current", a.CurrentTipSet.String()), + attribute.String("executed", a.ExecutedTipSet.String()), + attribute.String("miners", a.MinerActors.String()), + attribute.String("verifreg", a.VerifregActor.String()), + attribute.String("actors", a.ActorStates.String()), + attribute.String("init", a.InitActor.String()), + attribute.String("market", a.MarketActor.String()), + attribute.String("power", a.PowerActor.String()), + } +} + func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { bs := blockstore.NewMemorySync() store := adt2.WrapBlockStore(ctx, bs) @@ -45,6 +63,7 @@ func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io if err != nil { return err } + log.Infow("Wrote Delta", "root", actorStatesRoot.String(), actorStates) if err := v1car.WriteHeader(&v1car.CarHeader{ Roots: []cid.Cid{actorStatesRoot}, Version: 1, diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index ab865910b..3a134ff1f 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -1 +1,78 @@ package miner + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" + v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v3" + v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v4" + v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v5" + v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v6" + v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v7" + v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v8" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v9" +) + +func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange, version actortypes.Version) (model.Persistable, error) { + switch version { + case actortypes.Version0: + return v0.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version2: + return v2.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version3: + return v3.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version4: + return v4.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version5: + return v5.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version6: + return v6.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version7: + return v7.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version8: + return v8.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version9: + return v9.MinerStateHandler(ctx, current, executed, addr, change) + case actortypes.Version10: + panic("not yet implemented") + default: + return nil, fmt.Errorf("unsupported miner actor version: %d", version) + } +} + +type MinerHandler = func(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) + +func MakeMinerProcessor(av actortypes.Version) (MinerHandler, error) { + switch av { + case actortypes.Version0: + return v0.MinerHandler, nil + case actortypes.Version2: + return v2.MinerHandler, nil + case actortypes.Version3: + return v3.MinerHandler, nil + case actortypes.Version4: + return v4.MinerHandler, nil + case actortypes.Version5: + return v5.MinerHandler, nil + case actortypes.Version6: + return v6.MinerHandler, nil + case actortypes.Version7: + return v7.MinerHandler, nil + case actortypes.Version8: + return v8.MinerHandler, nil + case actortypes.Version9: + return v9.MinerHandler, nil + default: + return nil, fmt.Errorf("unsupported miner actor version: %d", av) + } +} diff --git a/pkg/transform/timescale/actors/miner/util/deals.go b/pkg/transform/timescale/actors/miner/util/deals.go new file mode 100644 index 000000000..48ee55b73 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/util/deals.go @@ -0,0 +1,31 @@ +package util + +import "github.com/filecoin-project/go-state-types/abi" + +func CompareDealIDs(cur, pre []abi.DealID) []abi.DealID { + var diff []abi.DealID + + // Loop two times, first to find cur dealIDs not in pre, + // second loop to find pre dealIDs not in cur + for i := 0; i < 2; i++ { + for _, s1 := range cur { + found := false + for _, s2 := range pre { + if s1 == s2 { + found = true + break + } + } + // DealID not found. We add it to return slice + if !found { + diff = append(diff, s1) + } + } + // Swap the slices, only if it was the first loop + if i == 0 { + cur, pre = pre, cur + } + } + + return diff +} diff --git a/pkg/transform/timescale/actors/miner/util/info.go b/pkg/transform/timescale/actors/miner/util/info.go new file mode 100644 index 000000000..89f8c6dd4 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/util/info.go @@ -0,0 +1,80 @@ +package util + +import ( + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "github.com/libp2p/go-libp2p/core/peer" + maddr "github.com/multiformats/go-multiaddr" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" +) + +type WorkerKeyChanges interface { + NewWorker() address.Address + EffectiveAt() abi.ChainEpoch +} + +type MinerInfo interface { + PendingWorkerKey() (WorkerKeyChanges, bool) + ControlAddresses() []address.Address + Multiaddrs() []abi.Multiaddrs + Owner() address.Address + Worker() address.Address + SectorSize() abi.SectorSize + PeerId() abi.PeerID +} + +func ExtractMinerInfo(ctx context.Context, current, executed *types.TipSet, addr address.Address, info MinerInfo) (model.Persistable, error) { + var newWorker string + var newWorkerEpoch int64 + if pendingWorkerKey, changed := info.PendingWorkerKey(); changed { + if pendingWorkerKey.NewWorker() != address.Undef { + newWorker = pendingWorkerKey.NewWorker().String() + } + newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt()) + } + + var newCtrlAddresses []string + for _, addr := range info.ControlAddresses() { + newCtrlAddresses = append(newCtrlAddresses, addr.String()) + } + + // best effort to decode, we have no control over what miners put in this field, its just bytes. + var newMultiAddrs []string + for _, addr := range info.Multiaddrs() { + newMaddr, err := maddr.NewMultiaddrBytes(addr) + if err == nil { + newMultiAddrs = append(newMultiAddrs, newMaddr.String()) + } else { + //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) + } + } + mi := &minermodel.MinerInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + OwnerID: info.Owner().String(), + WorkerID: info.Worker().String(), + NewWorker: newWorker, + WorkerChangeEpoch: newWorkerEpoch, + ConsensusFaultedElapsed: -1, + ControlAddresses: newCtrlAddresses, + MultiAddresses: newMultiAddrs, + SectorSize: uint64(info.SectorSize()), + } + + if info.PeerId() != nil { + newPeerID, err := peer.IDFromBytes(info.PeerId()) + if err != nil { + //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) + } else { + mi.PeerID = newPeerID.String() + } + } + + return mi, nil +} diff --git a/pkg/transform/timescale/actors/miner/v0/debt.go b/pkg/transform/timescale/actors/miner/v0/debt.go deleted file mode 100644 index 61453a8be..000000000 --- a/pkg/transform/timescale/actors/miner/v0/debt.go +++ /dev/null @@ -1,23 +0,0 @@ -package v0 - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" -) - -type Debt struct{} - -func (Debt) Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { - return &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - MinerID: addr.String(), - FeeDebt: change.DebtChange.FeeDebt.String(), - }, nil -} diff --git a/pkg/transform/timescale/actors/miner/v0/funds.go b/pkg/transform/timescale/actors/miner/v0/funds.go deleted file mode 100644 index 36574d00c..000000000 --- a/pkg/transform/timescale/actors/miner/v0/funds.go +++ /dev/null @@ -1,24 +0,0 @@ -package v0 - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/chain/actors/adt" - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" -) - -func HandleMinerFundsChange(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes *v0.FundsChange) (model.Persistable, error) { - return &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: changes.VestingFunds.String(), - InitialPledge: changes.InitialPledgeRequirement.String(), - PreCommitDeposits: changes.PreCommitDeposit.String(), - }, nil -} diff --git a/pkg/transform/timescale/actors/miner/v0/info.go b/pkg/transform/timescale/actors/miner/v0/info.go index 10a8c80b5..c7e0cba8f 100644 --- a/pkg/transform/timescale/actors/miner/v0/info.go +++ b/pkg/transform/timescale/actors/miner/v0/info.go @@ -1,87 +1,90 @@ package v0 import ( + "bytes" "context" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/libp2p/go-libp2p/core/peer" - maddr "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -func HandleMinerInfo(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { - info := change.InfoChange - var out model.Persistable - var err error - if err := core.StateReadDeferred(ctx, info.Info, func(in *miner0.MinerInfo) error { - out, err = MinerInfoAsModel(ctx, current, executed, addr, *in) - if err != nil { - return err - } - return nil - }); err != nil { +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { return nil, err } return out, nil } -func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner0.MinerInfo) (model.Persistable, error) { - return GenericMinerInfoAsModel(ctx, current, executed, addr, info) +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo } -func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner0.MinerInfo) (model.Persistable, error) { - var newWorker string - var newWorkerEpoch int64 - if pendingWorkerKey := info.PendingWorkerKey; pendingWorkerKey != nil { - if pendingWorkerKey.NewWorker != address.Undef { - newWorker = pendingWorkerKey.NewWorker.String() - } - newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt) - } +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} - var newCtrlAddresses []string - for _, addr := range info.ControlAddresses { - newCtrlAddresses = append(newCtrlAddresses, addr.String()) - } +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} - // best effort to decode, we have no control over what miners put in this field, its just bytes. - var newMultiAddrs []string - for _, addr := range info.Multiaddrs { - newMaddr, err := maddr.NewMultiaddrBytes(addr) - if err == nil { - newMultiAddrs = append(newMultiAddrs, newMaddr.String()) - } else { - //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) - } - } - mi := &minermodel.MinerInfo{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - OwnerID: info.Owner.String(), - WorkerID: info.Worker.String(), - NewWorker: newWorker, - WorkerChangeEpoch: newWorkerEpoch, - ConsensusFaultedElapsed: -1, - ControlAddresses: newCtrlAddresses, - MultiAddresses: newMultiAddrs, - SectorSize: uint64(info.SectorSize), - } +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} - if info.PeerId != nil { - newPeerID, err := peer.IDFromBytes(info.PeerId) - if err != nil { - //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) - } else { - mi.PeerID = newPeerID.String() - } +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} - return mi, nil +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId } diff --git a/pkg/transform/timescale/actors/miner/v0/precommits.go b/pkg/transform/timescale/actors/miner/v0/precommits.go index 7ff8498b6..50ace3984 100644 --- a/pkg/transform/timescale/actors/miner/v0/precommits.go +++ b/pkg/transform/timescale/actors/miner/v0/precommits.go @@ -1,45 +1,51 @@ package v0 import ( + "bytes" "context" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -type PreCommits struct{} +type PreCommit struct{} -func (PreCommits) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) { - var precommits []*miner0.SectorPreCommitOnChainInfo +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added if change.Change != core.ChangeTypeAdd { continue } - if err := core.StateReadDeferred(ctx, change.Current, func(precommit *miner0.SectorPreCommitOnChainInfo) error { - precommits = append(precommits, precommit) - return nil - }); err != nil { + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { return nil, err } + precommits = append(precommits, precommit) } return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner0.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } preCommitModel[i] = &minermodel.MinerPreCommitInfo{ Height: int64(current.Height()), - StateRoot: current.ParentState().String(), MinerID: addr.String(), SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), SealedCID: preCommit.Info.SealedCID.String(), SealRandEpoch: int64(preCommit.Info.SealRandEpoch), ExpirationEpoch: int64(preCommit.Info.Expiration), diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go index 6c978179f..bccb4c384 100644 --- a/pkg/transform/timescale/actors/miner/v0/processor.go +++ b/pkg/transform/timescale/actors/miner/v0/processor.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/store" + store "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -12,29 +12,67 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" ) +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) if err != nil { return nil, err } - minerState := new(v0.StateChange) + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) if err != nil { return err } + // register extractors to run over the miners state. stateExtractor := &StateExtract{ ExtractMethods: []Extractor{ - // TODO + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, }, } models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) @@ -48,23 +86,3 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut } return out, nil } - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v0.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v0.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v0/sector_deals.go index 6ff4d42d3..dda6fcdc3 100644 --- a/pkg/transform/timescale/actors/miner/v0/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v0/sector_deals.go @@ -5,47 +5,50 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, sectors v0.SectorChangeList) (model.Persistable, error) { +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) minerAddr := addr.String() for _, sector := range sectors { switch sector.Change { case core.ChangeTypeAdd: - if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner0.SectorOnChainInfo) error { - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } case core.ChangeTypeModify: - previousSector := new(miner0.SectorOnChainInfo) + previousSector := new(miner.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - currentSector := new(miner0.SectorOnChainInfo) + currentSector := new(miner.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } - for _, deal := range compareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { out = append(out, &minermodel.MinerSectorDeal{ Height: height, MinerID: minerAddr, @@ -57,31 +60,3 @@ func HandleMinerSectorDeals(ctx context.Context, store adt.Store, current, execu } return out, nil } - -func compareDealIDs(cur, pre []abi.DealID) []abi.DealID { - var diff []abi.DealID - - // Loop two times, first to find cur dealIDs not in pre, - // second loop to find pre dealIDs not in cur - for i := 0; i < 2; i++ { - for _, s1 := range cur { - found := false - for _, s2 := range pre { - if s1 == s2 { - found = true - break - } - } - // DealID not found. We add it to return slice - if !found { - diff = append(diff, s1) - } - } - // Swap the slices, only if it was the first loop - if i == 0 { - cur, pre = pre, cur - } - } - - return diff -} diff --git a/pkg/transform/timescale/actors/miner/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v0/sector_events.go index ba00fa676..8f117cfb3 100644 --- a/pkg/transform/timescale/actors/miner/v0/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v0/sector_events.go @@ -7,20 +7,28 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, precommits v0.PreCommitChangeList, sectors v0.SectorChangeList, sectorstatus *v0.SectorStatusChange) (model.Persistable, error) { - out := minermodel.MinerSectorEventList{} - height := int64(current.Height()) - minerAddr := addr.String() - stateRoot := current.ParentState().String() +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) for _, precommit := range precommits { // only care about new precommits if precommit.Change != core.ChangeTypeAdd { @@ -42,14 +50,13 @@ func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, exec switch sector.Change { case core.ChangeTypeAdd: event := minermodel.SectorAdded - if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner0.SectorOnChainInfo) error { - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded - } - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } out = append(out, &minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, @@ -58,11 +65,11 @@ func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, exec Event: event, }) case core.ChangeTypeModify: - previousSector := new(miner0.SectorOnChainInfo) + previousSector := new(miner.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - currentSector := new(miner0.SectorOnChainInfo) + currentSector := new(miner.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } @@ -75,19 +82,6 @@ func HandleMinerSectorEvents(ctx context.Context, store adt.Store, current, exec Event: minermodel.SectorExtended, }) } - // snapping didn't exist at this epoch - /* - if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sector.SectorNumber, - StateRoot: stateRoot, - Event: minermodel.SectorSnapped, - }) - } - - */ } } if sectorstatus == nil { diff --git a/pkg/transform/timescale/actors/miner/v0/sectors.go b/pkg/transform/timescale/actors/miner/v0/sectors.go index 9795c8bde..492a42c86 100644 --- a/pkg/transform/timescale/actors/miner/v0/sectors.go +++ b/pkg/transform/timescale/actors/miner/v0/sectors.go @@ -1,45 +1,49 @@ package v0 import ( + "bytes" "context" "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" - miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" - "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -func HandleMinerSectorChanges(ctx context.Context, store adt.Store, current, executed *types.TipSet, addr address.Address, changes v0.SectorChangeList) (model.Persistable, error) { - var sectors []*miner0.SectorOnChainInfo - for _, change := range changes { +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { // only care about modified and added sectors - if change.Change == core.ChangeTypeRemove { + if sector.Change == core.ChangeTypeRemove { continue } - // change.Current is the newly added sector, or its state after modification. - if err := core.StateReadDeferred(ctx, change.Current, func(sector *miner0.SectorOnChainInfo) error { - sectors = append(sectors, sector) - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + sectors = append(sectors, s) } return MinerSectorChangesAsModel(ctx, current, addr, sectors) } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner0.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ Height: int64(current.Height()), MinerID: addr.String(), - StateRoot: current.ParentState().String(), SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), SealedCID: sector.SealedCID.String(), ActivationEpoch: int64(sector.Activation), ExpirationEpoch: int64(sector.Expiration), diff --git a/pkg/transform/timescale/actors/miner/v0/state.go b/pkg/transform/timescale/actors/miner/v0/state.go new file mode 100644 index 000000000..7a815f589 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v0/state.go @@ -0,0 +1,63 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/big" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: big.Zero().String(), // v0 has no fee debt + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledgeRequirement.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledgeRequirement.Equals(previousState.InitialPledgeRequirement) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledgeRequirement.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v2/info.go b/pkg/transform/timescale/actors/miner/v2/info.go new file mode 100644 index 000000000..8d5373459 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/info.go @@ -0,0 +1,90 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v2/precommits.go b/pkg/transform/timescale/actors/miner/v2/precommits.go new file mode 100644 index 000000000..773af4829 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/precommits.go @@ -0,0 +1,65 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v2/processor.go b/pkg/transform/timescale/actors/miner/v2/processor.go index 9e8abb873..d3684d3ba 100644 --- a/pkg/transform/timescale/actors/miner/v2/processor.go +++ b/pkg/transform/timescale/actors/miner/v2/processor.go @@ -4,36 +4,75 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/filecoin-project/go-state-types/store" + store "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/model" - v2 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" ) +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) if err != nil { return nil, err } - minerState := new(v2.StateChange) + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) if err != nil { return err } + // register extractors to run over the miners state. stateExtractor := &StateExtract{ ExtractMethods: []Extractor{ - // TODO + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, }, } models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) @@ -47,23 +86,3 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut } return out, nil } - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v2.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v2.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v2/sector_deals.go new file mode 100644 index 000000000..e50c97211 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/sector_deals.go @@ -0,0 +1,62 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v2/sector_events.go new file mode 100644 index 000000000..8ca79914e --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/sector_events.go @@ -0,0 +1,143 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v2/sectors.go b/pkg/transform/timescale/actors/miner/v2/sectors.go new file mode 100644 index 000000000..99ad0eb44 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/sectors.go @@ -0,0 +1,59 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v2/state.go b/pkg/transform/timescale/actors/miner/v2/state.go new file mode 100644 index 000000000..f1f98bb18 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v2/state.go @@ -0,0 +1,70 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v3/info.go b/pkg/transform/timescale/actors/miner/v3/info.go new file mode 100644 index 000000000..b42474c1e --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/info.go @@ -0,0 +1,90 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v3/precommits.go b/pkg/transform/timescale/actors/miner/v3/precommits.go new file mode 100644 index 000000000..494c06740 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/precommits.go @@ -0,0 +1,65 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v3/processor.go b/pkg/transform/timescale/actors/miner/v3/processor.go new file mode 100644 index 000000000..4f411088c --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/processor.go @@ -0,0 +1,88 @@ +package v3 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v3/sector_deals.go new file mode 100644 index 000000000..466beaeae --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/sector_deals.go @@ -0,0 +1,62 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v3/sector_events.go new file mode 100644 index 000000000..2a4e056b4 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/sector_events.go @@ -0,0 +1,143 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v3/sectors.go b/pkg/transform/timescale/actors/miner/v3/sectors.go new file mode 100644 index 000000000..447b4ca58 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/sectors.go @@ -0,0 +1,59 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v3/state.go b/pkg/transform/timescale/actors/miner/v3/state.go new file mode 100644 index 000000000..b790b8c68 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v3/state.go @@ -0,0 +1,70 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v4/info.go b/pkg/transform/timescale/actors/miner/v4/info.go new file mode 100644 index 000000000..b000f78b2 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/info.go @@ -0,0 +1,90 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v4/precommits.go b/pkg/transform/timescale/actors/miner/v4/precommits.go new file mode 100644 index 000000000..352b64fda --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/precommits.go @@ -0,0 +1,65 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v4/processor.go b/pkg/transform/timescale/actors/miner/v4/processor.go new file mode 100644 index 000000000..0c487b400 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/processor.go @@ -0,0 +1,88 @@ +package v4 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v4/sector_deals.go new file mode 100644 index 000000000..8b0ef4dd8 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/sector_deals.go @@ -0,0 +1,62 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v4/sector_events.go new file mode 100644 index 000000000..22d52fb34 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/sector_events.go @@ -0,0 +1,143 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v4/sectors.go b/pkg/transform/timescale/actors/miner/v4/sectors.go new file mode 100644 index 000000000..7400a8058 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/sectors.go @@ -0,0 +1,59 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v4/state.go b/pkg/transform/timescale/actors/miner/v4/state.go new file mode 100644 index 000000000..deb1c9004 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v4/state.go @@ -0,0 +1,70 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v5/info.go b/pkg/transform/timescale/actors/miner/v5/info.go new file mode 100644 index 000000000..1261f4815 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/info.go @@ -0,0 +1,90 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v5/precommits.go b/pkg/transform/timescale/actors/miner/v5/precommits.go new file mode 100644 index 000000000..1925086ef --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/precommits.go @@ -0,0 +1,65 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v5/processor.go b/pkg/transform/timescale/actors/miner/v5/processor.go new file mode 100644 index 000000000..716492b6d --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/processor.go @@ -0,0 +1,88 @@ +package v5 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v5/sector_deals.go new file mode 100644 index 000000000..03baec818 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/sector_deals.go @@ -0,0 +1,62 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v5/sector_events.go new file mode 100644 index 000000000..8b854e658 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/sector_events.go @@ -0,0 +1,143 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v5/sectors.go b/pkg/transform/timescale/actors/miner/v5/sectors.go new file mode 100644 index 000000000..970ab2267 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/sectors.go @@ -0,0 +1,59 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v5/state.go b/pkg/transform/timescale/actors/miner/v5/state.go new file mode 100644 index 000000000..156348997 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v5/state.go @@ -0,0 +1,70 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v6/info.go b/pkg/transform/timescale/actors/miner/v6/info.go new file mode 100644 index 000000000..c67e58029 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/info.go @@ -0,0 +1,90 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v6/precommits.go b/pkg/transform/timescale/actors/miner/v6/precommits.go new file mode 100644 index 000000000..8dea5427f --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/precommits.go @@ -0,0 +1,65 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v6/processor.go b/pkg/transform/timescale/actors/miner/v6/processor.go new file mode 100644 index 000000000..fb1c3155e --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/processor.go @@ -0,0 +1,88 @@ +package v6 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v6/sector_deals.go new file mode 100644 index 000000000..8f9363c4b --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/sector_deals.go @@ -0,0 +1,62 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v6/sector_events.go new file mode 100644 index 000000000..7f20b46f0 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/sector_events.go @@ -0,0 +1,143 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v6/sectors.go b/pkg/transform/timescale/actors/miner/v6/sectors.go new file mode 100644 index 000000000..859061a57 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/sectors.go @@ -0,0 +1,59 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) + for i, sector := range sectors { + sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(sector.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v6/state.go b/pkg/transform/timescale/actors/miner/v6/state.go new file mode 100644 index 000000000..fa7d57f20 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v6/state.go @@ -0,0 +1,70 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v7/info.go b/pkg/transform/timescale/actors/miner/v7/info.go new file mode 100644 index 000000000..d5c325bfe --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/info.go @@ -0,0 +1,90 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v7/precommits.go b/pkg/transform/timescale/actors/miner/v7/precommits.go new file mode 100644 index 000000000..52659edd3 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/precommits.go @@ -0,0 +1,65 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v7/processor.go b/pkg/transform/timescale/actors/miner/v7/processor.go new file mode 100644 index 000000000..06c69f294 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/processor.go @@ -0,0 +1,88 @@ +package v7 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v7/sector_deals.go new file mode 100644 index 000000000..8e1639b48 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/sector_deals.go @@ -0,0 +1,62 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v7/sector_events.go new file mode 100644 index 000000000..7e7afea53 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/sector_events.go @@ -0,0 +1,152 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorSnapped, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v7/sectors.go b/pkg/transform/timescale/actors/miner/v7/sectors.go new file mode 100644 index 000000000..afd482b80 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/sectors.go @@ -0,0 +1,64 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v7/state.go b/pkg/transform/timescale/actors/miner/v7/state.go new file mode 100644 index 000000000..c54b8bfc2 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v7/state.go @@ -0,0 +1,70 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v8/info.go b/pkg/transform/timescale/actors/miner/v8/info.go new file mode 100644 index 000000000..4f220aadc --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/info.go @@ -0,0 +1,90 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type Info struct{} + +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. + if change.InfoChange == nil { + return nil, nil + } + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { + return nil, err + } + return out, nil +} + +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo +} + +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} + +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} + +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} + +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false + } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} + +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId +} diff --git a/pkg/transform/timescale/actors/miner/v8/precommits.go b/pkg/transform/timescale/actors/miner/v8/precommits.go new file mode 100644 index 000000000..5e2c1bc50 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/precommits.go @@ -0,0 +1,65 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type PreCommit struct{} + +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range change.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + precommits = append(precommits, precommit) + } + return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) +} + +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { + preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) + for i, preCommit := range precommits { + deals := make([]uint64, len(preCommit.Info.DealIDs)) + for didx, deal := range preCommit.Info.DealIDs { + deals[didx] = uint64(deal) + } + preCommitModel[i] = &minermodel.MinerPreCommitInfo{ + Height: int64(current.Height()), + MinerID: addr.String(), + SectorID: uint64(preCommit.Info.SectorNumber), + StateRoot: current.ParentState().String(), + SealedCID: preCommit.Info.SealedCID.String(), + SealRandEpoch: int64(preCommit.Info.SealRandEpoch), + ExpirationEpoch: int64(preCommit.Info.Expiration), + PreCommitDeposit: preCommit.PreCommitDeposit.String(), + PreCommitEpoch: int64(preCommit.PreCommitEpoch), + DealWeight: preCommit.DealWeight.String(), + VerifiedDealWeight: preCommit.VerifiedDealWeight.String(), + IsReplaceCapacity: preCommit.Info.ReplaceCapacity, + ReplaceSectorDeadline: preCommit.Info.ReplaceSectorDeadline, + ReplaceSectorPartition: preCommit.Info.ReplaceSectorPartition, + ReplaceSectorNumber: uint64(preCommit.Info.ReplaceSectorNumber), + } + } + + return preCommitModel, nil + +} diff --git a/pkg/transform/timescale/actors/miner/v8/processor.go b/pkg/transform/timescale/actors/miner/v8/processor.go new file mode 100644 index 000000000..35511ffb8 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/processor.go @@ -0,0 +1,88 @@ +package v8 + +import ( + "context" + + "github.com/filecoin-project/go-address" + store "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" +) + +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + +func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + if err != nil { + return err + } + // register extractors to run over the miners state. + stateExtractor := &StateExtract{ + ExtractMethods: []Extractor{ + Info{}, + PreCommit{}, + SectorDeal{}, + SectorEvent{}, + Sector{}, + }, + } + models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, models...) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v8/sector_deals.go new file mode 100644 index 000000000..d4e361a5c --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/sector_deals.go @@ -0,0 +1,62 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type SectorDeal struct{} + +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + sectors := change.SectorChanges + out := minermodel.MinerSectorDealList{} + height := int64(current.Height()) + minerAddr := addr.String() + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v8/sector_events.go new file mode 100644 index 000000000..833b04fba --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/sector_events.go @@ -0,0 +1,152 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type SectorEvent struct{} + +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var ( + precommits = change.PreCommitChanges + sectors = change.SectorChanges + sectorstatus = change.SectorStatusChanges + height = int64(current.Height()) + minerAddr = addr.String() + stateRoot = current.ParentState().String() + out = minermodel.MinerSectorEventList{} + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue + } + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + return nil, err + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sectorID, + StateRoot: stateRoot, + Event: minermodel.PreCommitAdded, + }) + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: event, + }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + return nil, err + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + if previousSector.Expiration != currentSector.Expiration { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } + if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorSnapped, + }) + } + } + } + if sectorstatus == nil { + return out, nil + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + return nil, err + } + + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + out = append(out, &minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v8/sectors.go b/pkg/transform/timescale/actors/miner/v8/sectors.go new file mode 100644 index 000000000..d419c6afe --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/sectors.go @@ -0,0 +1,64 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +type Sector struct { +} + +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo + changes := change.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + return nil, err + } + sectors = append(sectors, s) + } + return MinerSectorChangesAsModel(ctx, current, addr, sectors) +} + +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { + sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) + for i, sector := range sectors { + sectorKeyCID := "" + if sector.SectorKeyCID != nil { + sectorKeyCID = sector.SectorKeyCID.String() + } + sectorModel[i] = &minermodel.MinerSectorInfoV7{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + SectorID: uint64(sector.SectorNumber), + SealedCID: sector.SealedCID.String(), + ActivationEpoch: int64(sector.Activation), + ExpirationEpoch: int64(sector.Expiration), + DealWeight: sector.DealWeight.String(), + VerifiedDealWeight: sector.VerifiedDealWeight.String(), + InitialPledge: sector.InitialPledge.String(), + ExpectedDayReward: sector.ExpectedDayReward.String(), + ExpectedStoragePledge: sector.ExpectedStoragePledge.String(), + SectorKeyCID: sectorKeyCID, + } + } + + return sectorModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/v8/state.go b/pkg/transform/timescale/actors/miner/v8/state.go new file mode 100644 index 000000000..21cfbfe6c --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v8/state.go @@ -0,0 +1,70 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/miner/v9/debt.go b/pkg/transform/timescale/actors/miner/v9/debt.go deleted file mode 100644 index 14971abce..000000000 --- a/pkg/transform/timescale/actors/miner/v9/debt.go +++ /dev/null @@ -1,26 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" -) - -type Debt struct{} - -func (Debt) Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { - if change.DebtChange == nil { - return nil, nil - } - return &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - MinerID: addr.String(), - FeeDebt: change.DebtChange.FeeDebt.String(), - }, nil -} diff --git a/pkg/transform/timescale/actors/miner/v9/funds.go b/pkg/transform/timescale/actors/miner/v9/funds.go deleted file mode 100644 index c9d75cd9b..000000000 --- a/pkg/transform/timescale/actors/miner/v9/funds.go +++ /dev/null @@ -1,29 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" -) - -type Fund struct{} - -func (Fund) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { - if change.FundsChange == nil { - return nil, nil - } - funds := change.FundsChange - return &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: funds.VestingFunds.String(), - InitialPledge: funds.InitialPledgeRequirement.String(), - PreCommitDeposits: funds.PreCommitDeposit.String(), - }, nil -} diff --git a/pkg/transform/timescale/actors/miner/v9/info.go b/pkg/transform/timescale/actors/miner/v9/info.go index 4a36d0336..bcb1a7ac4 100644 --- a/pkg/transform/timescale/actors/miner/v9/info.go +++ b/pkg/transform/timescale/actors/miner/v9/info.go @@ -1,92 +1,90 @@ package v9 import ( + "bytes" "context" "github.com/filecoin-project/go-address" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" + "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - "github.com/libp2p/go-libp2p/core/peer" - maddr "github.com/multiformats/go-multiaddr" "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { +func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil } - info := change.InfoChange - var out model.Persistable - var err error - if err := core.StateReadDeferred(ctx, info.Info, func(in *miner9.MinerInfo) error { - out, err = MinerInfoAsModel(ctx, current, executed, addr, *in) - if err != nil { - return err - } - return nil - }); err != nil { + // if the info was removed there is nothing to record + if change.InfoChange.Change == core.ChangeTypeRemove { + return nil, nil + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { + return nil, err + } + // wrap the versioned miner info type in an interface for reusable extraction + out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) + if err != nil { return nil, err } return out, nil } -func MinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner9.MinerInfo) (model.Persistable, error) { - return GenericMinerInfoAsModel(ctx, current, executed, addr, info) +// InfoWrapper satisfies the interface required by ExtractMinerInfo. +type InfoWrapper struct { + info *miner.MinerInfo } -func GenericMinerInfoAsModel(ctx context.Context, current, executed *types.TipSet, addr address.Address, info miner9.MinerInfo) (model.Persistable, error) { - var newWorker string - var newWorkerEpoch int64 - if pendingWorkerKey := info.PendingWorkerKey; pendingWorkerKey != nil { - if pendingWorkerKey.NewWorker != address.Undef { - newWorker = pendingWorkerKey.NewWorker.String() - } - newWorkerEpoch = int64(pendingWorkerKey.EffectiveAt) - } +type WorkerKeyChangeWrapper struct { + keys *miner.WorkerKeyChange +} - var newCtrlAddresses []string - for _, addr := range info.ControlAddresses { - newCtrlAddresses = append(newCtrlAddresses, addr.String()) - } +func (v *WorkerKeyChangeWrapper) NewWorker() address.Address { + return v.keys.NewWorker +} - // best effort to decode, we have no control over what miners put in this field, its just bytes. - var newMultiAddrs []string - for _, addr := range info.Multiaddrs { - newMaddr, err := maddr.NewMultiaddrBytes(addr) - if err == nil { - newMultiAddrs = append(newMultiAddrs, newMaddr.String()) - } else { - //log.Debugw("failed to decode miner multiaddr", "miner", a.Address, "multiaddress", addr, "error", err) - } - } - mi := &minermodel.MinerInfo{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - OwnerID: info.Owner.String(), - WorkerID: info.Worker.String(), - NewWorker: newWorker, - WorkerChangeEpoch: newWorkerEpoch, - ConsensusFaultedElapsed: -1, - ControlAddresses: newCtrlAddresses, - MultiAddresses: newMultiAddrs, - SectorSize: uint64(info.SectorSize), - } +func (v *WorkerKeyChangeWrapper) EffectiveAt() abi.ChainEpoch { + return v.keys.EffectiveAt +} - if info.PeerId != nil { - newPeerID, err := peer.IDFromBytes(info.PeerId) - if err != nil { - //log.Warnw("failed to decode miner peerID", "miner", a.Address, "head", a.Actor.Head.String(), "error", err) - } else { - mi.PeerID = newPeerID.String() - } +func (v *InfoWrapper) PendingWorkerKey() (util.WorkerKeyChanges, bool) { + if v.info.PendingWorkerKey == nil { + return nil, false } + return &WorkerKeyChangeWrapper{keys: v.info.PendingWorkerKey}, true +} + +func (v *InfoWrapper) ControlAddresses() []address.Address { + return v.info.ControlAddresses +} + +func (v *InfoWrapper) Multiaddrs() []abi.Multiaddrs { + return v.info.Multiaddrs +} + +func (v *InfoWrapper) Owner() address.Address { + return v.info.Owner +} + +func (v *InfoWrapper) Worker() address.Address { + return v.info.Worker +} + +func (v *InfoWrapper) SectorSize() abi.SectorSize { + return v.info.SectorSize +} - return mi, nil +func (v *InfoWrapper) PeerId() abi.PeerID { + return v.info.PeerId } diff --git a/pkg/transform/timescale/actors/miner/v9/precommits.go b/pkg/transform/timescale/actors/miner/v9/precommits.go index 53eb47f4d..2bb4e4237 100644 --- a/pkg/transform/timescale/actors/miner/v9/precommits.go +++ b/pkg/transform/timescale/actors/miner/v9/precommits.go @@ -1,38 +1,40 @@ package v9 import ( + "bytes" "context" "github.com/filecoin-project/go-address" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { - var precommits []*miner9.SectorPreCommitOnChainInfo +func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added if change.Change != core.ChangeTypeAdd { continue } - if err := core.StateReadDeferred(ctx, change.Current, func(precommit *miner9.SectorPreCommitOnChainInfo) error { - precommits = append(precommits, precommit) - return nil - }); err != nil { + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { return nil, err } + precommits = append(precommits, precommit) } return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner9.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { preCommitModel := make(minermodel.MinerPreCommitInfoV9List, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) diff --git a/pkg/transform/timescale/actors/miner/v9/processor.go b/pkg/transform/timescale/actors/miner/v9/processor.go index 2a7f2d02d..a6683990a 100644 --- a/pkg/transform/timescale/actors/miner/v9/processor.go +++ b/pkg/transform/timescale/actors/miner/v9/processor.go @@ -12,30 +12,62 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) +type Extractor interface { + Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + ExtractMethods []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.ExtractMethods)) + for _, e := range se.ExtractMethods { + m, err := e.Extract(ctx, current, executed, addr, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + return ExtractMinerStateChanges(ctx, current, executed, addr, change) +} + func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} adtStore := store.WrapBlockStore(ctx, bs) + + // a map of all miners whose state has changes minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) if err != nil { return nil, err } - minerState := new(v9.StateChange) + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - minerStateDiff, err := minerState.StateChangeAsStateDiffResult(ctx, bs) + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) if err != nil { return err } + // register extractors to run over the miners state. stateExtractor := &StateExtract{ ExtractMethods: []Extractor{ - Debt{}, - Fund{}, Info{}, PreCommit{}, SectorDeal{}, @@ -54,25 +86,3 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut } return out, nil } - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *v9.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v9/sector_deals.go index 5dbf9ff4d..235976d1a 100644 --- a/pkg/transform/timescale/actors/miner/v9/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_deals.go @@ -5,19 +5,21 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) @@ -25,29 +27,28 @@ func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet for _, sector := range sectors { switch sector.Change { case core.ChangeTypeAdd: - if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner9.SectorOnChainInfo) error { - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + for _, deal := range s.DealIDs { + out = append(out, &minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } case core.ChangeTypeModify: - previousSector := new(miner9.SectorOnChainInfo) + previousSector := new(miner.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - currentSector := new(miner9.SectorOnChainInfo) + currentSector := new(miner.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } - for _, deal := range compareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { out = append(out, &minermodel.MinerSectorDeal{ Height: height, MinerID: minerAddr, @@ -59,31 +60,3 @@ func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet } return out, nil } - -func compareDealIDs(cur, pre []abi.DealID) []abi.DealID { - var diff []abi.DealID - - // Loop two times, first to find cur dealIDs not in pre, - // second loop to find pre dealIDs not in cur - for i := 0; i < 2; i++ { - for _, s1 := range cur { - found := false - for _, s2 := range pre { - if s1 == s2 { - found = true - break - } - } - // DealID not found. We add it to return slice - if !found { - diff = append(diff, s1) - } - } - // Swap the slices, only if it was the first loop - if i == 0 { - cur, pre = pre, cur - } - } - - return diff -} diff --git a/pkg/transform/timescale/actors/miner/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v9/sector_events.go index c624b58ae..d74ce0d82 100644 --- a/pkg/transform/timescale/actors/miner/v9/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_events.go @@ -6,18 +6,20 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges @@ -48,14 +50,13 @@ func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSe switch sector.Change { case core.ChangeTypeAdd: event := minermodel.SectorAdded - if err := core.StateReadDeferred(ctx, sector.Current, func(s *miner9.SectorOnChainInfo) error { - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded - } - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } out = append(out, &minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, @@ -64,11 +65,11 @@ func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSe Event: event, }) case core.ChangeTypeModify: - previousSector := new(miner9.SectorOnChainInfo) + previousSector := new(miner.SectorOnChainInfo) if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { return nil, err } - currentSector := new(miner9.SectorOnChainInfo) + currentSector := new(miner.SectorOnChainInfo) if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } diff --git a/pkg/transform/timescale/actors/miner/v9/sectors.go b/pkg/transform/timescale/actors/miner/v9/sectors.go index f98fd5a83..4e08e93e7 100644 --- a/pkg/transform/timescale/actors/miner/v9/sectors.go +++ b/pkg/transform/timescale/actors/miner/v9/sectors.go @@ -1,41 +1,42 @@ package v9 import ( + "bytes" "context" "github.com/filecoin-project/go-address" - miner9 "github.com/filecoin-project/go-state-types/builtin/v9/miner" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *v9.StateDiffResult) (model.Persistable, error) { - var sectors []*miner9.SectorOnChainInfo +func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { + var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { // only care about modified and added sectors if sector.Change == core.ChangeTypeRemove { continue } - // change.Current is the newly added sector, or its state after modification. - if err := core.StateReadDeferred(ctx, sector.Current, func(sector *miner9.SectorOnChainInfo) error { - sectors = append(sectors, sector) - return nil - }); err != nil { + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { return nil, err } + sectors = append(sectors, s) } return MinerSectorChangesAsModel(ctx, current, addr, sectors) } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner9.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) for i, sector := range sectors { sectorKeyCID := "" diff --git a/pkg/transform/timescale/actors/miner/v9/state.go b/pkg/transform/timescale/actors/miner/v9/state.go new file mode 100644 index 000000000..96fc33311 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v9/state.go @@ -0,0 +1,70 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + minermodel "github.com/filecoin-project/lily/model/actors/miner" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + + miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" +) + +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + var out model.PersistableList + currentState := new(miner.State) + if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + if change.Change == core.ChangeTypeAdd { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + if change.Change == core.ChangeTypeModify { + previousState := new(miner.State) + if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { + return nil, err + } + if !currentState.FeeDebt.Equals(previousState.FeeDebt) { + out = append(out, &minermodel.MinerFeeDebt{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + FeeDebt: currentState.FeeDebt.String(), + }) + } + if !currentState.LockedFunds.Equals(previousState.LockedFunds) || + !currentState.InitialPledge.Equals(previousState.InitialPledge) || + !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { + out = append(out, &minermodel.MinerLockedFund{ + Height: int64(current.Height()), + MinerID: addr.String(), + StateRoot: current.ParentState().String(), + LockedFunds: currentState.LockedFunds.String(), + InitialPledge: currentState.InitialPledge.String(), + PreCommitDeposits: currentState.PreCommitDeposits.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/actors/processor.go index 52db021bf..3188a38f3 100644 --- a/pkg/transform/timescale/actors/processor.go +++ b/pkg/transform/timescale/actors/processor.go @@ -5,8 +5,10 @@ import ( "fmt" "io" + "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" @@ -16,10 +18,11 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" "github.com/filecoin-project/lily/pkg/transform/cbor" - "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" - v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v9" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" ) type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version @@ -47,25 +50,72 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe if err != nil { return err } - mapHandler := MakeMinerProcessor(av) - minerModels, err := mapHandler(ctx, bs, current, executed, actorIPLDContainer.MinerActors) + + var models model.PersistableList + actorModels, err := ProcessActorStates(ctx, bs, current, executed, av, actorIPLDContainer.ActorStates) + if err != nil { + return err + } + models = append(models, actorModels) + + minerModels, err := ProcessMiners(ctx, bs, current, executed, av, actorIPLDContainer.MinerActors) if err != nil { return err } + models = append(models, minerModels) - return strg.PersistBatch(ctx, minerModels...) + return strg.PersistBatch(ctx, models...) } -type MinerHandler = func(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) +func ProcessMiners(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { + minerHandler, err := miner.MakeMinerProcessor(av) + if err != nil { + return nil, err + } + return minerHandler(ctx, bs, current, executed, root) +} + +func ProcessActorStates(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, av actorstypes.Version, actorMapRoot cid.Cid) (model.Persistable, error) { + var out = model.PersistableList{} + adtStore := store.WrapBlockStore(ctx, bs) + actorMap, err := adt.AsMap(adtStore, actorMapRoot, 5) + if err != nil { + return nil, err + } + actorState := new(actordiff.ActorChange) + if err := actorMap.ForEach(actorState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + + m, err := raw.RawActorHandler(ctx, current, executed, addr, actorState) + if err != nil { + return err + } + if m != nil { + out = append(out, m) + } + + if core.RewardCodes.Has(actorState.Actor.Code) { + m, err := reward.HandleReward(ctx, current, executed, addr, actorState, av) + if err != nil { + return err + } + out = append(out, m) + } + + if core.MinerCodes.Has(actorState.Actor.Code) { + m, err := miner.HandleMiner(ctx, current, executed, addr, actorState, av) + if err != nil { + return err + } + out = append(out, m) + } + return nil -func MakeMinerProcessor(av actorstypes.Version) MinerHandler { - switch av { - case actorstypes.Version0: - return v0.MinerHandler - case actorstypes.Version2: - return v2.MinerHandler - case actorstypes.Version9: - return v9.MinerHandler + }); err != nil { + return nil, err } - panic("developer error") + return out, nil } diff --git a/pkg/transform/timescale/actors/raw/state.go b/pkg/transform/timescale/actors/raw/state.go new file mode 100644 index 000000000..fa6c49273 --- /dev/null +++ b/pkg/transform/timescale/actors/raw/state.go @@ -0,0 +1,33 @@ +package raw + +import ( + "context" + "encoding/json" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + commonmodel "github.com/filecoin-project/lily/model/actors/common" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RawActorHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + // noop when actor is removed + if change.Change == core.ChangeTypeRemove { + return nil, nil + } + + state, err := json.Marshal(change.Current) + if err != nil { + return nil, err + } + + return &commonmodel.ActorState{ + Height: int64(current.Height()), + Head: change.Actor.Head.String(), + Code: change.Actor.Code.String(), + State: string(state), + }, nil +} diff --git a/pkg/transform/timescale/actors/reward/router.go b/pkg/transform/timescale/actors/reward/router.go new file mode 100644 index 000000000..ef6abf967 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/router.go @@ -0,0 +1,49 @@ +package reward + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v0" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v2" + v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v3" + v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v4" + v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v5" + v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v6" + v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v7" + v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v8" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v9" +) + +func HandleReward(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange, version actortypes.Version) (model.Persistable, error) { + switch version { + case actortypes.Version0: + return v0.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version2: + return v2.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version3: + return v3.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version4: + return v4.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version5: + return v5.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version6: + return v6.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version7: + return v7.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version8: + return v8.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version9: + return v9.RewardHandler(ctx, current, executed, addr, change) + case actortypes.Version10: + panic("not yet implemented") + default: + return nil, fmt.Errorf("unsupported reward actor version: %d", version) + } +} diff --git a/pkg/transform/timescale/actors/reward/v0/state.go b/pkg/transform/timescale/actors/reward/v0/state.go new file mode 100644 index 000000000..b3d78b0c9 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v0/state.go @@ -0,0 +1,39 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward0.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalMined.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v2/state.go b/pkg/transform/timescale/actors/reward/v2/state.go new file mode 100644 index 000000000..93f251fb8 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v2/state.go @@ -0,0 +1,39 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward2.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v3/state.go b/pkg/transform/timescale/actors/reward/v3/state.go new file mode 100644 index 000000000..6f383b8e4 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v3/state.go @@ -0,0 +1,39 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward3.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v4/state.go b/pkg/transform/timescale/actors/reward/v4/state.go new file mode 100644 index 000000000..f6f36a988 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v4/state.go @@ -0,0 +1,39 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward4.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v5/state.go b/pkg/transform/timescale/actors/reward/v5/state.go new file mode 100644 index 000000000..d4610446f --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v5/state.go @@ -0,0 +1,39 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward5.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v6/state.go b/pkg/transform/timescale/actors/reward/v6/state.go new file mode 100644 index 000000000..ca9c0ed1f --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v6/state.go @@ -0,0 +1,39 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward6.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v7/state.go b/pkg/transform/timescale/actors/reward/v7/state.go new file mode 100644 index 000000000..e6707cab1 --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v7/state.go @@ -0,0 +1,39 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + reward7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/reward" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward7.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v8/state.go b/pkg/transform/timescale/actors/reward/v8/state.go new file mode 100644 index 000000000..ae8963f2f --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v8/state.go @@ -0,0 +1,39 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + reward8 "github.com/filecoin-project/go-state-types/builtin/v8/reward" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward8.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} diff --git a/pkg/transform/timescale/actors/reward/v9/state.go b/pkg/transform/timescale/actors/reward/v9/state.go new file mode 100644 index 000000000..28aa84c3c --- /dev/null +++ b/pkg/transform/timescale/actors/reward/v9/state.go @@ -0,0 +1,39 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + reward9 "github.com/filecoin-project/go-state-types/builtin/v9/reward" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + rewardmodel "github.com/filecoin-project/lily/model/actors/reward" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" +) + +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("reward is a singleton actor and cannot be removed") + } + state := new(reward9.State) + if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &rewardmodel.ChainReward{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + CumSumBaseline: state.CumsumBaseline.String(), + CumSumRealized: state.CumsumRealized.String(), + EffectiveBaselinePower: state.EffectiveBaselinePower.String(), + NewBaselinePower: state.ThisEpochBaselinePower.String(), + NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), + NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), + TotalMinedReward: state.TotalStoragePowerReward.String(), + NewReward: state.ThisEpochReward.String(), + EffectiveNetworkTime: int64(state.EffectiveNetworkTime), + }, nil + +} From fd5ae8e12204e11cfada1aeb5f910cd6e3c1eb47 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 12 Jan 2023 19:20:05 -0800 Subject: [PATCH 22/48] fix cbor gen for ipld container and actor statee to json --- pkg/gen/main.go | 10 + pkg/transform/cbor/cbor_gen.go | 209 ++++++++++++++++++-- pkg/transform/cbor/router.go | 36 ++-- pkg/transform/timescale/actors/processor.go | 10 +- pkg/transform/timescale/actors/raw/state.go | 9 +- 5 files changed, 240 insertions(+), 34 deletions(-) diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 9c721940c..38995b7e8 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -10,6 +10,7 @@ import ( powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" + "github.com/filecoin-project/lily/pkg/transform/cbor" ) const actorDiffPath = "pkg/extract/actors/actordiff/cbor_gen.go" @@ -33,6 +34,9 @@ const marketDiffPkg = "v0" const powerDiffPath = "pkg/extract/actors/powerdiff/v0/cbor_gen.go" const powerDiffPkg = "v0" +const IPLDActorContainerPath = "pkg/transform/cbor/cbor_gen.go" +const IPLDActorContainerPkg = "cbor" + func main() { if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, actordiff.ActorChange{}, @@ -90,4 +94,10 @@ func main() { ); err != nil { panic(err) } + + if err := cbg.WriteMapEncodersToFile(IPLDActorContainerPath, IPLDActorContainerPkg, + cbor.ActorIPLDContainer{}, + ); err != nil { + panic(err) + } } diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go index 4ec281db8..d3014e3a7 100644 --- a/pkg/transform/cbor/cbor_gen.go +++ b/pkg/transform/cbor/cbor_gen.go @@ -27,19 +27,19 @@ func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{163}); err != nil { + if _, err := cw.Write([]byte{168}); err != nil { return err } // t.CurrentTipSet (types.TipSet) (struct) - if len("CurrentTipSet") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"CurrentTipSet\" was too long") + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("CurrentTipSet"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { return err } - if _, err := io.WriteString(w, string("CurrentTipSet")); err != nil { + if _, err := io.WriteString(w, string("current")); err != nil { return err } @@ -48,14 +48,14 @@ func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { } // t.ExecutedTipSet (types.TipSet) (struct) - if len("ExecutedTipSet") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"ExecutedTipSet\" was too long") + if len("executed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"executed\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("ExecutedTipSet"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("executed"))); err != nil { return err } - if _, err := io.WriteString(w, string("ExecutedTipSet")); err != nil { + if _, err := io.WriteString(w, string("executed")); err != nil { return err } @@ -64,14 +64,14 @@ func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { } // t.MinerActors (cid.Cid) (struct) - if len("MinerActors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"MinerActors\" was too long") + if len("miners") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miners\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MinerActors"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miners"))); err != nil { return err } - if _, err := io.WriteString(w, string("MinerActors")); err != nil { + if _, err := io.WriteString(w, string("miners")); err != nil { return err } @@ -79,6 +79,98 @@ func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { return xerrors.Errorf("failed to write cid field t.MinerActors: %w", err) } + // t.VerifregActor (cid.Cid) (struct) + if len("verifreg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifreg\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifreg"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifreg")); err != nil { + return err + } + + if t.VerifregActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.VerifregActor); err != nil { + return xerrors.Errorf("failed to write cid field t.VerifregActor: %w", err) + } + } + + // t.ActorStates (cid.Cid) (struct) + if len("actors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ActorStates); err != nil { + return xerrors.Errorf("failed to write cid field t.ActorStates: %w", err) + } + + // t.InitActor (cid.Cid) (struct) + if len("init") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"init\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("init"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("init")); err != nil { + return err + } + + if t.InitActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.InitActor); err != nil { + return xerrors.Errorf("failed to write cid field t.InitActor: %w", err) + } + } + + // t.MarketActor (cid.Cid) (struct) + if len("market") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"market\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("market"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("market")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.MarketActor); err != nil { + return xerrors.Errorf("failed to write cid field t.MarketActor: %w", err) + } + + // t.PowerActor (cid.Cid) (struct) + if len("power") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"power\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("power"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("power")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.PowerActor); err != nil { + return xerrors.Errorf("failed to write cid field t.PowerActor: %w", err) + } + return nil } @@ -121,7 +213,7 @@ func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { switch name { // t.CurrentTipSet (types.TipSet) (struct) - case "CurrentTipSet": + case "current": { @@ -141,7 +233,7 @@ func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { } // t.ExecutedTipSet (types.TipSet) (struct) - case "ExecutedTipSet": + case "executed": { @@ -161,7 +253,7 @@ func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { } // t.MinerActors (cid.Cid) (struct) - case "MinerActors": + case "miners": { @@ -173,6 +265,91 @@ func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { t.MinerActors = c } + // t.VerifregActor (cid.Cid) (struct) + case "verifreg": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VerifregActor: %w", err) + } + + t.VerifregActor = &c + } + + } + // t.ActorStates (cid.Cid) (struct) + case "actors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ActorStates: %w", err) + } + + t.ActorStates = c + + } + // t.InitActor (cid.Cid) (struct) + case "init": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.InitActor: %w", err) + } + + t.InitActor = &c + } + + } + // t.MarketActor (cid.Cid) (struct) + case "market": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MarketActor: %w", err) + } + + t.MarketActor = c + + } + // t.PowerActor (cid.Cid) (struct) + case "power": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PowerActor: %w", err) + } + + t.PowerActor = c + + } default: // Field doesn't exist on this type, so ignore it diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 7953a625d..caeefe7ed 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -11,6 +11,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car/util" "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap" "go.uber.org/zap/zapcore" v1car "github.com/ipld/go-car" @@ -28,29 +29,34 @@ var log = logging.Logger("lily/transform/cbor") type ActorIPLDContainer struct { // TODO this needs to be versioned - CurrentTipSet *types.TipSet - ExecutedTipSet *types.TipSet - MinerActors cid.Cid // HAMT[Address]MinerStateChange - VerifregActor *cid.Cid // VerifregStateChange or empty - ActorStates cid.Cid // HAMT[Address]ActorStateChange - InitActor cid.Cid // HAMT[Address]AddressChanges. - MarketActor cid.Cid // MarketStateChange or empty - PowerActor cid.Cid // PowerStateChange or empty + CurrentTipSet *types.TipSet `cborgen:"current"` + ExecutedTipSet *types.TipSet `cborgen:"executed"` + MinerActors cid.Cid `cborgen:"miners"` + VerifregActor *cid.Cid `cborgen:"verifreg"` + ActorStates cid.Cid `cborgen:"actors"` + InitActor *cid.Cid `cborgen:"init"` + MarketActor cid.Cid `cborgen:"market"` + PowerActor cid.Cid `cborgen:"power"` } -func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) []attribute.KeyValue { +func (a *ActorIPLDContainer) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ - attribute.String("current", a.CurrentTipSet.String()), - attribute.String("executed", a.ExecutedTipSet.String()), + attribute.String("current", a.CurrentTipSet.Key().String()), + attribute.String("executed", a.ExecutedTipSet.Key().String()), attribute.String("miners", a.MinerActors.String()), - attribute.String("verifreg", a.VerifregActor.String()), attribute.String("actors", a.ActorStates.String()), - attribute.String("init", a.InitActor.String()), attribute.String("market", a.MarketActor.String()), attribute.String("power", a.PowerActor.String()), } } +func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { bs := blockstore.NewMemorySync() store := adt2.WrapBlockStore(ctx, bs) @@ -63,7 +69,7 @@ func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io if err != nil { return err } - log.Infow("Wrote Delta", "root", actorStatesRoot.String(), actorStates) + log.Infow("Wrote Delta", "root", actorStatesRoot.String(), zap.Inline(actorStates)) if err := v1car.WriteHeader(&v1car.CarHeader{ Roots: []cid.Cid{actorStatesRoot}, Version: 1, @@ -120,7 +126,7 @@ func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *proce if err != nil { return nil, err } - out.InitActor = initRoot + out.InitActor = &initRoot } if changes.MarketActor != nil { diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/actors/processor.go index 3188a38f3..2715b4c73 100644 --- a/pkg/transform/timescale/actors/processor.go +++ b/pkg/transform/timescale/actors/processor.go @@ -14,7 +14,9 @@ import ( "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" v1car "github.com/ipld/go-car" + "go.uber.org/zap" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" @@ -25,6 +27,8 @@ import ( "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" ) +var log = logging.Logger("/lily/transform/timescale") + type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVersionGetter) error { @@ -39,11 +43,13 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe adtStore := store.WrapBlockStore(ctx, bs) - var actorIPLDContainer cbor.ActorIPLDContainer - if err := adtStore.Get(ctx, header.Roots[0], &actorIPLDContainer); err != nil { + actorIPLDContainer := new(cbor.ActorIPLDContainer) + if err := adtStore.Get(ctx, header.Roots[0], actorIPLDContainer); err != nil { return err } + log.Infow("Open Delta", "root", header.Roots[0], zap.Inline(actorIPLDContainer)) + current := actorIPLDContainer.CurrentTipSet executed := actorIPLDContainer.ExecutedTipSet av, err := core.ActorVersionForTipSet(ctx, current, nvg) diff --git a/pkg/transform/timescale/actors/raw/state.go b/pkg/transform/timescale/actors/raw/state.go index fa6c49273..3c2073f5c 100644 --- a/pkg/transform/timescale/actors/raw/state.go +++ b/pkg/transform/timescale/actors/raw/state.go @@ -6,7 +6,9 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/model" commonmodel "github.com/filecoin-project/lily/model/actors/common" "github.com/filecoin-project/lily/pkg/core" @@ -19,7 +21,12 @@ func RawActorHandler(ctx context.Context, current, executed *types.TipSet, addr return nil, nil } - state, err := json.Marshal(change.Current) + stateDump, err := vm.DumpActorState(util.ActorRegistry, change.Actor, change.Current) + if err != nil { + return nil, err + } + + state, err := json.Marshal(stateDump) if err != nil { return nil, err } From 9f950028ab39b9b8402b2b20c3298431d9db2b3e Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 13 Jan 2023 19:12:05 -0800 Subject: [PATCH 23/48] checking --- chain/datasource/datasource.go | 4 + lens/interface.go | 12 + lens/lily/impl.go | 55 ++++- .../process.go => processor/actors.go} | 4 +- pkg/extract/processor/messages.go | 216 ++++++++++++++++++ pkg/extract/processor/vm.go | 157 +++++++++++++ pkg/gen/main.go | 38 ++- pkg/transform/cbor/{ => actors}/cbor_gen.go | 2 +- .../cbor/{ => actors}/init_/state.go | 0 .../cbor/{ => actors}/market/state.go | 0 .../cbor/{ => actors}/miner/state.go | 0 .../cbor/{ => actors}/power/state.go | 0 .../cbor/{actor => actors/raw}/state.go | 2 +- pkg/transform/cbor/actors/serialize.go | 104 +++++++++ .../cbor/{ => actors}/verifreg/state.go | 0 pkg/transform/cbor/messages/serialize.go | 177 ++++++++++++++ pkg/transform/cbor/router.go | 122 ++-------- pkg/transform/timescale/actors/processor.go | 4 +- tasks/api.go | 1 + 19 files changed, 782 insertions(+), 116 deletions(-) rename pkg/extract/{procesor/process.go => processor/actors.go} (97%) create mode 100644 pkg/extract/processor/messages.go create mode 100644 pkg/extract/processor/vm.go rename pkg/transform/cbor/{ => actors}/cbor_gen.go (99%) rename pkg/transform/cbor/{ => actors}/init_/state.go (100%) rename pkg/transform/cbor/{ => actors}/market/state.go (100%) rename pkg/transform/cbor/{ => actors}/miner/state.go (100%) rename pkg/transform/cbor/{ => actors}/power/state.go (100%) rename pkg/transform/cbor/{actor => actors/raw}/state.go (98%) create mode 100644 pkg/transform/cbor/actors/serialize.go rename pkg/transform/cbor/{ => actors}/verifreg/state.go (100%) create mode 100644 pkg/transform/cbor/messages/serialize.go diff --git a/chain/datasource/datasource.go b/chain/datasource/datasource.go index 45960746c..fddc18b33 100644 --- a/chain/datasource/datasource.go +++ b/chain/datasource/datasource.go @@ -238,6 +238,10 @@ func (t *DataSource) CirculatingSupply(ctx context.Context, ts *types.TipSet) (a return t.node.CirculatingSupply(ctx, ts.Key()) } +func (t *DataSource) MessageExecutionsV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) { + return t.node.GetMessageExecutionsForTipSetV2(ctx, ts, pts) +} + func (t *DataSource) MessageExecutions(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecution, error) { metrics.RecordInc(ctx, metrics.DataSourceMessageExecutionRead) ctx, span := otel.Tracer("").Start(ctx, "DataSource.MessageExecutions") diff --git a/lens/interface.go b/lens/interface.go index f6f267bb9..e9fd91d17 100644 --- a/lens/interface.go +++ b/lens/interface.go @@ -23,6 +23,7 @@ type API interface { VMAPI GetMessageExecutionsForTipSet(ctx context.Context, ts, pts *types.TipSet) ([]*MessageExecution, error) + GetMessageExecutionsForTipSetV2(ctx context.Context, ts, pts *types.TipSet) ([]*MessageExecutionV2, error) } type StoreAPI interface { // TODO this should be the lotus store not the specs-actors store. @@ -85,6 +86,17 @@ type MessageExecution struct { Implicit bool } +type MessageExecutionV2 struct { + Cid cid.Cid + StateRoot cid.Cid + Height abi.ChainEpoch + + Message *types.Message + Ret *vm.ApplyRet + + Implicit bool +} + type BlockMessages struct { Block *types.BlockHeader // block messages appeared in BlsMessages []*types.Message // BLS messages in block `Block` diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 81ae6efad..b631d378d 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -42,7 +42,7 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" - "github.com/filecoin-project/lily/pkg/extract/procesor" + "github.com/filecoin-project/lily/pkg/extract/processor" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/timescale/actors" "github.com/filecoin-project/lily/schedule" @@ -162,11 +162,18 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } - changes, err := procesor.ProcessActorStateChanges(ctx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) + // TODO parallelize the below + messageChanges, err := processor.Messages(ctx, taskAPI, currentTs, executedTs) if err != nil { return false, err } - if err := cbor.ProcessState(ctx, changes, f); err != nil { + + actorChanges, err := processor.Actors(ctx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) + if err != nil { + return false, err + } + + if err := cbor.Process(ctx, messageChanges, actorChanges, f); err != nil { return false, err } return true, nil @@ -605,6 +612,48 @@ func (m *LilyNodeAPI) GetMessageExecutionsForTipSet(ctx context.Context, next *t return out, nil } +func (m *LilyNodeAPI) GetMessageExecutionsForTipSetV2(ctx context.Context, next *types.TipSet, current *types.TipSet) ([]*lens.MessageExecutionV2, error) { + // this is defined in the lily daemon dep injection constructor, failure here is a developer error. + msgMonitor, ok := m.ExecMonitor.(*modules.BufferedExecMonitor) + if !ok { + panic(fmt.Sprintf("bad cast, developer error expected modules.BufferedExecMonitor, got %T", m.ExecMonitor)) + } + + // if lily was watching the chain when this tipset was applied then its exec monitor will already + // contain executions for this tipset. + executions, err := msgMonitor.ExecutionFor(current) + if err != nil { + if err == modules.ErrExecutionTraceNotFound { + // if lily hasn't watched this tipset be applied then we need to compute its execution trace. + // this will likely be the case for most walk tasks. + _, err := m.StateManager.ExecutionTraceWithMonitor(ctx, current, msgMonitor) + if err != nil { + return nil, fmt.Errorf("failed to compute execution trace for tipset %s: %w", current.Key().String(), err) + } + // the above call will populate the msgMonitor with an execution trace for this tipset, get it. + executions, err = msgMonitor.ExecutionFor(current) + if err != nil { + return nil, fmt.Errorf("failed to find execution trace for tipset %s: %w", current.Key().String(), err) + } + } else { + return nil, fmt.Errorf("failed to extract message execution for tipset %s: %w", next, err) + } + } + + out := make([]*lens.MessageExecutionV2, len(executions)) + for idx, execution := range executions { + out[idx] = &lens.MessageExecutionV2{ + Cid: execution.Mcid, + StateRoot: execution.TipSet.ParentState(), + Height: execution.TipSet.Height(), + Message: execution.Msg, + Ret: execution.Ret, + Implicit: execution.Implicit, + } + } + return out, nil +} + // ComputeBaseFee calculates the base-fee of the specified tipset. func (m *LilyNodeAPI) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) { return m.ChainAPI.Chain.ComputeBaseFee(ctx, ts) diff --git a/pkg/extract/procesor/process.go b/pkg/extract/processor/actors.go similarity index 97% rename from pkg/extract/procesor/process.go rename to pkg/extract/processor/actors.go index 51d74f906..21887d4d4 100644 --- a/pkg/extract/procesor/process.go +++ b/pkg/extract/processor/actors.go @@ -1,4 +1,4 @@ -package procesor +package processor import ( "context" @@ -60,7 +60,7 @@ type StateDiffResult struct { type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version -func ProcessActorStateChanges(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, nvg NetworkVersionGetter) (*ActorStateChanges, error) { +func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, nvg NetworkVersionGetter) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { return nil, err diff --git a/pkg/extract/processor/messages.go b/pkg/extract/processor/messages.go new file mode 100644 index 000000000..8204331ba --- /dev/null +++ b/pkg/extract/processor/messages.go @@ -0,0 +1,216 @@ +package processor + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/lens" + "github.com/filecoin-project/lily/tasks" +) + +type MessageStateChanges struct { + Current *types.TipSet + Executed *types.TipSet + BaseFee abi.TokenAmount + FullBlocks map[cid.Cid]*FullBlock + ImplicitMessages []*ImplicitMessage +} + +type FullBlock struct { + Block *types.BlockHeader + SecpMessages []*SignedChainMessage + BlsMessages []*ChainMessage +} + +// SignedChainMessage is a signed (secp) message appearing on chain. Receipt is null if the message was not executed. +type SignedChainMessage struct { + Message *types.SignedMessage + Receipt *ChainMessageReceipt + VmMessages VmMessageList +} + +// ChainMessage is an unsigned (bls) message appearing on chain. Receipt is null if the message was not executed. +type ChainMessage struct { + Message *types.Message + Receipt *ChainMessageReceipt + VmMessages VmMessageList +} + +// ImplicitMessage is an implicitly executed message not appearing on chain. +type ImplicitMessage struct { + Message *types.Message + Receipt *ImplicitMessageReceipt + VmMessages VmMessageList +} + +// ChainMessageReceipt contains a MessageReceipt and other metadata. +type ChainMessageReceipt struct { + Receipt types.MessageReceipt `cborgen:"receipt"` + GasOutputs *MessageGasOutputs `cborgen:"gas"` + ActorError *ActorError `cborgen:"errors"` + Index int `cborgen:"index"` +} + +type ImplicitMessageReceipt struct { + Receipt types.MessageReceipt `cborgen:"receipt"` + GasOutputs *MessageGasOutputs `cborgen:"gas"` + ActorError *ActorError `cborgen:"errors"` +} + +// MessageGasOutputs contains the gas used during a message's execution. +type MessageGasOutputs struct { + BaseFeeBurn abi.TokenAmount `cborgen:"basefeeburn"` + OverEstimationBurn abi.TokenAmount `cborgen:"overestimationburn"` + MinerPenalty abi.TokenAmount `cborgen:"minerpenalty"` + MinerTip abi.TokenAmount `cborgen:"minertip"` + Refund abi.TokenAmount `cborgen:"refund"` + GasRefund int64 `cborgen:"gasrufund"` + GasBurned int64 `cborgen:"gasburned"` +} + +// ActorError contains any errors encountered during a message's execution. +type ActorError struct { + Fatal bool `cborgen:"fatal"` + RetCode exitcode.ExitCode `cborgen:"retcode"` + Error string `cborgen:"error"` +} + +func Messages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*MessageStateChanges, error) { + // fist get all messages included in the executed tipset, not all of these messages will have receipts since some were not executed. + blkMsgs, err := api.TipSetBlockMessages(ctx, executed) + if err != nil { + return nil, err + } + // build two maps containing all signed (secpMsgs) and unsigned (blsMsgs) messages. + secpBlkMsgs := make(map[cid.Cid][]*SignedChainMessage) + blsBlkMsgs := make(map[cid.Cid][]*ChainMessage) + secpMsgs := make(map[cid.Cid]*SignedChainMessage) + blsMsgs := make(map[cid.Cid]*ChainMessage) + for _, blk := range blkMsgs { + blkCid := blk.Block.Cid() + for _, msg := range blk.SecpMessages { + secpMsg := &SignedChainMessage{Message: msg} + secpBlkMsgs[blkCid] = append(secpBlkMsgs[blkCid], secpMsg) + secpMsgs[msg.Cid()] = secpMsg + } + for _, msg := range blk.BlsMessages { + blsMsg := &ChainMessage{Message: msg} + blsBlkMsgs[blkCid] = append(blsBlkMsgs[blkCid], blsMsg) + blsMsgs[msg.Cid()] = blsMsg + } + } + + exeBlkMsgs, err := api.TipSetMessageReceipts(ctx, current, executed) + if err != nil { + return nil, err + } + for _, ebm := range exeBlkMsgs { + itr, err := ebm.Iterator() + if err != nil { + return nil, err + } + for itr.HasNext() { + msg, recIdx, rec := itr.Next() + if secpMsg, ok := secpMsgs[msg.Cid()]; ok { + secpMsg.Receipt = &ChainMessageReceipt{ + Receipt: *rec, + Index: recIdx, + } + } else if blsMsg, ok := blsMsgs[msg.Cid()]; ok { + blsMsg.Receipt = &ChainMessageReceipt{ + Receipt: *rec, + Index: recIdx, + } + } else { + panic("developer error") + } + } + } + + msgExe, err := api.MessageExecutionsV2(ctx, current, executed) + if err != nil { + return nil, err + } + + var im []*ImplicitMessage + for _, emsg := range msgExe { + vmMsgs, err := ProcessVmMessages(ctx, emsg) + if err != nil { + return nil, err + } + if emsg.Implicit { + im = append(im, &ImplicitMessage{ + Message: emsg.Message, + VmMessages: vmMsgs, + Receipt: &ImplicitMessageReceipt{ + Receipt: emsg.Ret.MessageReceipt, + GasOutputs: GetMessageGasOutputs(emsg), + ActorError: GetActorError(emsg), + }, + }) + } else { + if secpMsg, ok := secpMsgs[emsg.Cid]; ok { + secpMsg.Receipt.GasOutputs = GetMessageGasOutputs(emsg) + secpMsg.Receipt.ActorError = GetActorError(emsg) + secpMsg.VmMessages = vmMsgs + } else if blsMsg, ok := blsMsgs[emsg.Cid]; ok { + blsMsg.Receipt.ActorError = GetActorError(emsg) + blsMsg.Receipt.GasOutputs = GetMessageGasOutputs(emsg) + blsMsg.VmMessages = vmMsgs + } else { + panic("developer error") + } + } + } + + baseFee, err := api.ComputeBaseFee(ctx, executed) + if err != nil { + return nil, err + } + + out := &MessageStateChanges{ + Current: current, + Executed: executed, + BaseFee: baseFee, + FullBlocks: make(map[cid.Cid]*FullBlock), + ImplicitMessages: im, + } + for _, blk := range blkMsgs { + out.FullBlocks[blk.Block.Cid()] = &FullBlock{ + Block: blk.Block, + SecpMessages: secpBlkMsgs[blk.Block.Cid()], + BlsMessages: blsBlkMsgs[blk.Block.Cid()], + } + } + return out, nil +} + +func GetMessageGasOutputs(msg *lens.MessageExecutionV2) *MessageGasOutputs { + if msg.Ret.GasCosts != nil { + return &MessageGasOutputs{ + BaseFeeBurn: msg.Ret.GasCosts.BaseFeeBurn, + OverEstimationBurn: msg.Ret.GasCosts.OverEstimationBurn, + MinerPenalty: msg.Ret.GasCosts.MinerPenalty, + MinerTip: msg.Ret.GasCosts.MinerTip, + Refund: msg.Ret.GasCosts.Refund, + GasRefund: msg.Ret.GasCosts.GasRefund, + GasBurned: msg.Ret.GasCosts.GasBurned, + } + } + return nil +} + +func GetActorError(msg *lens.MessageExecutionV2) *ActorError { + if msg.Ret.ActorErr != nil { + return &ActorError{ + Fatal: msg.Ret.ActorErr.IsFatal(), + RetCode: msg.Ret.ActorErr.RetCode(), + Error: msg.Ret.ActorErr.Error(), + } + } + return nil +} diff --git a/pkg/extract/processor/vm.go b/pkg/extract/processor/vm.go new file mode 100644 index 000000000..58db3606e --- /dev/null +++ b/pkg/extract/processor/vm.go @@ -0,0 +1,157 @@ +package processor + +import ( + "context" + "io" + "time" + + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/lens" +) + +type VmMessage struct { + Source cid.Cid `cborgen:"source"` + Message *types.Message `cborgen:"message"` + Receipt types.MessageReceipt `cborgen:"receipt"` + GasTrace []*VmMessageGasTrace `cborgen:"gas"` + Error string `cborgen:"error"` + Index int `cborgen:"index"` +} + +type VmMessageGasTrace struct { + Name string `cborgen:"name"` + Location []Loc `cborgen:"location"` + TotalGas int64 `cborgen:"totalgas"` + ComputeGas int64 `cborgen:"computegas"` + StorageGas int64 `cborgen:"storagegas"` + TotalVirtualGas int64 `cborgen:"totalvirtgas"` + VirtualComputeGas int64 `cborgen:"virtcomputegas"` + VirtualStorageGas int64 `cborgen:"cirtstoragegas"` +} + +type Loc struct { + File string `cborgen:"file"` + Line int `cborgen:"line"` + Function string `cborgen:"function"` +} + +func (v *VmMessage) UnmarshalCBOR(r io.Reader) error { + //TODO implement me + panic("implement me") +} + +func (v *VmMessage) MarshalCBOR(w io.Writer) error { + //TODO implement me + panic("implement me") +} + +type VmMessageList []*VmMessage + +func (vml VmMessageList) ToAdtArray(store adt.Store, bw int) (cid.Cid, error) { + msgAmt, err := adt.MakeEmptyArray(store, bw) + if err != nil { + return cid.Undef, nil + } + + for _, msg := range vml { + if err := msgAmt.Set(uint64(msg.Index), msg); err != nil { + return cid.Undef, err + } + } + return msgAmt.Root() +} + +func VmMessageListFromAdtArray(store adt.Store, root cid.Cid, bw int) (VmMessageList, error) { + arr, err := adt.AsArray(store, root, bw) + if err != nil { + return nil, err + } + out := make(VmMessageList, arr.Length()) + msg := new(VmMessage) + idx := 0 + if err := arr.ForEach(msg, func(i int64) error { + val := new(VmMessage) + *val = *msg + out[idx] = val + idx++ + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +func ProcessVmMessages(ctx context.Context, source *lens.MessageExecutionV2) (VmMessageList, error) { + trace := GetChildMessagesOf(source) + out := make([]*VmMessage, len(trace)) + for traceIdx, vmmsg := range trace { + vmGas := make([]*VmMessageGasTrace, len(vmmsg.GasCharge)) + + for gasIdx, g := range vmmsg.GasCharge { + loc := make([]Loc, len(g.Location)) + + for locIdx, l := range g.Location { + loc[locIdx] = Loc{ + File: l.File, + Line: l.Line, + Function: l.Function, + } + } + + vmGas[gasIdx] = &VmMessageGasTrace{ + Name: g.Name, + Location: loc, + TotalGas: g.TotalGas, + ComputeGas: g.ComputeGas, + StorageGas: g.StorageGas, + TotalVirtualGas: g.TotalVirtualGas, + VirtualComputeGas: g.VirtualComputeGas, + VirtualStorageGas: g.VirtualStorageGas, + } + } + + out[traceIdx] = &VmMessage{ + Source: source.Cid, + Message: vmmsg.Message, + Receipt: *vmmsg.Receipt, + GasTrace: vmGas, + Error: vmmsg.Error, + Index: vmmsg.Index, + } + } + return out, nil +} + +type vmMessageTrace struct { + Message *types.Message + Receipt *types.MessageReceipt + Error string + Duration time.Duration + GasCharge []*types.GasTrace + Index int +} + +func GetChildMessagesOf(m *lens.MessageExecutionV2) []*vmMessageTrace { + var out []*vmMessageTrace + index := 0 + walkExecutionTrace(&m.Ret.ExecutionTrace, &out, &index) + return out +} + +func walkExecutionTrace(et *types.ExecutionTrace, trace *[]*vmMessageTrace, index *int) { + for _, sub := range et.Subcalls { + *trace = append(*trace, &vmMessageTrace{ + Message: sub.Msg, + Receipt: sub.MsgRct, + Error: sub.Error, + Duration: sub.Duration, + GasCharge: sub.GasCharges, + Index: *index, + }) + *index++ + walkExecutionTrace(&sub, trace, index) //nolint:scopelint,gosec + } +} diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 38995b7e8..7e536bdc9 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -10,7 +10,9 @@ import ( powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" - "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) const actorDiffPath = "pkg/extract/actors/actordiff/cbor_gen.go" @@ -34,8 +36,14 @@ const marketDiffPkg = "v0" const powerDiffPath = "pkg/extract/actors/powerdiff/v0/cbor_gen.go" const powerDiffPkg = "v0" -const IPLDActorContainerPath = "pkg/transform/cbor/cbor_gen.go" -const IPLDActorContainerPkg = "cbor" +const IPLDActorContainerPath = "pkg/transform/cbor/actors/cbor_gen.go" +const IPLDActorContainerPkg = "actors" + +const MessageStatePath = "pkg/extract/processor/cbor_gen.go" +const MessageStatePkg = "processor" + +const MessageContainerPath = "pkg/transform/cbor/messages/cbor_gen.go" +const MessageContainerPkg = "messages" func main() { if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, @@ -96,7 +104,29 @@ func main() { } if err := cbg.WriteMapEncodersToFile(IPLDActorContainerPath, IPLDActorContainerPkg, - cbor.ActorIPLDContainer{}, + actors.ActorIPLDContainer{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(MessageStatePath, MessageStatePkg, + processor.ChainMessageReceipt{}, + processor.ImplicitMessageReceipt{}, + processor.MessageGasOutputs{}, + processor.ActorError{}, + processor.VmMessage{}, + processor.VmMessageGasTrace{}, + processor.Loc{}, + ); err != nil { + panic(err) + } + + if err := cbg.WriteMapEncodersToFile(MessageContainerPath, MessageContainerPkg, + messages.MessageIPLDContainer{}, + messages.FullBlockIPLDContainer{}, + messages.ChainMessageIPLDContainer{}, + messages.SignedChainMessageIPLDContainer{}, + messages.ImplicitMessageIPLDContainer{}, ); err != nil { panic(err) } diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/actors/cbor_gen.go similarity index 99% rename from pkg/transform/cbor/cbor_gen.go rename to pkg/transform/cbor/actors/cbor_gen.go index d3014e3a7..f1e2904eb 100644 --- a/pkg/transform/cbor/cbor_gen.go +++ b/pkg/transform/cbor/actors/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package cbor +package actors import ( "fmt" diff --git a/pkg/transform/cbor/init_/state.go b/pkg/transform/cbor/actors/init_/state.go similarity index 100% rename from pkg/transform/cbor/init_/state.go rename to pkg/transform/cbor/actors/init_/state.go diff --git a/pkg/transform/cbor/market/state.go b/pkg/transform/cbor/actors/market/state.go similarity index 100% rename from pkg/transform/cbor/market/state.go rename to pkg/transform/cbor/actors/market/state.go diff --git a/pkg/transform/cbor/miner/state.go b/pkg/transform/cbor/actors/miner/state.go similarity index 100% rename from pkg/transform/cbor/miner/state.go rename to pkg/transform/cbor/actors/miner/state.go diff --git a/pkg/transform/cbor/power/state.go b/pkg/transform/cbor/actors/power/state.go similarity index 100% rename from pkg/transform/cbor/power/state.go rename to pkg/transform/cbor/actors/power/state.go diff --git a/pkg/transform/cbor/actor/state.go b/pkg/transform/cbor/actors/raw/state.go similarity index 98% rename from pkg/transform/cbor/actor/state.go rename to pkg/transform/cbor/actors/raw/state.go index 09577869f..8f2f06d0d 100644 --- a/pkg/transform/cbor/actor/state.go +++ b/pkg/transform/cbor/actors/raw/state.go @@ -1,4 +1,4 @@ -package actor +package raw import ( "context" diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go new file mode 100644 index 000000000..9f1bba637 --- /dev/null +++ b/pkg/transform/cbor/actors/serialize.go @@ -0,0 +1,104 @@ +package actors + +import ( + "context" + + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" + + "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/init_" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/market" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/miner" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/power" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/raw" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors/verifreg" +) + +type ActorIPLDContainer struct { + // TODO this needs to be versioned + CurrentTipSet *types.TipSet `cborgen:"current"` + ExecutedTipSet *types.TipSet `cborgen:"executed"` + MinerActors cid.Cid `cborgen:"miners"` + VerifregActor *cid.Cid `cborgen:"verifreg"` + ActorStates cid.Cid `cborgen:"actors"` + InitActor *cid.Cid `cborgen:"init"` + MarketActor cid.Cid `cborgen:"market"` + PowerActor cid.Cid `cborgen:"power"` +} + +func (a *ActorIPLDContainer) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("current", a.CurrentTipSet.Key().String()), + attribute.String("executed", a.ExecutedTipSet.Key().String()), + attribute.String("miners", a.MinerActors.String()), + attribute.String("actors", a.ActorStates.String()), + attribute.String("market", a.MarketActor.String()), + attribute.String("power", a.PowerActor.String()), + } +} + +func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range a.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + +func ProcessActorsStates(ctx context.Context, bs blockstore.Blockstore, changes *processor.ActorStateChanges) (*ActorIPLDContainer, error) { + out := &ActorIPLDContainer{ + CurrentTipSet: changes.Current, + ExecutedTipSet: changes.Executed, + } + if changes.MinerActors != nil { + minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) + if err != nil { + return nil, err + } + out.MinerActors = minerRoot + } + + if changes.VerifregActor != nil { + verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) + if err != nil { + return nil, err + } + out.VerifregActor = &verifregRoot + } + + if changes.ActorStates != nil { + actorsRoot, err := raw.HandleChanges(ctx, bs, changes.ActorStates) + if err != nil { + return nil, err + } + out.ActorStates = actorsRoot + } + + if changes.InitActor != nil { + initRoot, err := init_.HandleChanges(ctx, bs, changes.InitActor) + if err != nil { + return nil, err + } + out.InitActor = &initRoot + } + + if changes.MarketActor != nil { + marketRoot, err := market.HandleChange(ctx, bs, changes.MarketActor) + if err != nil { + return nil, err + } + out.MarketActor = marketRoot + } + + if changes.PowerActor != nil { + powerRoot, err := power.HandleChange(ctx, bs, changes.PowerActor) + if err != nil { + return nil, err + } + out.PowerActor = powerRoot + } + return out, nil +} diff --git a/pkg/transform/cbor/verifreg/state.go b/pkg/transform/cbor/actors/verifreg/state.go similarity index 100% rename from pkg/transform/cbor/verifreg/state.go rename to pkg/transform/cbor/actors/verifreg/state.go diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go new file mode 100644 index 000000000..4e6f33db9 --- /dev/null +++ b/pkg/transform/cbor/messages/serialize.go @@ -0,0 +1,177 @@ +package messages + +import ( + "context" + "io" + + "github.com/filecoin-project/go-state-types/abi" + adtstore "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/processor" +) + +type MessageIPLDContainer struct { + CurrentTipSet *types.TipSet `cborgen:"current"` + ExecutedTipSet *types.TipSet `cborgen:"executed"` + BaseFee abi.TokenAmount `cborgen:"base_fee"` + FullBlocks cid.Cid `cborgen:"full_blocks"` // HAMT[blkCid]FullBlockIPLDContainer + ImplicitMessages cid.Cid `cborgen:"implicit_messages"` // HAMT[implicitCID]ImplicitMessageIPLDContainer +} + +type FullBlockIPLDContainer struct { + BlockHeader *types.BlockHeader `cborgen:"block_header"` + SecpMessages cid.Cid `cborgen:"secp_messages"` + BlsMessages cid.Cid `cborgen:"bls_messages"` +} + +func (f *FullBlockIPLDContainer) MarshalCBOR(w io.Writer) error { + //TODO implement me + panic("implement me") +} + +func ProcessMessages(ctx context.Context, store adtstore.Store, changes *processor.MessageStateChanges) (*MessageIPLDContainer, error) { + fullBlkHamt, err := adt2.MakeEmptyMap(store, 5) + if err != nil { + return nil, err + } + for blkCid, fb := range changes.FullBlocks { + blsMsgHamt, err := ProcessChainMessages(ctx, store, fb.BlsMessages) + if err != nil { + return nil, err + } + + secpMsgHamt, err := ProcessSignedChainMessages(ctx, store, fb.SecpMessages) + if err != nil { + return nil, err + } + + if err := fullBlkHamt.Put(abi.CidKey(blkCid), &FullBlockIPLDContainer{ + BlockHeader: fb.Block, + SecpMessages: secpMsgHamt, + BlsMessages: blsMsgHamt, + }); err != nil { + return nil, err + } + } + + implicitMsgHamt, err := ProcessImplicitMessages(ctx, store, changes.ImplicitMessages) + if err != nil { + return nil, err + } + + fullBlkRoot, err := fullBlkHamt.Root() + if err != nil { + return nil, err + } + + return &MessageIPLDContainer{ + CurrentTipSet: changes.Current, + ExecutedTipSet: changes.Executed, + BaseFee: changes.BaseFee, + FullBlocks: fullBlkRoot, + ImplicitMessages: implicitMsgHamt, + }, nil +} + +type ChainMessageIPLDContainer struct { + Message *types.Message `cborgen:"message"` + Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +func (c *ChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + //TODO implement me + panic("implement me") +} + +func ProcessChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { + messageHamt, err := adt2.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &ChainMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} + +type SignedChainMessageIPLDContainer struct { + Message *types.SignedMessage `cborgen:"message"` + Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +func (s *SignedChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + //TODO implement me + panic("implement me") +} + +func ProcessSignedChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { + messageHamt, err := adt2.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &SignedChainMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} + +type ImplicitMessageIPLDContainer struct { + Message *types.Message `cborgen:"message"` + Receipt *processor.ImplicitMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` +} + +func (i *ImplicitMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + //TODO implement me + panic("implement me") +} + +// ProcessImplicitMessages returns the root of a hamt node containing the set of implicit messages +func ProcessImplicitMessages(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { + messageHamt, err := adt2.MakeEmptyMap(store, 5) + if err != nil { + return cid.Undef, err + } + + for _, msg := range messages { + vmMsgRoot, err := msg.VmMessages.ToAdtArray(store, 5) + if err != nil { + return cid.Undef, err + } + if err := messageHamt.Put(abi.CidKey(msg.Message.Cid()), &ImplicitMessageIPLDContainer{ + Message: msg.Message, + Receipt: msg.Receipt, + VmMessagesAmt: vmMsgRoot, + }); err != nil { + return cid.Undef, err + } + } + return messageHamt.Root() +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index caeefe7ed..42531c0bf 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -4,74 +4,45 @@ import ( "context" "io" + adtStore "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/chain/types" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "github.com/ipld/go-car/util" - "go.opentelemetry.io/otel/attribute" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - v1car "github.com/ipld/go-car" + "github.com/ipld/go-car/util" - "github.com/filecoin-project/lily/pkg/extract/procesor" - "github.com/filecoin-project/lily/pkg/transform/cbor/actor" - "github.com/filecoin-project/lily/pkg/transform/cbor/init_" - "github.com/filecoin-project/lily/pkg/transform/cbor/market" - "github.com/filecoin-project/lily/pkg/transform/cbor/miner" - "github.com/filecoin-project/lily/pkg/transform/cbor/power" - "github.com/filecoin-project/lily/pkg/transform/cbor/verifreg" + "github.com/filecoin-project/lily/pkg/extract/processor" + cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + messages2 "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) var log = logging.Logger("lily/transform/cbor") -type ActorIPLDContainer struct { - // TODO this needs to be versioned - CurrentTipSet *types.TipSet `cborgen:"current"` - ExecutedTipSet *types.TipSet `cborgen:"executed"` - MinerActors cid.Cid `cborgen:"miners"` - VerifregActor *cid.Cid `cborgen:"verifreg"` - ActorStates cid.Cid `cborgen:"actors"` - InitActor *cid.Cid `cborgen:"init"` - MarketActor cid.Cid `cborgen:"market"` - PowerActor cid.Cid `cborgen:"power"` -} +func Process(ctx context.Context, messages *processor.MessageStateChanges, actors *processor.ActorStateChanges, w io.Writer) error { + bs := blockstore.NewMemorySync() + store := adtStore.WrapBlockStore(ctx, bs) -func (a *ActorIPLDContainer) Attributes() []attribute.KeyValue { - return []attribute.KeyValue{ - attribute.String("current", a.CurrentTipSet.Key().String()), - attribute.String("executed", a.ExecutedTipSet.Key().String()), - attribute.String("miners", a.MinerActors.String()), - attribute.String("actors", a.ActorStates.String()), - attribute.String("market", a.MarketActor.String()), - attribute.String("power", a.PowerActor.String()), + messageStateContainer, err := messages2.ProcessMessages(ctx, store, messages) + if err != nil { + return err } -} - -func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) error { - for _, a := range a.Attributes() { - enc.AddString(string(a.Key), a.Value.Emit()) + messageStatesRoot, err := store.Put(ctx, messageStateContainer) + if err != nil { + return err } - return nil -} -func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io.Writer) error { - bs := blockstore.NewMemorySync() - store := adt2.WrapBlockStore(ctx, bs) - actorStates, err := ProcessActors(ctx, bs, changes) + // TODO pass the adtStore not the blockstore. + actorStateContainer, err := cboractors.ProcessActorsStates(ctx, bs, actors) if err != nil { return err } - - actorStatesRoot, err := store.Put(ctx, actorStates) + actorStatesRoot, err := store.Put(ctx, actorStateContainer) if err != nil { return err } - log.Infow("Wrote Delta", "root", actorStatesRoot.String(), zap.Inline(actorStates)) + if err := v1car.WriteHeader(&v1car.CarHeader{ - Roots: []cid.Cid{actorStatesRoot}, + Roots: []cid.Cid{actorStatesRoot, messageStatesRoot}, Version: 1, }, w); err != nil { return err @@ -91,58 +62,3 @@ func ProcessState(ctx context.Context, changes *procesor.ActorStateChanges, w io } return nil } - -func ProcessActors(ctx context.Context, bs blockstore.Blockstore, changes *procesor.ActorStateChanges) (*ActorIPLDContainer, error) { - out := &ActorIPLDContainer{ - CurrentTipSet: changes.Current, - ExecutedTipSet: changes.Executed, - } - if changes.MinerActors != nil { - minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) - if err != nil { - return nil, err - } - out.MinerActors = minerRoot - } - - if changes.VerifregActor != nil { - verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) - if err != nil { - return nil, err - } - out.VerifregActor = &verifregRoot - } - - if changes.ActorStates != nil { - actorsRoot, err := actor.HandleChanges(ctx, bs, changes.ActorStates) - if err != nil { - return nil, err - } - out.ActorStates = actorsRoot - } - - if changes.InitActor != nil { - initRoot, err := init_.HandleChanges(ctx, bs, changes.InitActor) - if err != nil { - return nil, err - } - out.InitActor = &initRoot - } - - if changes.MarketActor != nil { - marketRoot, err := market.HandleChange(ctx, bs, changes.MarketActor) - if err != nil { - return nil, err - } - out.MarketActor = marketRoot - } - - if changes.PowerActor != nil { - powerRoot, err := power.HandleChange(ctx, bs, changes.PowerActor) - if err != nil { - return nil, err - } - out.PowerActor = powerRoot - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/actors/processor.go index 2715b4c73..1009ae6e4 100644 --- a/pkg/transform/timescale/actors/processor.go +++ b/pkg/transform/timescale/actors/processor.go @@ -21,7 +21,7 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" - "github.com/filecoin-project/lily/pkg/transform/cbor" + cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" @@ -43,7 +43,7 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe adtStore := store.WrapBlockStore(ctx, bs) - actorIPLDContainer := new(cbor.ActorIPLDContainer) + actorIPLDContainer := new(cboractors.ActorIPLDContainer) if err := adtStore.Get(ctx, header.Roots[0], actorIPLDContainer); err != nil { return err } diff --git a/tasks/api.go b/tasks/api.go index 9700a7a81..21a5460d8 100644 --- a/tasks/api.go +++ b/tasks/api.go @@ -53,6 +53,7 @@ type DataSource interface { MinerPower(ctx context.Context, addr address.Address, ts *types.TipSet) (*api.MinerPower, error) ActorStateChanges(ctx context.Context, ts, pts *types.TipSet) (ActorStateChangeDiff, error) MessageExecutions(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecution, error) + MessageExecutionsV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) Store() adt.Store ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) From bd75f435ab9d6b43f371dae76f70cf9a82ad5482 Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 13 Jan 2023 19:49:11 -0800 Subject: [PATCH 24/48] add message extraction to car file --- lens/lily/impl.go | 28 +- pkg/extract/processor/cbor_gen.go | 1701 ++++++++++++++++++++++ pkg/extract/processor/messages.go | 6 +- pkg/extract/processor/vm.go | 83 +- pkg/transform/cbor/messages/cbor_gen.go | 885 +++++++++++ pkg/transform/cbor/messages/serialize.go | 21 - 6 files changed, 2650 insertions(+), 74 deletions(-) create mode 100644 pkg/extract/processor/cbor_gen.go create mode 100644 pkg/transform/cbor/messages/cbor_gen.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index b631d378d..122215770 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -27,6 +27,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -162,14 +163,29 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } + grp, grpCtx := errgroup.WithContext(ctx) // TODO parallelize the below - messageChanges, err := processor.Messages(ctx, taskAPI, currentTs, executedTs) - if err != nil { - return false, err - } + var ( + messageChanges *processor.MessageStateChanges + actorChanges *processor.ActorStateChanges + ) + grp.Go(func() error { + messageChanges, err = processor.Messages(grpCtx, taskAPI, currentTs, executedTs) + if err != nil { + return err + } + return nil + }) + grp.Go(func() error { + actorChanges, err = processor.Actors(grpCtx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) + if err != nil { + return err + } - actorChanges, err := processor.Actors(ctx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) - if err != nil { + return nil + }) + + if err := grp.Wait(); err != nil { return false, err } diff --git a/pkg/extract/processor/cbor_gen.go b/pkg/extract/processor/cbor_gen.go new file mode 100644 index 000000000..1ca287217 --- /dev/null +++ b/pkg/extract/processor/cbor_gen.go @@ -0,0 +1,1701 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package processor + +import ( + "fmt" + "io" + "math" + "sort" + + exitcode "github.com/filecoin-project/go-state-types/exitcode" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *ChainMessageReceipt) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasOutputs (processor.MessageGasOutputs) (struct) + if len("gas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gas")); err != nil { + return err + } + + if err := t.GasOutputs.MarshalCBOR(cw); err != nil { + return err + } + + // t.ActorError (processor.ActorError) (struct) + if len("errors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"errors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("errors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("errors")); err != nil { + return err + } + + if err := t.ActorError.MarshalCBOR(cw); err != nil { + return err + } + + // t.Index (int64) (int64) + if len("index") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"index\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("index")); err != nil { + return err + } + + if t.Index >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Index)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Index-1)); err != nil { + return err + } + } + return nil +} + +func (t *ChainMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { + *t = ChainMessageReceipt{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ChainMessageReceipt: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.GasOutputs (processor.MessageGasOutputs) (struct) + case "gas": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.GasOutputs = new(MessageGasOutputs) + if err := t.GasOutputs.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.GasOutputs pointer: %w", err) + } + } + + } + // t.ActorError (processor.ActorError) (struct) + case "errors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ActorError = new(ActorError) + if err := t.ActorError.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ActorError pointer: %w", err) + } + } + + } + // t.Index (int64) (int64) + case "index": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Index = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ImplicitMessageReceipt) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasOutputs (processor.MessageGasOutputs) (struct) + if len("gas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gas")); err != nil { + return err + } + + if err := t.GasOutputs.MarshalCBOR(cw); err != nil { + return err + } + + // t.ActorError (processor.ActorError) (struct) + if len("errors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"errors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("errors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("errors")); err != nil { + return err + } + + if err := t.ActorError.MarshalCBOR(cw); err != nil { + return err + } + return nil +} + +func (t *ImplicitMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { + *t = ImplicitMessageReceipt{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ImplicitMessageReceipt: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.GasOutputs (processor.MessageGasOutputs) (struct) + case "gas": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.GasOutputs = new(MessageGasOutputs) + if err := t.GasOutputs.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.GasOutputs pointer: %w", err) + } + } + + } + // t.ActorError (processor.ActorError) (struct) + case "errors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ActorError = new(ActorError) + if err := t.ActorError.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ActorError pointer: %w", err) + } + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *MessageGasOutputs) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{167}); err != nil { + return err + } + + // t.BaseFeeBurn (big.Int) (struct) + if len("basefeeburn") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"basefeeburn\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("basefeeburn"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("basefeeburn")); err != nil { + return err + } + + if err := t.BaseFeeBurn.MarshalCBOR(cw); err != nil { + return err + } + + // t.OverEstimationBurn (big.Int) (struct) + if len("overestimationburn") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"overestimationburn\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("overestimationburn"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("overestimationburn")); err != nil { + return err + } + + if err := t.OverEstimationBurn.MarshalCBOR(cw); err != nil { + return err + } + + // t.MinerPenalty (big.Int) (struct) + if len("minerpenalty") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"minerpenalty\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("minerpenalty"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("minerpenalty")); err != nil { + return err + } + + if err := t.MinerPenalty.MarshalCBOR(cw); err != nil { + return err + } + + // t.MinerTip (big.Int) (struct) + if len("minertip") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"minertip\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("minertip"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("minertip")); err != nil { + return err + } + + if err := t.MinerTip.MarshalCBOR(cw); err != nil { + return err + } + + // t.Refund (big.Int) (struct) + if len("refund") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"refund\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("refund"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("refund")); err != nil { + return err + } + + if err := t.Refund.MarshalCBOR(cw); err != nil { + return err + } + + // t.GasRefund (int64) (int64) + if len("gasrufund") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gasrufund\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gasrufund"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gasrufund")); err != nil { + return err + } + + if t.GasRefund >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasRefund)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasRefund-1)); err != nil { + return err + } + } + + // t.GasBurned (int64) (int64) + if len("gasburned") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"gasburned\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("gasburned"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("gasburned")); err != nil { + return err + } + + if t.GasBurned >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.GasBurned)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.GasBurned-1)); err != nil { + return err + } + } + return nil +} + +func (t *MessageGasOutputs) UnmarshalCBOR(r io.Reader) (err error) { + *t = MessageGasOutputs{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("MessageGasOutputs: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.BaseFeeBurn (big.Int) (struct) + case "basefeeburn": + + { + + if err := t.BaseFeeBurn.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BaseFeeBurn: %w", err) + } + + } + // t.OverEstimationBurn (big.Int) (struct) + case "overestimationburn": + + { + + if err := t.OverEstimationBurn.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.OverEstimationBurn: %w", err) + } + + } + // t.MinerPenalty (big.Int) (struct) + case "minerpenalty": + + { + + if err := t.MinerPenalty.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.MinerPenalty: %w", err) + } + + } + // t.MinerTip (big.Int) (struct) + case "minertip": + + { + + if err := t.MinerTip.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.MinerTip: %w", err) + } + + } + // t.Refund (big.Int) (struct) + case "refund": + + { + + if err := t.Refund.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Refund: %w", err) + } + + } + // t.GasRefund (int64) (int64) + case "gasrufund": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.GasRefund = int64(extraI) + } + // t.GasBurned (int64) (int64) + case "gasburned": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.GasBurned = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ActorError) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Fatal (bool) (bool) + if len("fatal") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fatal\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fatal"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fatal")); err != nil { + return err + } + + if err := cbg.WriteBool(w, t.Fatal); err != nil { + return err + } + + // t.RetCode (exitcode.ExitCode) (int64) + if len("retcode") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"retcode\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("retcode"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("retcode")); err != nil { + return err + } + + if t.RetCode >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.RetCode)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.RetCode-1)); err != nil { + return err + } + } + + // t.Error (string) (string) + if len("error") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"error\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("error"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("error")); err != nil { + return err + } + + if len(t.Error) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Error was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Error)); err != nil { + return err + } + return nil +} + +func (t *ActorError) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorError{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorError: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Fatal (bool) (bool) + case "fatal": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajOther { + return fmt.Errorf("booleans must be major type 7") + } + switch extra { + case 20: + t.Fatal = false + case 21: + t.Fatal = true + default: + return fmt.Errorf("booleans are either major type 7, value 20 or 21 (got %d)", extra) + } + // t.RetCode (exitcode.ExitCode) (int64) + case "retcode": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.RetCode = exitcode.ExitCode(extraI) + } + // t.Error (string) (string) + case "error": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Error = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VmMessage) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Source (cid.Cid) (struct) + if len("source") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"source\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("source"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("source")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Source); err != nil { + return xerrors.Errorf("failed to write cid field t.Source: %w", err) + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (types.MessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.Error (string) (string) + if len("error") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"error\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("error"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("error")); err != nil { + return err + } + + if len(t.Error) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Error was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Error))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Error)); err != nil { + return err + } + + // t.Index (int64) (int64) + if len("index") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"index\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("index"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("index")); err != nil { + return err + } + + if t.Index >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Index)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Index-1)); err != nil { + return err + } + } + return nil +} + +func (t *VmMessage) UnmarshalCBOR(r io.Reader) (err error) { + *t = VmMessage{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VmMessage: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Source (cid.Cid) (struct) + case "source": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Source: %w", err) + } + + t.Source = c + + } + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (types.MessageReceipt) (struct) + case "receipt": + + { + + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt: %w", err) + } + + } + // t.Error (string) (string) + case "error": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Error = string(sval) + } + // t.Index (int64) (int64) + case "index": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Index = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *VmMessageGasTrace) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{168}); err != nil { + return err + } + + // t.Name (string) (string) + if len("name") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"name\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("name")); err != nil { + return err + } + + if len(t.Name) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Name was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Name)); err != nil { + return err + } + + // t.Location ([]processor.Loc) (slice) + if len("location") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"location\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("location"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("location")); err != nil { + return err + } + + if len(t.Location) > cbg.MaxLength { + return xerrors.Errorf("Slice value in field t.Location was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Location))); err != nil { + return err + } + for _, v := range t.Location { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + } + + // t.TotalGas (int64) (int64) + if len("totalgas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"totalgas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("totalgas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("totalgas")); err != nil { + return err + } + + if t.TotalGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TotalGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TotalGas-1)); err != nil { + return err + } + } + + // t.ComputeGas (int64) (int64) + if len("computegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"computegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("computegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("computegas")); err != nil { + return err + } + + if t.ComputeGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.ComputeGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.ComputeGas-1)); err != nil { + return err + } + } + + // t.StorageGas (int64) (int64) + if len("storagegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"storagegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("storagegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("storagegas")); err != nil { + return err + } + + if t.StorageGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StorageGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.StorageGas-1)); err != nil { + return err + } + } + + // t.TotalVirtualGas (int64) (int64) + if len("totalvirtgas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"totalvirtgas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("totalvirtgas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("totalvirtgas")); err != nil { + return err + } + + if t.TotalVirtualGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.TotalVirtualGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.TotalVirtualGas-1)); err != nil { + return err + } + } + + // t.VirtualComputeGas (int64) (int64) + if len("virtcomputegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"virtcomputegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("virtcomputegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("virtcomputegas")); err != nil { + return err + } + + if t.VirtualComputeGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.VirtualComputeGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.VirtualComputeGas-1)); err != nil { + return err + } + } + + // t.VirtualStorageGas (int64) (int64) + if len("cirtstoragegas") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"cirtstoragegas\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("cirtstoragegas"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("cirtstoragegas")); err != nil { + return err + } + + if t.VirtualStorageGas >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.VirtualStorageGas)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.VirtualStorageGas-1)); err != nil { + return err + } + } + return nil +} + +func (t *VmMessageGasTrace) UnmarshalCBOR(r io.Reader) (err error) { + *t = VmMessageGasTrace{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("VmMessageGasTrace: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Name (string) (string) + case "name": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Name = string(sval) + } + // t.Location ([]processor.Loc) (slice) + case "location": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.MaxLength { + return fmt.Errorf("t.Location: array too large (%d)", extra) + } + + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Location = make([]Loc, extra) + } + + for i := 0; i < int(extra); i++ { + + var v Loc + if err := v.UnmarshalCBOR(cr); err != nil { + return err + } + + t.Location[i] = v + } + + // t.TotalGas (int64) (int64) + case "totalgas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.TotalGas = int64(extraI) + } + // t.ComputeGas (int64) (int64) + case "computegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.ComputeGas = int64(extraI) + } + // t.StorageGas (int64) (int64) + case "storagegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.StorageGas = int64(extraI) + } + // t.TotalVirtualGas (int64) (int64) + case "totalvirtgas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.TotalVirtualGas = int64(extraI) + } + // t.VirtualComputeGas (int64) (int64) + case "virtcomputegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.VirtualComputeGas = int64(extraI) + } + // t.VirtualStorageGas (int64) (int64) + case "cirtstoragegas": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.VirtualStorageGas = int64(extraI) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *Loc) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.File (string) (string) + if len("file") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"file\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("file"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("file")); err != nil { + return err + } + + if len(t.File) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.File was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.File))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.File)); err != nil { + return err + } + + // t.Line (int64) (int64) + if len("line") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"line\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("line"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("line")); err != nil { + return err + } + + if t.Line >= 0 { + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Line)); err != nil { + return err + } + } else { + if err := cw.WriteMajorTypeHeader(cbg.MajNegativeInt, uint64(-t.Line-1)); err != nil { + return err + } + } + + // t.Function (string) (string) + if len("function") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"function\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("function"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("function")); err != nil { + return err + } + + if len(t.Function) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.Function was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Function))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.Function)); err != nil { + return err + } + return nil +} + +func (t *Loc) UnmarshalCBOR(r io.Reader) (err error) { + *t = Loc{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("Loc: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.File (string) (string) + case "file": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.File = string(sval) + } + // t.Line (int64) (int64) + case "line": + { + maj, extra, err := cr.ReadHeader() + var extraI int64 + if err != nil { + return err + } + switch maj { + case cbg.MajUnsignedInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 positive overflow") + } + case cbg.MajNegativeInt: + extraI = int64(extra) + if extraI < 0 { + return fmt.Errorf("int64 negative oveflow") + } + extraI = -1 - extraI + default: + return fmt.Errorf("wrong type for int64 field: %d", maj) + } + + t.Line = int64(extraI) + } + // t.Function (string) (string) + case "function": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.Function = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/processor/messages.go b/pkg/extract/processor/messages.go index 8204331ba..7d9e50f02 100644 --- a/pkg/extract/processor/messages.go +++ b/pkg/extract/processor/messages.go @@ -52,7 +52,7 @@ type ChainMessageReceipt struct { Receipt types.MessageReceipt `cborgen:"receipt"` GasOutputs *MessageGasOutputs `cborgen:"gas"` ActorError *ActorError `cborgen:"errors"` - Index int `cborgen:"index"` + Index int64 `cborgen:"index"` } type ImplicitMessageReceipt struct { @@ -118,12 +118,12 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type if secpMsg, ok := secpMsgs[msg.Cid()]; ok { secpMsg.Receipt = &ChainMessageReceipt{ Receipt: *rec, - Index: recIdx, + Index: int64(recIdx), } } else if blsMsg, ok := blsMsgs[msg.Cid()]; ok { blsMsg.Receipt = &ChainMessageReceipt{ Receipt: *rec, - Index: recIdx, + Index: int64(recIdx), } } else { panic("developer error") diff --git a/pkg/extract/processor/vm.go b/pkg/extract/processor/vm.go index 58db3606e..d11443e6a 100644 --- a/pkg/extract/processor/vm.go +++ b/pkg/extract/processor/vm.go @@ -2,7 +2,6 @@ package processor import ( "context" - "io" "time" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" @@ -13,12 +12,14 @@ import ( ) type VmMessage struct { - Source cid.Cid `cborgen:"source"` - Message *types.Message `cborgen:"message"` - Receipt types.MessageReceipt `cborgen:"receipt"` - GasTrace []*VmMessageGasTrace `cborgen:"gas"` - Error string `cborgen:"error"` - Index int `cborgen:"index"` + Source cid.Cid `cborgen:"source"` + Message *types.Message `cborgen:"message"` + Receipt types.MessageReceipt `cborgen:"receipt"` + // TODO these traces can become very long (over 200,000 entires) meaning its a bad idea to encode this as an array in cbor (it will break) + // if there is value in gathering this information then we'll want to put it in an AMT. + //GasTrace []*VmMessageGasTrace `cborgen:"gas"` + Error string `cborgen:"error"` + Index int64 `cborgen:"index"` } type VmMessageGasTrace struct { @@ -34,20 +35,10 @@ type VmMessageGasTrace struct { type Loc struct { File string `cborgen:"file"` - Line int `cborgen:"line"` + Line int64 `cborgen:"line"` Function string `cborgen:"function"` } -func (v *VmMessage) UnmarshalCBOR(r io.Reader) error { - //TODO implement me - panic("implement me") -} - -func (v *VmMessage) MarshalCBOR(w io.Writer) error { - //TODO implement me - panic("implement me") -} - type VmMessageList []*VmMessage func (vml VmMessageList) ToAdtArray(store adt.Store, bw int) (cid.Cid, error) { @@ -88,38 +79,42 @@ func ProcessVmMessages(ctx context.Context, source *lens.MessageExecutionV2) (Vm trace := GetChildMessagesOf(source) out := make([]*VmMessage, len(trace)) for traceIdx, vmmsg := range trace { - vmGas := make([]*VmMessageGasTrace, len(vmmsg.GasCharge)) - - for gasIdx, g := range vmmsg.GasCharge { - loc := make([]Loc, len(g.Location)) + // see TODO on VmMessage struct + /* + vmGas := make([]*VmMessageGasTrace, len(vmmsg.GasCharge)) + + for gasIdx, g := range vmmsg.GasCharge { + loc := make([]Loc, len(g.Location)) + + for locIdx, l := range g.Location { + loc[locIdx] = Loc{ + File: l.File, + Line: int64(l.Line), + Function: l.Function, + } + } - for locIdx, l := range g.Location { - loc[locIdx] = Loc{ - File: l.File, - Line: l.Line, - Function: l.Function, + vmGas[gasIdx] = &VmMessageGasTrace{ + Name: g.Name, + Location: loc, + TotalGas: g.TotalGas, + ComputeGas: g.ComputeGas, + StorageGas: g.StorageGas, + TotalVirtualGas: g.TotalVirtualGas, + VirtualComputeGas: g.VirtualComputeGas, + VirtualStorageGas: g.VirtualStorageGas, } } - vmGas[gasIdx] = &VmMessageGasTrace{ - Name: g.Name, - Location: loc, - TotalGas: g.TotalGas, - ComputeGas: g.ComputeGas, - StorageGas: g.StorageGas, - TotalVirtualGas: g.TotalVirtualGas, - VirtualComputeGas: g.VirtualComputeGas, - VirtualStorageGas: g.VirtualStorageGas, - } - } + */ out[traceIdx] = &VmMessage{ - Source: source.Cid, - Message: vmmsg.Message, - Receipt: *vmmsg.Receipt, - GasTrace: vmGas, - Error: vmmsg.Error, - Index: vmmsg.Index, + Source: source.Cid, + Message: vmmsg.Message, + Receipt: *vmmsg.Receipt, + //GasTrace: vmGas, + Error: vmmsg.Error, + Index: int64(vmmsg.Index), } } return out, nil diff --git a/pkg/transform/cbor/messages/cbor_gen.go b/pkg/transform/cbor/messages/cbor_gen.go new file mode 100644 index 000000000..c886fdde1 --- /dev/null +++ b/pkg/transform/cbor/messages/cbor_gen.go @@ -0,0 +1,885 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package messages + +import ( + "fmt" + "io" + "math" + "sort" + + processor "github.com/filecoin-project/lily/pkg/extract/processor" + types "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *MessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.CurrentTipSet (types.TipSet) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.CurrentTipSet.MarshalCBOR(cw); err != nil { + return err + } + + // t.ExecutedTipSet (types.TipSet) (struct) + if len("executed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"executed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("executed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("executed")); err != nil { + return err + } + + if err := t.ExecutedTipSet.MarshalCBOR(cw); err != nil { + return err + } + + // t.BaseFee (big.Int) (struct) + if len("base_fee") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"base_fee\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("base_fee"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("base_fee")); err != nil { + return err + } + + if err := t.BaseFee.MarshalCBOR(cw); err != nil { + return err + } + + // t.FullBlocks (cid.Cid) (struct) + if len("full_blocks") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"full_blocks\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("full_blocks"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("full_blocks")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.FullBlocks); err != nil { + return xerrors.Errorf("failed to write cid field t.FullBlocks: %w", err) + } + + // t.ImplicitMessages (cid.Cid) (struct) + if len("implicit_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"implicit_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("implicit_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("implicit_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ImplicitMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.ImplicitMessages: %w", err) + } + + return nil +} + +func (t *MessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = MessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("MessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.CurrentTipSet (types.TipSet) (struct) + case "current": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.CurrentTipSet = new(types.TipSet) + if err := t.CurrentTipSet.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.CurrentTipSet pointer: %w", err) + } + } + + } + // t.ExecutedTipSet (types.TipSet) (struct) + case "executed": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.ExecutedTipSet = new(types.TipSet) + if err := t.ExecutedTipSet.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.ExecutedTipSet pointer: %w", err) + } + } + + } + // t.BaseFee (big.Int) (struct) + case "base_fee": + + { + + if err := t.BaseFee.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BaseFee: %w", err) + } + + } + // t.FullBlocks (cid.Cid) (struct) + case "full_blocks": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FullBlocks: %w", err) + } + + t.FullBlocks = c + + } + // t.ImplicitMessages (cid.Cid) (struct) + case "implicit_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ImplicitMessages: %w", err) + } + + t.ImplicitMessages = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *FullBlockIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.BlockHeader (types.BlockHeader) (struct) + if len("block_header") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"block_header\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("block_header"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("block_header")); err != nil { + return err + } + + if err := t.BlockHeader.MarshalCBOR(cw); err != nil { + return err + } + + // t.SecpMessages (cid.Cid) (struct) + if len("secp_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"secp_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("secp_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("secp_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.SecpMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.SecpMessages: %w", err) + } + + // t.BlsMessages (cid.Cid) (struct) + if len("bls_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"bls_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("bls_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("bls_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.BlsMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.BlsMessages: %w", err) + } + + return nil +} + +func (t *FullBlockIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = FullBlockIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FullBlockIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.BlockHeader (types.BlockHeader) (struct) + case "block_header": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.BlockHeader = new(types.BlockHeader) + if err := t.BlockHeader.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BlockHeader pointer: %w", err) + } + } + + } + // t.SecpMessages (cid.Cid) (struct) + case "secp_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SecpMessages: %w", err) + } + + t.SecpMessages = c + + } + // t.BlsMessages (cid.Cid) (struct) + case "bls_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.BlsMessages: %w", err) + } + + t.BlsMessages = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (processor.ChainMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *ChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = ChainMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ChainMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (processor.ChainMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(processor.ChainMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *SignedChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.SignedMessage) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (processor.ChainMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *SignedChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = SignedChainMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("SignedChainMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.SignedMessage) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.SignedMessage) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (processor.ChainMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(processor.ChainMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *ImplicitMessageIPLDContainer) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{163}); err != nil { + return err + } + + // t.Message (types.Message) (struct) + if len("message") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"message\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("message"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("message")); err != nil { + return err + } + + if err := t.Message.MarshalCBOR(cw); err != nil { + return err + } + + // t.Receipt (processor.ImplicitMessageReceipt) (struct) + if len("receipt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"receipt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("receipt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("receipt")); err != nil { + return err + } + + if err := t.Receipt.MarshalCBOR(cw); err != nil { + return err + } + + // t.VmMessagesAmt (cid.Cid) (struct) + if len("vm_messages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"vm_messages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vm_messages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("vm_messages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.VmMessagesAmt); err != nil { + return xerrors.Errorf("failed to write cid field t.VmMessagesAmt: %w", err) + } + + return nil +} + +func (t *ImplicitMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { + *t = ImplicitMessageIPLDContainer{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ImplicitMessageIPLDContainer: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Message (types.Message) (struct) + case "message": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Message = new(types.Message) + if err := t.Message.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Message pointer: %w", err) + } + } + + } + // t.Receipt (processor.ImplicitMessageReceipt) (struct) + case "receipt": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.Receipt = new(processor.ImplicitMessageReceipt) + if err := t.Receipt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) + } + } + + } + // t.VmMessagesAmt (cid.Cid) (struct) + case "vm_messages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VmMessagesAmt: %w", err) + } + + t.VmMessagesAmt = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go index 4e6f33db9..b06d3548a 100644 --- a/pkg/transform/cbor/messages/serialize.go +++ b/pkg/transform/cbor/messages/serialize.go @@ -2,7 +2,6 @@ package messages import ( "context" - "io" "github.com/filecoin-project/go-state-types/abi" adtstore "github.com/filecoin-project/go-state-types/store" @@ -27,11 +26,6 @@ type FullBlockIPLDContainer struct { BlsMessages cid.Cid `cborgen:"bls_messages"` } -func (f *FullBlockIPLDContainer) MarshalCBOR(w io.Writer) error { - //TODO implement me - panic("implement me") -} - func ProcessMessages(ctx context.Context, store adtstore.Store, changes *processor.MessageStateChanges) (*MessageIPLDContainer, error) { fullBlkHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { @@ -82,11 +76,6 @@ type ChainMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func (c *ChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { - //TODO implement me - panic("implement me") -} - func ProcessChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { @@ -115,11 +104,6 @@ type SignedChainMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func (s *SignedChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { - //TODO implement me - panic("implement me") -} - func ProcessSignedChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { @@ -148,11 +132,6 @@ type ImplicitMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func (i *ImplicitMessageIPLDContainer) MarshalCBOR(w io.Writer) error { - //TODO implement me - panic("implement me") -} - // ProcessImplicitMessages returns the root of a hamt node containing the set of implicit messages func ProcessImplicitMessages(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) From 6745112965d8a0a509429f1385780a82056cfd50 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 17 Jan 2023 18:08:54 -0800 Subject: [PATCH 25/48] checking --- lens/lily/impl.go | 8 +- pkg/extract/actors/minerdiff/v0/state.go | 66 ----------- pkg/extract/actors/minerdiff/version.go | 18 --- pkg/extract/actors/powerdiff/v0/state.go | 8 +- pkg/extract/actors/types.go | 4 +- pkg/extract/actors/verifregdiff/v0/state.go | 16 ++- pkg/extract/actors/verifregdiff/v9/state.go | 12 +- pkg/extract/processor/actors.go | 6 +- pkg/extract/processor/messages.go | 55 ++++++++++ pkg/extract/processor/vm.go | 7 +- pkg/transform/cbor/actors/init_/state.go | 11 +- pkg/transform/cbor/actors/market/state.go | 10 +- pkg/transform/cbor/actors/miner/state.go | 10 +- pkg/transform/cbor/actors/power/state.go | 10 +- pkg/transform/cbor/actors/serialize.go | 116 +++++++++++--------- pkg/transform/cbor/messages/serialize.go | 51 +++------ pkg/transform/cbor/router.go | 102 +++++++++++++---- pkg/transform/cbor/types.go | 1 + testutil/lens.go | 5 + 19 files changed, 267 insertions(+), 249 deletions(-) create mode 100644 pkg/transform/cbor/types.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 122215770..e070f2de9 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -11,6 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" network2 "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/state" @@ -189,7 +190,12 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } - if err := cbor.Process(ctx, messageChanges, actorChanges, f); err != nil { + bs := blockstore.NewMemorySync() + root, err := cbor.PersistToStore(ctx, bs, currentTs, executedTs, messageChanges, actorChanges) + if err != nil { + return false, err + } + if err := cbor.WriteCar(ctx, root, 1, bs, f); err != nil { return false, err } return true, nil diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v0/state.go index 7db40d07d..754a41beb 100644 --- a/pkg/extract/actors/minerdiff/v0/state.go +++ b/pkg/extract/actors/minerdiff/v0/state.go @@ -56,10 +56,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.SectorChanges = stateChange.(SectorChangeList) case KindMinerPreCommit: stateDiff.PreCommitChanges = stateChange.(PreCommitChangeList) - case KindMinerFunds: - stateDiff.FundsChange = stateChange.(*FundsChange) - case KindMinerDebt: - stateDiff.DebtChange = stateChange.(*DebtChange) case KindMinerSectorStatus: stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) default: @@ -72,8 +68,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors type StateDiffResult struct { InfoChange *InfoChange - FundsChange *FundsChange - DebtChange *DebtChange PreCommitChanges PreCommitChangeList SectorChanges SectorChangeList SectorStatusChanges *SectorStatusChange @@ -84,10 +78,6 @@ func (s *StateDiffResult) Kind() string { } type StateChange struct { - // Funds is the funds that changed for this miner or empty. - Funds *cid.Cid `cborgen:"funds"` - // Debt is the debt that changed for this miner or empty. - Debt *cid.Cid `cborgen:"debt"` // SectorStatus is the sectors whose status changed for this miner or empty. SectorStatus *cid.Cid `cborgen:"sector_status"` // Info is the cid of the miner change info that changed for this miner or empty. @@ -176,38 +166,6 @@ func DecodeStateDiffResultFromStateChange(ctx context.Context, bs blockstore.Blo } } - // - // Funds - { - if sc.Funds != nil { - blk, err := bs.Get(ctx, *sc.Funds) - if err != nil { - return nil, err - } - funds, err := DecodeFunds(blk.RawData()) - if err != nil { - return nil, err - } - out.FundsChange = funds - } - } - - // - // Debt - { - if sc.Debt != nil { - blk, err := bs.Get(ctx, *sc.Debt) - if err != nil { - return nil, err - } - debt, err := DecodeDebt(blk.RawData()) - if err != nil { - return nil, err - } - out.DebtChange = debt - } - } - // // SectorStatus { @@ -260,30 +218,6 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore out.Info = &c } - if funds := sd.FundsChange; funds != nil { - blk, err := funds.ToStorageBlock() - if err != nil { - return nil, err - } - if err := bs.Put(ctx, blk); err != nil { - return nil, err - } - c := blk.Cid() - out.Funds = &c - } - - if debt := sd.DebtChange; debt != nil { - blk, err := debt.ToStorageBlock() - if err != nil { - return nil, err - } - if err := bs.Put(ctx, blk); err != nil { - return nil, err - } - c := blk.Cid() - out.Debt = &c - } - if sectorstatus := sd.SectorStatusChanges; sectorstatus != nil { blk, err := sectorstatus.ToStorageBlock() if err != nil { diff --git a/pkg/extract/actors/minerdiff/version.go b/pkg/extract/actors/minerdiff/version.go index b149e19b1..4de371d8b 100644 --- a/pkg/extract/actors/minerdiff/version.go +++ b/pkg/extract/actors/minerdiff/version.go @@ -22,8 +22,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version0: return &v0.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v0.Debt{}, - v0.Funds{}, v0.Info{}, v0.PreCommit{}, v0.SectorStatus{}, @@ -32,8 +30,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version2: return &v2.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v2.Debt{}, - v2.Funds{}, v2.Info{}, v2.PreCommit{}, v2.SectorStatus{}, @@ -42,8 +38,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version3: return &v3.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v3.Debt{}, - v3.Funds{}, v3.Info{}, v3.PreCommit{}, v3.SectorStatus{}, @@ -52,8 +46,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version4: return &v4.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v4.Debt{}, - v4.Funds{}, v4.Info{}, v4.PreCommit{}, v4.SectorStatus{}, @@ -62,8 +54,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version5: return &v5.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v5.Debt{}, - v5.Funds{}, v5.Info{}, v5.PreCommit{}, v5.SectorStatus{}, @@ -72,8 +62,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version6: return &v6.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v6.Debt{}, - v6.Funds{}, v6.Info{}, v6.PreCommit{}, v6.SectorStatus{}, @@ -82,8 +70,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version7: return &v7.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v7.Debt{}, - v7.Funds{}, v7.Info{}, v7.PreCommit{}, v7.SectorStatus{}, @@ -92,8 +78,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version8: return &v8.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v8.Debt{}, - v8.Funds{}, v8.Info{}, v8.PreCommit{}, v8.SectorStatus{}, @@ -102,8 +86,6 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { case actortypes.Version9: return &v9.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v9.Debt{}, - v9.Funds{}, v9.Info{}, v9.PreCommit{}, v9.SectorStatus{}, diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v0/state.go index 14c590cdd..3c9acdd04 100644 --- a/pkg/extract/actors/powerdiff/v0/state.go +++ b/pkg/extract/actors/powerdiff/v0/state.go @@ -5,7 +5,6 @@ import ( "time" "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -17,18 +16,15 @@ type StateDiffResult struct { ClaimsChanges ClaimsChangeList } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) - if claims := sd.ClaimsChanges; claims != nil { - root, err := claims.ToAdtMap(adtStore, 5) + root, err := claims.ToAdtMap(s, 5) if err != nil { return nil, err } out.Claims = root } - return out, nil } diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index 6de41aad4..40013aeff 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -4,7 +4,7 @@ import ( "context" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" "go.opentelemetry.io/otel/attribute" @@ -17,7 +17,7 @@ import ( type ActorDiffResult interface { Kind() string - MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) + MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) } type ActorDiff interface { diff --git a/pkg/extract/actors/verifregdiff/v0/state.go b/pkg/extract/actors/verifregdiff/v0/state.go index f20528fda..6676990e1 100644 --- a/pkg/extract/actors/verifregdiff/v0/state.go +++ b/pkg/extract/actors/verifregdiff/v0/state.go @@ -5,7 +5,6 @@ import ( "time" "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" @@ -23,28 +22,27 @@ func (s *StateDiffResult) Kind() string { } type StateChange struct { - Verifiers cid.Cid `cborgen:"verifiers"` - Clients cid.Cid `cborgen:"clients"` + Verifiers *cid.Cid `cborgen:"verifiers"` + Clients *cid.Cid `cborgen:"clients"` } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) { +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) if clients := sd.ClientChanges; clients != nil { - root, err := clients.ToAdtMap(adtStore, 5) + root, err := clients.ToAdtMap(s, 5) if err != nil { return nil, err } - out.Clients = root + out.Clients = &root } if verifiers := sd.VerifierChanges; verifiers != nil { - root, err := verifiers.ToAdtMap(adtStore, 5) + root, err := verifiers.ToAdtMap(s, 5) if err != nil { return nil, err } - out.Verifiers = root + out.Verifiers = &root } return out, nil } diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index cafd30fe8..c35beed30 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -4,8 +4,7 @@ import ( "context" "time" - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" + adtstore "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" @@ -28,12 +27,11 @@ func (sd *StateDiffResult) Kind() string { return "verifreg" } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store adtstore.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) if verifiers := sd.VerifierChanges; verifiers != nil { - root, err := verifiers.ToAdtMap(adtStore, 5) + root, err := verifiers.ToAdtMap(store, 5) if err != nil { return nil, err } @@ -41,7 +39,7 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore } if claims := sd.ClaimChanges; claims != nil { - root, err := claims.ToAdtMap(adtStore, 5) + root, err := claims.ToAdtMap(store, 5) if err != nil { return nil, err } @@ -49,7 +47,7 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore } if allocations := sd.AllocationsChanges; allocations != nil { - root, err := allocations.ToAdtMap(adtStore, 5) + root, err := allocations.ToAdtMap(store, 5) if err != nil { return nil, err } diff --git a/pkg/extract/processor/actors.go b/pkg/extract/processor/actors.go index 21887d4d4..2dbdf26c4 100644 --- a/pkg/extract/processor/actors.go +++ b/pkg/extract/processor/actors.go @@ -30,7 +30,7 @@ var log = logging.Logger("lily/extract/processor") type ActorStateChanges struct { Current *types.TipSet Executed *types.TipSet - ActorStates map[address.Address]actors.ActorDiffResult + RawActors map[address.Address]actors.ActorDiffResult MinerActors map[address.Address]actors.ActorDiffResult VerifregActor actors.ActorDiffResult InitActor actors.ActorDiffResult @@ -69,7 +69,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. Current: current, Executed: executed, MinerActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries - ActorStates: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries + RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } actorVersion, err := core.ActorVersionForTipSet(ctx, current, nvg) @@ -196,7 +196,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. for stateDiff := range results { switch stateDiff.ActorDiff.Kind() { case "actor": - asc.ActorStates[stateDiff.Address] = stateDiff.ActorDiff + asc.RawActors[stateDiff.Address] = stateDiff.ActorDiff case "miner": asc.MinerActors[stateDiff.Address] = stateDiff.ActorDiff case "verifreg": diff --git a/pkg/extract/processor/messages.go b/pkg/extract/processor/messages.go index 7d9e50f02..7ac16387b 100644 --- a/pkg/extract/processor/messages.go +++ b/pkg/extract/processor/messages.go @@ -12,6 +12,61 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type RootState struct { + // StateVersion contains the version of the State data structure. + // If changes are made to its structure, this version will increment. + StateVersion uint64 + + // State contains a single StateExtraction. + State cid.Cid +} + +type StateExtraction struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. + + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks map[cid.Cid]*FullBlock + // ImplicitMessages contains a list of all implicit messages executed at Parent. + ImplicitMessages []*ImplicitMessage + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors ActorStateChanges +} + +type StateExtractionIPLD struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and the messages respective receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. + + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet + + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount + + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks cid.Cid // HAMT[BlockHeaderCID]FullBlock + + // ImplicitMessages contains a map of all implicit messages executed at Parent. + ImplicitMessages cid.Cid // HAMT[MessageCID]ImplicitMessage + + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors cid.Cid // ActorStateChanges +} + type MessageStateChanges struct { Current *types.TipSet Executed *types.TipSet diff --git a/pkg/extract/processor/vm.go b/pkg/extract/processor/vm.go index d11443e6a..b66cd3d2a 100644 --- a/pkg/extract/processor/vm.go +++ b/pkg/extract/processor/vm.go @@ -15,11 +15,8 @@ type VmMessage struct { Source cid.Cid `cborgen:"source"` Message *types.Message `cborgen:"message"` Receipt types.MessageReceipt `cborgen:"receipt"` - // TODO these traces can become very long (over 200,000 entires) meaning its a bad idea to encode this as an array in cbor (it will break) - // if there is value in gathering this information then we'll want to put it in an AMT. - //GasTrace []*VmMessageGasTrace `cborgen:"gas"` - Error string `cborgen:"error"` - Index int64 `cborgen:"index"` + Error string `cborgen:"error"` + Index int64 `cborgen:"index"` } type VmMessageGasTrace struct { diff --git a/pkg/transform/cbor/actors/init_/state.go b/pkg/transform/cbor/actors/init_/state.go index 22d695bcd..72388ce84 100644 --- a/pkg/transform/cbor/actors/init_/state.go +++ b/pkg/transform/cbor/actors/init_/state.go @@ -3,18 +3,17 @@ package init_ import ( "context" - "github.com/filecoin-project/lotus/blockstore" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lily/pkg/extract/actors" ) -func HandleChanges(ctx context.Context, bs blockstore.Blockstore, addresses actors.ActorDiffResult) (cid.Cid, error) { - store := adt2.WrapBlockStore(ctx, bs) - isc, err := addresses.MarshalStateChange(ctx, bs) +func HandleChanges(ctx context.Context, s store.Store, addresses actors.ActorDiffResult) (cid.Cid, error) { + isc, err := addresses.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err } - return store.Put(ctx, isc) + return s.Put(ctx, isc) } diff --git a/pkg/transform/cbor/actors/market/state.go b/pkg/transform/cbor/actors/market/state.go index d30235281..74a07f87c 100644 --- a/pkg/transform/cbor/actors/market/state.go +++ b/pkg/transform/cbor/actors/market/state.go @@ -3,19 +3,17 @@ package market import ( "context" - "github.com/filecoin-project/lotus/blockstore" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/pkg/extract/actors" ) -func HandleChange(ctx context.Context, bs blockstore.Blockstore, market actors.ActorDiffResult) (cid.Cid, error) { - store := adt2.WrapBlockStore(ctx, bs) - vsc, err := market.MarshalStateChange(ctx, bs) +func HandleChange(ctx context.Context, s store.Store, market actors.ActorDiffResult) (cid.Cid, error) { + vsc, err := market.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err } - return store.Put(ctx, vsc) + return s.Put(ctx, vsc) } diff --git a/pkg/transform/cbor/actors/miner/state.go b/pkg/transform/cbor/actors/miner/state.go index 377058df5..fe134a255 100644 --- a/pkg/transform/cbor/actors/miner/state.go +++ b/pkg/transform/cbor/actors/miner/state.go @@ -6,21 +6,19 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/filecoin-project/lotus/blockstore" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/pkg/extract/actors" ) -func HandleChanges(ctx context.Context, bs blockstore.Blockstore, miners map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { - store := adt2.WrapBlockStore(ctx, bs) - minerHamt, err := adt.MakeEmptyMap(store, 5 /*TODO*/) +func HandleChanges(ctx context.Context, s store.Store, miners map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { + minerHamt, err := adt.MakeEmptyMap(s, 5 /*TODO*/) if err != nil { return cid.Undef, err } for addr, change := range miners { - msc, err := change.MarshalStateChange(ctx, bs) + msc, err := change.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err } diff --git a/pkg/transform/cbor/actors/power/state.go b/pkg/transform/cbor/actors/power/state.go index f58e41525..3ac0c2ff2 100644 --- a/pkg/transform/cbor/actors/power/state.go +++ b/pkg/transform/cbor/actors/power/state.go @@ -3,18 +3,16 @@ package power import ( "context" - "github.com/filecoin-project/lotus/blockstore" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/pkg/extract/actors" ) -func HandleChange(ctx context.Context, bs blockstore.Blockstore, power actors.ActorDiffResult) (cid.Cid, error) { - store := adt2.WrapBlockStore(ctx, bs) - vsc, err := power.MarshalStateChange(ctx, bs) +func HandleChange(ctx context.Context, s store.Store, power actors.ActorDiffResult) (cid.Cid, error) { + vsc, err := power.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err } - return store.Put(ctx, vsc) + return s.Put(ctx, vsc) } diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 9f1bba637..9f0f150aa 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -3,102 +3,120 @@ package actors import ( "context" - "github.com/filecoin-project/lotus/blockstore" - "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" + "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/processor" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/init_" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/market" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/miner" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/power" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/raw" - "github.com/filecoin-project/lily/pkg/transform/cbor/actors/verifreg" ) -type ActorIPLDContainer struct { - // TODO this needs to be versioned - CurrentTipSet *types.TipSet `cborgen:"current"` - ExecutedTipSet *types.TipSet `cborgen:"executed"` - MinerActors cid.Cid `cborgen:"miners"` - VerifregActor *cid.Cid `cborgen:"verifreg"` - ActorStates cid.Cid `cborgen:"actors"` - InitActor *cid.Cid `cborgen:"init"` - MarketActor cid.Cid `cborgen:"market"` - PowerActor cid.Cid `cborgen:"power"` +type ActorStateChangesIPLD struct { + DataCapActor *cid.Cid // DataCap + InitActor *cid.Cid // Init + MarketActor *cid.Cid // Market + MinerActors *cid.Cid // HAMT[address]Miner + PowerActor *cid.Cid // Power + RawActors *cid.Cid // HAMT[address]Raw + VerifregActor *cid.Cid // Veriferg } -func (a *ActorIPLDContainer) Attributes() []attribute.KeyValue { +func (a *ActorStateChangesIPLD) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ - attribute.String("current", a.CurrentTipSet.Key().String()), - attribute.String("executed", a.ExecutedTipSet.Key().String()), - attribute.String("miners", a.MinerActors.String()), - attribute.String("actors", a.ActorStates.String()), - attribute.String("market", a.MarketActor.String()), - attribute.String("power", a.PowerActor.String()), + // TODO } } -func (a *ActorIPLDContainer) MarshalLogObject(enc zapcore.ObjectEncoder) error { +func (a *ActorStateChangesIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { for _, a := range a.Attributes() { enc.AddString(string(a.Key), a.Value.Emit()) } return nil } -func ProcessActorsStates(ctx context.Context, bs blockstore.Blockstore, changes *processor.ActorStateChanges) (*ActorIPLDContainer, error) { - out := &ActorIPLDContainer{ - CurrentTipSet: changes.Current, - ExecutedTipSet: changes.Executed, - } - if changes.MinerActors != nil { - minerRoot, err := miner.HandleChanges(ctx, bs, changes.MinerActors) +func ProcessActorsStates(ctx context.Context, s store.Store, changes *processor.ActorStateChanges) (*ActorStateChangesIPLD, error) { + out := &ActorStateChangesIPLD{} + + // TODO DataCap + + if changes.InitActor != nil { + initRoot, err := PutActorDiffResult(ctx, s, changes.InitActor) if err != nil { return nil, err } - out.MinerActors = minerRoot + out.InitActor = &initRoot } - if changes.VerifregActor != nil { - verifregRoot, err := verifreg.HandleChanges(ctx, bs, changes.VerifregActor) + if changes.MarketActor != nil { + marketRoot, err := PutActorDiffResult(ctx, s, changes.MarketActor) if err != nil { return nil, err } - out.VerifregActor = &verifregRoot + out.MarketActor = &marketRoot } - if changes.ActorStates != nil { - actorsRoot, err := raw.HandleChanges(ctx, bs, changes.ActorStates) + if changes.MinerActors != nil { + minerRoot, err := PutActorDiffResultMap(ctx, s, changes.MinerActors) if err != nil { return nil, err } - out.ActorStates = actorsRoot + out.MinerActors = &minerRoot } - if changes.InitActor != nil { - initRoot, err := init_.HandleChanges(ctx, bs, changes.InitActor) + if changes.PowerActor != nil { + powerRoot, err := PutActorDiffResult(ctx, s, changes.PowerActor) if err != nil { return nil, err } - out.InitActor = &initRoot + out.PowerActor = &powerRoot } - if changes.MarketActor != nil { - marketRoot, err := market.HandleChange(ctx, bs, changes.MarketActor) + if changes.RawActors != nil { + actorsRoot, err := PutActorDiffResultMap(ctx, s, changes.RawActors) if err != nil { return nil, err } - out.MarketActor = marketRoot + out.RawActors = &actorsRoot } - if changes.PowerActor != nil { - powerRoot, err := power.HandleChange(ctx, bs, changes.PowerActor) + if changes.VerifregActor != nil { + verifregRoot, err := PutActorDiffResult(ctx, s, changes.VerifregActor) if err != nil { return nil, err } - out.PowerActor = powerRoot + + out.VerifregActor = &verifregRoot } + return out, nil } + +func PutActorDiffResult(ctx context.Context, s store.Store, result actors.ActorDiffResult) (cid.Cid, error) { + changes, err := result.MarshalStateChange(ctx, s) + if err != nil { + return cid.Undef, err + } + return s.Put(ctx, changes) +} + +func PutActorDiffResultMap(ctx context.Context, s store.Store, results map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { + actorHamt, err := adt.MakeEmptyMap(s, 5 /*TODO*/) + if err != nil { + return cid.Undef, err + } + for addr, change := range results { + msc, err := change.MarshalStateChange(ctx, s) + if err != nil { + return cid.Undef, err + } + + if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { + return cid.Undef, err + } + } + return actorHamt.Root() +} diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go index b06d3548a..27844db74 100644 --- a/pkg/transform/cbor/messages/serialize.go +++ b/pkg/transform/cbor/messages/serialize.go @@ -12,34 +12,27 @@ import ( "github.com/filecoin-project/lily/pkg/extract/processor" ) -type MessageIPLDContainer struct { - CurrentTipSet *types.TipSet `cborgen:"current"` - ExecutedTipSet *types.TipSet `cborgen:"executed"` - BaseFee abi.TokenAmount `cborgen:"base_fee"` - FullBlocks cid.Cid `cborgen:"full_blocks"` // HAMT[blkCid]FullBlockIPLDContainer - ImplicitMessages cid.Cid `cborgen:"implicit_messages"` // HAMT[implicitCID]ImplicitMessageIPLDContainer -} - type FullBlockIPLDContainer struct { BlockHeader *types.BlockHeader `cborgen:"block_header"` SecpMessages cid.Cid `cborgen:"secp_messages"` BlsMessages cid.Cid `cborgen:"bls_messages"` } -func ProcessMessages(ctx context.Context, store adtstore.Store, changes *processor.MessageStateChanges) (*MessageIPLDContainer, error) { +func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[cid.Cid]*processor.FullBlock) (cid.Cid, error) { fullBlkHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { - return nil, err + return cid.Undef, err } - for blkCid, fb := range changes.FullBlocks { - blsMsgHamt, err := ProcessChainMessages(ctx, store, fb.BlsMessages) + + for blkCid, fb := range fullBlks { + blsMsgHamt, err := MakeChainMessagesHAMT(ctx, store, fb.BlsMessages) if err != nil { - return nil, err + return cid.Undef, err } - secpMsgHamt, err := ProcessSignedChainMessages(ctx, store, fb.SecpMessages) + secpMsgHamt, err := MakeSignedChainMessagesHAMT(ctx, store, fb.SecpMessages) if err != nil { - return nil, err + return cid.Undef, err } if err := fullBlkHamt.Put(abi.CidKey(blkCid), &FullBlockIPLDContainer{ @@ -47,27 +40,11 @@ func ProcessMessages(ctx context.Context, store adtstore.Store, changes *process SecpMessages: secpMsgHamt, BlsMessages: blsMsgHamt, }); err != nil { - return nil, err + return cid.Undef, err } } - implicitMsgHamt, err := ProcessImplicitMessages(ctx, store, changes.ImplicitMessages) - if err != nil { - return nil, err - } - - fullBlkRoot, err := fullBlkHamt.Root() - if err != nil { - return nil, err - } - - return &MessageIPLDContainer{ - CurrentTipSet: changes.Current, - ExecutedTipSet: changes.Executed, - BaseFee: changes.BaseFee, - FullBlocks: fullBlkRoot, - ImplicitMessages: implicitMsgHamt, - }, nil + return fullBlkHamt.Root() } type ChainMessageIPLDContainer struct { @@ -76,7 +53,7 @@ type ChainMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func ProcessChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { +func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -104,7 +81,7 @@ type SignedChainMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func ProcessSignedChainMessages(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { +func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -132,8 +109,8 @@ type ImplicitMessageIPLDContainer struct { VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -// ProcessImplicitMessages returns the root of a hamt node containing the set of implicit messages -func ProcessImplicitMessages(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { +// MakeImplicitMessagesAMT returns the root of a hamt node containing the set of implicit messages +func MakeImplicitMessagesAMT(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 42531c0bf..5f7938d46 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -2,10 +2,13 @@ package cbor import ( "context" + "fmt" "io" + "github.com/filecoin-project/go-state-types/abi" adtStore "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" v1car "github.com/ipld/go-car" @@ -18,39 +21,36 @@ import ( var log = logging.Logger("lily/transform/cbor") -func Process(ctx context.Context, messages *processor.MessageStateChanges, actors *processor.ActorStateChanges, w io.Writer) error { - bs := blockstore.NewMemorySync() - store := adtStore.WrapBlockStore(ctx, bs) +type RootStateIPLD struct { + StateVersion uint64 - messageStateContainer, err := messages2.ProcessMessages(ctx, store, messages) - if err != nil { - return err - } - messageStatesRoot, err := store.Put(ctx, messageStateContainer) - if err != nil { - return err - } + State cid.Cid +} - // TODO pass the adtStore not the blockstore. - actorStateContainer, err := cboractors.ProcessActorsStates(ctx, bs, actors) - if err != nil { - return err - } - actorStatesRoot, err := store.Put(ctx, actorStateContainer) - if err != nil { - return err - } +type StateExtractionIPLD struct { + Current types.TipSet + Parent types.TipSet + + BaseFee abi.TokenAmount + FullBlocks cid.Cid + ImplicitMessages cid.Cid + Actors cid.Cid +} + +func WriteCar(ctx context.Context, root cid.Cid, carVersion uint64, bs blockstore.Blockstore, w io.Writer) error { if err := v1car.WriteHeader(&v1car.CarHeader{ - Roots: []cid.Cid{actorStatesRoot, messageStatesRoot}, - Version: 1, + Roots: []cid.Cid{root}, + Version: carVersion, }, w); err != nil { return err } + keyCh, err := bs.AllKeysChan(ctx) if err != nil { return err } + for key := range keyCh { blk, err := bs.Get(ctx, key) if err != nil { @@ -62,3 +62,61 @@ func Process(ctx context.Context, messages *processor.MessageStateChanges, actor } return nil } + +func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, messages *processor.MessageStateChanges, actors *processor.ActorStateChanges) (cid.Cid, error) { + store := adtStore.WrapBlockStore(ctx, bs) + + // sanity check + if !messages.Current.Equals(actors.Current) { + return cid.Undef, fmt.Errorf("actor and message current tipset does not match") + } + if !messages.Executed.Equals(actors.Executed) { + return cid.Undef, fmt.Errorf("actor and message executed tipset does not match") + } + + implicitMsgsAMT, err := messages2.MakeImplicitMessagesAMT(ctx, store, messages.ImplicitMessages) + if err != nil { + return cid.Undef, err + } + + fullBlkHAMT, err := messages2.MakeFullBlockHAMT(ctx, store, messages.FullBlocks) + if err != nil { + return cid.Undef, err + } + + // TODO pass the adtStore not the blockstore. + actorStateContainer, err := cboractors.ProcessActorsStates(ctx, bs, actors) + if err != nil { + return cid.Undef, err + } + + actorStatesRoot, err := store.Put(ctx, actorStateContainer) + if err != nil { + return cid.Undef, err + } + + extractedState := &StateExtractionIPLD{ + Current: *current, + Parent: *executed, + BaseFee: messages.BaseFee, + FullBlocks: fullBlkHAMT, + ImplicitMessages: implicitMsgsAMT, + Actors: actorStatesRoot, + } + + extractedStateRoot, err := store.Put(ctx, extractedState) + if err != nil { + return cid.Undef, err + } + + rootState := &RootStateIPLD{ + StateVersion: 0, + State: extractedStateRoot, + } + + root, err := store.Put(ctx, rootState) + if err != nil { + return cid.Undef, err + } + return root, nil +} diff --git a/pkg/transform/cbor/types.go b/pkg/transform/cbor/types.go new file mode 100644 index 000000000..42893b414 --- /dev/null +++ b/pkg/transform/cbor/types.go @@ -0,0 +1 @@ +package cbor diff --git a/testutil/lens.go b/testutil/lens.go index b17428e94..cbf8d87a2 100644 --- a/testutil/lens.go +++ b/testutil/lens.go @@ -25,6 +25,11 @@ type APIWrapper struct { ctx context.Context } +func (aw *APIWrapper) GetMessageExecutionsForTipSetV2(ctx context.Context, ts, pts *types.TipSet) ([]*lens.MessageExecutionV2, error) { + //TODO implement me + panic("implement me") +} + func (aw *APIWrapper) BurnFundsFn(ctx context.Context, ts *types.TipSet) (lens.ShouldBurnFn, error) { //TODO implement me panic("implement me") From 0bd8b34def27e113ab456d47fa09de18afb0f4b0 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 18 Jan 2023 11:44:45 -0800 Subject: [PATCH 26/48] need to fix cborgen --- pkg/extract/actors/actordiff/state.go | 50 ++--- pkg/extract/actors/initdiff/v0/addresses.go | 8 +- pkg/extract/actors/initdiff/v0/state.go | 71 +++++-- pkg/extract/actors/marketdiff/v0/deals.go | 6 +- pkg/extract/actors/marketdiff/v0/state.go | 128 ++++++++--- pkg/extract/actors/minerdiff/v0/debt.go | 112 ---------- pkg/extract/actors/minerdiff/v0/funds.go | 141 ------------- pkg/extract/actors/minerdiff/v0/infos.go | 36 ---- pkg/extract/actors/minerdiff/v0/precommits.go | 33 +-- .../actors/minerdiff/v0/sector_status.go | 44 +--- pkg/extract/actors/minerdiff/v0/sectors.go | 28 +-- pkg/extract/actors/minerdiff/v0/state.go | 198 ++++++++---------- pkg/extract/actors/minerdiff/v2/debt.go | 6 - pkg/extract/actors/minerdiff/v2/funds.go | 6 - pkg/extract/actors/minerdiff/v2/state.go | 2 - pkg/extract/actors/minerdiff/v3/debt.go | 6 - pkg/extract/actors/minerdiff/v3/funds.go | 6 - pkg/extract/actors/minerdiff/v3/state.go | 2 - pkg/extract/actors/minerdiff/v4/debt.go | 6 - pkg/extract/actors/minerdiff/v4/funds.go | 6 - pkg/extract/actors/minerdiff/v4/state.go | 2 - pkg/extract/actors/minerdiff/v5/debt.go | 6 - pkg/extract/actors/minerdiff/v5/funds.go | 6 - pkg/extract/actors/minerdiff/v5/state.go | 2 - pkg/extract/actors/minerdiff/v6/debt.go | 6 - pkg/extract/actors/minerdiff/v6/funds.go | 6 - pkg/extract/actors/minerdiff/v6/state.go | 2 - pkg/extract/actors/minerdiff/v7/debt.go | 6 - pkg/extract/actors/minerdiff/v7/funds.go | 6 - pkg/extract/actors/minerdiff/v7/state.go | 2 - pkg/extract/actors/minerdiff/v8/debt.go | 6 - pkg/extract/actors/minerdiff/v8/funds.go | 6 - pkg/extract/actors/minerdiff/v8/state.go | 2 - pkg/extract/actors/minerdiff/v9/debt.go | 6 - pkg/extract/actors/minerdiff/v9/funds.go | 6 - pkg/extract/actors/minerdiff/v9/state.go | 2 - pkg/extract/actors/powerdiff/v0/claims.go | 6 +- pkg/extract/actors/powerdiff/v0/state.go | 65 ++++-- pkg/extract/actors/verifregdiff/v0/state.go | 85 ++++++-- pkg/gen/main.go | 3 +- pkg/transform/cbor/router.go | 2 +- pkg/transform/cbor/types.go | 1 - .../timescale/actors/miner/v0/processor.go | 2 +- 43 files changed, 371 insertions(+), 760 deletions(-) delete mode 100644 pkg/extract/actors/minerdiff/v0/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v0/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v2/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v2/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v3/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v3/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v4/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v4/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v5/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v5/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v6/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v6/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v7/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v7/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v8/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v8/funds.go delete mode 100644 pkg/extract/actors/minerdiff/v9/debt.go delete mode 100644 pkg/extract/actors/minerdiff/v9/funds.go delete mode 100644 pkg/transform/cbor/types.go diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/actordiff/state.go index 9eedc6e4d..203ebdd94 100644 --- a/pkg/extract/actors/actordiff/state.go +++ b/pkg/extract/actors/actordiff/state.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -12,38 +12,6 @@ import ( "github.com/filecoin-project/lily/tasks" ) -type StateDiffResult struct { - ActorStateChanges *ActorChange -} - -func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { - return s.ActorStateChanges, nil - out := &StateChange{} - - if actorChanges := s.ActorStateChanges; actorChanges != nil { - blk, err := actorChanges.ToStorageBlock() - if err != nil { - return nil, err - } - if err := bs.Put(ctx, blk); err != nil { - return nil, err - } - c := blk.Cid() - out.ActorState = c - } else { - return nil, nil - } - return out, nil -} - -func (s *StateDiffResult) Kind() string { - return "actor" -} - -type StateChange struct { - ActorState cid.Cid `cborgen:"actors"` -} - type StateDiff struct { DiffMethods []actors.ActorStateDiff } @@ -67,3 +35,19 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors log.Infow("Extracted Raw Actor State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } + +type StateDiffResult struct { + ActorStateChanges *ActorChange +} + +func (sdr *StateDiffResult) Kind() string { + return "actor" +} + +func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + return sdr.ActorStateChanges, nil +} + +type StateChange struct { + ActorState cid.Cid `cborgen:"actors"` +} diff --git a/pkg/extract/actors/initdiff/v0/addresses.go b/pkg/extract/actors/initdiff/v0/addresses.go index 7c4c05fe9..7885a2a09 100644 --- a/pkg/extract/actors/initdiff/v0/addresses.go +++ b/pkg/extract/actors/initdiff/v0/addresses.go @@ -28,10 +28,6 @@ type AddressChange struct { Change core.ChangeType `cborgen:"change"` } -func (t *AddressChange) Key() string { - return core.StringKey(t.Address).Key() -} - type AddressChangeList []*AddressChange const KindInitAddresses = "init_addresses" @@ -45,8 +41,8 @@ func (a AddressChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { if err != nil { return cid.Undef, err } - for _, a := range a { - if err := node.Put(a, a); err != nil { + for _, l := range a { + if err := node.Put(core.StringKey(l.Address), l); err != nil { return cid.Undef, err } } diff --git a/pkg/extract/actors/initdiff/v0/state.go b/pkg/extract/actors/initdiff/v0/state.go index d74d255f6..17b2e5e53 100644 --- a/pkg/extract/actors/initdiff/v0/state.go +++ b/pkg/extract/actors/initdiff/v0/state.go @@ -4,8 +4,8 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -13,6 +13,30 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindInitAddresses: + stateDiff.AddressesChanges = stateChange.(AddressChangeList) + } + } + log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} + type StateDiffResult struct { AddressesChanges AddressChangeList } @@ -21,44 +45,45 @@ func (s *StateDiffResult) Kind() string { return "init" } -func (s *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { +func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) - if addresses := s.AddressesChanges; addresses != nil { - root, err := addresses.ToAdtMap(adtStore, 5) + if addresses := sdr.AddressesChanges; addresses != nil { + root, err := addresses.ToAdtMap(s, 5) if err != nil { return nil, err } - out.Addresses = root + out.Addresses = &root } return out, nil } type StateChange struct { - Addresses cid.Cid `cborgen:"addresses"` + Addresses *cid.Cid `cborgen:"addresses"` } -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{AddressesChanges: AddressChangeList{}} -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() - var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) + if sc.Addresses != nil { + addressMap, err := adt.AsMap(s, *sc.Addresses, 5) if err != nil { return nil, err } - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindInitAddresses: - stateDiff.AddressesChanges = stateChange.(AddressChangeList) + + addresses := AddressChangeList{} + addressChange := new(AddressChange) + if err := addressMap.ForEach(addressChange, func(key string) error { + val := new(AddressChange) + *val = *addressChange + // NB: not required + // TODO consider removing they key from the structure. + val.Address = []byte(key) + addresses = append(addresses, val) + return nil + }); err != nil { + return nil, err } } - log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) - return stateDiff, nil + return out, nil } diff --git a/pkg/extract/actors/marketdiff/v0/deals.go b/pkg/extract/actors/marketdiff/v0/deals.go index 0540a9a89..f69b9c1d0 100644 --- a/pkg/extract/actors/marketdiff/v0/deals.go +++ b/pkg/extract/actors/marketdiff/v0/deals.go @@ -24,10 +24,6 @@ type DealChange struct { Change core.ChangeType `cborgen:"change"` } -func (t *DealChange) Key() string { - return abi.UIntKey(t.DealID).Key() -} - type DealChangeList []*DealChange const KindMarketDeal = "market_deal" @@ -42,7 +38,7 @@ func (p DealChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { return cid.Undef, err } for _, l := range p { - if err := node.Put(l, l); err != nil { + if err := node.Put(abi.UIntKey(l.DealID), l); err != nil { return cid.Undef, err } } diff --git a/pkg/extract/actors/marketdiff/v0/state.go b/pkg/extract/actors/marketdiff/v0/state.go index 4e270c0e7..e4b4f8679 100644 --- a/pkg/extract/actors/marketdiff/v0/state.go +++ b/pkg/extract/actors/marketdiff/v0/state.go @@ -4,8 +4,9 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -13,66 +14,129 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindMarketDeal: + stateDiff.DealStateChanges = stateChange.(DealChangeList) + case KindMarketProposal: + stateDiff.DealProposalChanges = stateChange.(ProposalChangeList) + default: + panic(stateChange.Kind()) + } + } + log.Infow("Extracted Market State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} + type StateDiffResult struct { DealStateChanges DealChangeList DealProposalChanges ProposalChangeList } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (cbg.CBORMarshaler, error) { +func (sd *StateDiffResult) Kind() string { + return "market" +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) if deals := sd.DealStateChanges; deals != nil { - root, err := deals.ToAdtMap(adtStore, 5) + root, err := deals.ToAdtMap(s, 5) if err != nil { return nil, err } - out.Deals = root + out.Deals = &root } if proposals := sd.DealProposalChanges; proposals != nil { - root, err := proposals.ToAdtMap(adtStore, 5) + root, err := proposals.ToAdtMap(s, 5) if err != nil { return nil, err } - out.Proposals = root + out.Proposals = &root } return out, nil } -func (sd *StateDiffResult) Kind() string { - return "market" -} - type StateChange struct { - Deals cid.Cid `cborgen:"deals"` - Proposals cid.Cid `cborgen:"proposals"` + Deals *cid.Cid `cborgen:"deals"` + Proposals *cid.Cid `cborgen:"proposals"` } -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + DealStateChanges: DealChangeList{}, + DealProposalChanges: ProposalChangeList{}, + } -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() - var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) + if sc.Deals != nil { + dealsMap, err := adt.AsMap(s, *sc.Deals, 5) if err != nil { return nil, err } - if stateChange == nil { - continue + deals := DealChangeList{} + dealChange := new(DealChange) + if err := dealsMap.ForEach(dealChange, func(key string) error { + val := new(DealChange) + *val = *dealChange + // NB: this is optinal since the dealChange already contains the dealID + // TODO consider removeing the key from the structure to save space + dealID, err := abi.ParseUIntKey(key) + if err != nil { + return err + } + if dealID != val.DealID { + panic("here") + } + val.DealID = dealID + deals = append(deals, val) + return nil + }); err != nil { + return nil, err } - switch stateChange.Kind() { - case KindMarketDeal: - stateDiff.DealStateChanges = stateChange.(DealChangeList) - case KindMarketProposal: - stateDiff.DealProposalChanges = stateChange.(ProposalChangeList) - default: - panic(stateChange.Kind()) + out.DealStateChanges = deals + } + + if sc.Proposals != nil { + propsMap, err := adt.AsMap(s, *sc.Proposals, 5) + if err != nil { + return nil, err + } + props := ProposalChangeList{} + propChange := new(ProposalChange) + if err := propsMap.ForEach(propChange, func(key string) error { + val := new(ProposalChange) + *val = *propChange + // NB: this is optinal since the dealChange already contains the dealID + // TODO consider removeing the key from the structure to save space + dealID, err := abi.ParseUIntKey(key) + if err != nil { + return err + } + if dealID != val.DealID { + panic("here") + } + val.DealID = dealID + props = append(props, val) + return nil + }); err != nil { + return nil, err } + out.DealProposalChanges = props } - log.Infow("Extracted Market State Diff", "address", act.Address, "duration", time.Since(start)) - return stateDiff, nil + return out, nil } diff --git a/pkg/extract/actors/minerdiff/v0/debt.go b/pkg/extract/actors/minerdiff/v0/debt.go deleted file mode 100644 index d02951d28..000000000 --- a/pkg/extract/actors/minerdiff/v0/debt.go +++ /dev/null @@ -1,112 +0,0 @@ -package v0 - -import ( - "bytes" - "context" - "time" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - block "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - "go.uber.org/zap" - - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" -) - -var _ actors.ActorStateChange = (*DebtChange)(nil) - -type DebtChange struct { - FeeDebt abi.TokenAmount `cborgen:"fee_debt"` - Change core.ChangeType `cborgen:"change"` -} - -func (d *DebtChange) Serialize() ([]byte, error) { - buf := new(bytes.Buffer) - if err := d.MarshalCBOR(buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -func (d *DebtChange) ToStorageBlock() (block.Block, error) { - data, err := d.Serialize() - if err != nil { - return nil, err - } - - c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) - if err != nil { - return nil, err - } - - return block.NewBlockWithCid(data, c) -} - -func DecodeDebt(b []byte) (*DebtChange, error) { - var debt DebtChange - if err := debt.UnmarshalCBOR(bytes.NewReader(b)); err != nil { - return nil, err - } - - return &debt, nil -} - -const KindMinerDebt = "miner_debt" - -func (d *DebtChange) Kind() actors.ActorStateKind { - return KindMinerDebt -} - -var _ actors.ActorStateDiff = (*Debt)(nil) - -type Debt struct{} - -func (Debt) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - start := time.Now() - defer func() { - log.Debugw("Diff", "kind", KindMinerDebt, zap.Inline(act), "duration", time.Since(start)) - }() - return DebtDiff(ctx, api, act) -} - -func DebtDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // was removed, its debt is gone... - // TODO is this correct? Can a miner be removed from the state who has debt? Would it be better to persist its last known debt value? the modified case below will have persisted that. - if act.Type == core.ChangeTypeRemove { - return &DebtChange{ - FeeDebt: big.Zero(), - Change: core.ChangeTypeRemove, - }, nil - } - currentMiner, err := api.MinerLoad(api.Store(), act.Current) - if err != nil { - return nil, err - } - currentDebt, err := currentMiner.FeeDebt() - if err != nil { - return nil, err - } - // added, all debt (assumed to be zero) is new - if act.Type == core.ChangeTypeAdd { - return &DebtChange{FeeDebt: currentDebt, Change: core.ChangeTypeAdd}, nil - } - // actor state was modified, check if debt differ. - - executedMiner, err := api.MinerLoad(api.Store(), act.Executed) - if err != nil { - return nil, err - } - executedDebt, err := executedMiner.FeeDebt() - if err != nil { - return nil, err - } - // no change if equal. - if executedDebt.Equals(currentDebt) { - return nil, nil - } - return &DebtChange{FeeDebt: currentDebt, Change: core.ChangeTypeModify}, nil -} diff --git a/pkg/extract/actors/minerdiff/v0/funds.go b/pkg/extract/actors/minerdiff/v0/funds.go deleted file mode 100644 index 7eb01165e..000000000 --- a/pkg/extract/actors/minerdiff/v0/funds.go +++ /dev/null @@ -1,141 +0,0 @@ -package v0 - -import ( - "bytes" - "context" - _ "embed" - "time" - - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/big" - block "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" - "go.uber.org/zap" - - "github.com/filecoin-project/lily/chain/actors/builtin/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" -) - -var _ actors.ActorStateChange = (*FundsChange)(nil) - -type FundsChange struct { - VestingFunds abi.TokenAmount `cborgen:"vesting_funds"` - InitialPledgeRequirement abi.TokenAmount `cborgen:"initial_pledge_requirement"` - PreCommitDeposit abi.TokenAmount `cborgen:"pre_commit_deposit"` - Change core.ChangeType `cborgen:"change"` -} - -func (f *FundsChange) Serialize() ([]byte, error) { - buf := new(bytes.Buffer) - if err := f.MarshalCBOR(buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -func (f *FundsChange) ToStorageBlock() (block.Block, error) { - data, err := f.Serialize() - if err != nil { - return nil, err - } - - c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) - if err != nil { - return nil, err - } - - return block.NewBlockWithCid(data, c) -} - -func DecodeFunds(b []byte) (*FundsChange, error) { - var funds FundsChange - if err := funds.UnmarshalCBOR(bytes.NewReader(b)); err != nil { - return nil, err - } - - return &funds, nil -} - -const KindMinerFunds = "miner_funds" - -func (f *FundsChange) Kind() actors.ActorStateKind { - return KindMinerFunds -} - -var _ actors.ActorStateDiff = (*Funds)(nil) - -type Funds struct{} - -func (Funds) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - start := time.Now() - defer func() { - log.Debugw("Diff", "kind", KindMinerFunds, zap.Inline(act), "duration", time.Since(start)) - }() - return FundsDiff(ctx, api, act) -} - -func FundsDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // was removed, no change - if act.Type == core.ChangeTypeRemove { - // TODO is this correct? Can a miner be removed from the state who still has funds? Would it be better to persist its last known funds value? the modified case below will have persisted that. - return &FundsChange{ - VestingFunds: big.Zero(), - InitialPledgeRequirement: big.Zero(), - PreCommitDeposit: big.Zero(), - Change: core.ChangeTypeRemove, - }, nil - } - currentMiner, err := api.MinerLoad(api.Store(), act.Current) - if err != nil { - return nil, err - } - currentFunds, err := currentMiner.LockedFunds() - if err != nil { - return nil, err - } - // added, all funds are new. - if act.Type == core.ChangeTypeAdd { - return &FundsChange{ - VestingFunds: currentFunds.VestingFunds, - InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, - PreCommitDeposit: currentFunds.PreCommitDeposits, - Change: core.ChangeTypeAdd, - }, nil - } - - executedMiner, err := api.MinerLoad(api.Store(), act.Executed) - if err != nil { - return nil, err - } - executedFunds, err := executedMiner.LockedFunds() - if err != nil { - return nil, err - } - // no change if equal - if LockedFundsEqual(currentFunds, executedFunds) { - return nil, nil - } - // funds differ - return &FundsChange{ - VestingFunds: currentFunds.VestingFunds, - InitialPledgeRequirement: currentFunds.InitialPledgeRequirement, - PreCommitDeposit: currentFunds.PreCommitDeposits, - Change: core.ChangeTypeModify, - }, nil -} - -func LockedFundsEqual(cur, pre miner.LockedFunds) bool { - if !cur.VestingFunds.Equals(pre.VestingFunds) { - return true - } - if !cur.PreCommitDeposits.Equals(pre.PreCommitDeposits) { - return true - } - if !cur.InitialPledgeRequirement.Equals(pre.InitialPledgeRequirement) { - return true - } - return false -} diff --git a/pkg/extract/actors/minerdiff/v0/infos.go b/pkg/extract/actors/minerdiff/v0/infos.go index 422335d6b..32406e226 100644 --- a/pkg/extract/actors/minerdiff/v0/infos.go +++ b/pkg/extract/actors/minerdiff/v0/infos.go @@ -1,14 +1,11 @@ package v0 import ( - "bytes" "context" "time" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - block "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -27,39 +24,6 @@ type InfoChange struct { Change core.ChangeType `cborgen:"change"` } -func (i *InfoChange) Serialize() ([]byte, error) { - buf := new(bytes.Buffer) - if err := i.MarshalCBOR(buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil - -} - -func (i *InfoChange) ToStorageBlock() (block.Block, error) { - data, err := i.Serialize() - if err != nil { - return nil, err - } - - c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) - if err != nil { - return nil, err - } - - return block.NewBlockWithCid(data, c) -} - -func DecodeInfo(b []byte) (*InfoChange, error) { - var info InfoChange - if err := info.UnmarshalCBOR(bytes.NewReader(b)); err != nil { - return nil, err - } - - return &info, nil -} - const KindMinerInfo = "miner_info" func (i *InfoChange) Kind() actors.ActorStateKind { diff --git a/pkg/extract/actors/minerdiff/v0/precommits.go b/pkg/extract/actors/minerdiff/v0/precommits.go index 610f6fb7b..d9277aa22 100644 --- a/pkg/extract/actors/minerdiff/v0/precommits.go +++ b/pkg/extract/actors/minerdiff/v0/precommits.go @@ -34,6 +34,12 @@ func (t *PreCommitChange) Key() string { type PreCommitChangeList []*PreCommitChange +const KindMinerPreCommit = "miner_precommit" + +func (p PreCommitChangeList) Kind() actors.ActorStateKind { + return KindMinerPreCommit +} + func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { node, err := adt.MakeEmptyMap(store, bw) if err != nil { @@ -47,33 +53,6 @@ func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) return node.Root() } -func (p *PreCommitChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { - precommitMap, err := adt.AsMap(store, root, bw) - if err != nil { - return err - } - - precommits := new(PreCommitChangeList) - precommitChange := new(PreCommitChange) - if err := precommitMap.ForEach(precommitChange, func(sectorNumber string) error { - val := new(PreCommitChange) - *val = *precommitChange - *precommits = append(*precommits, val) - return nil - }); err != nil { - return err - } - *p = *precommits - return nil - -} - -const KindMinerPreCommit = "miner_precommit" - -func (p PreCommitChangeList) Kind() actors.ActorStateKind { - return KindMinerPreCommit -} - type PreCommit struct{} func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/minerdiff/v0/sector_status.go b/pkg/extract/actors/minerdiff/v0/sector_status.go index 2533b6c7c..d62e2035f 100644 --- a/pkg/extract/actors/minerdiff/v0/sector_status.go +++ b/pkg/extract/actors/minerdiff/v0/sector_status.go @@ -1,15 +1,11 @@ package v0 import ( - "bytes" "context" "fmt" "time" "github.com/filecoin-project/go-bitfield" - "github.com/filecoin-project/go-state-types/abi" - block "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" "go.uber.org/zap" "golang.org/x/sync/errgroup" @@ -32,38 +28,6 @@ type SectorStatusChange struct { Recovered bitfield.BitField `cborgen:"recovered"` } -func (s *SectorStatusChange) Serialize() ([]byte, error) { - buf := new(bytes.Buffer) - if err := s.MarshalCBOR(buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -func (s *SectorStatusChange) ToStorageBlock() (block.Block, error) { - data, err := s.Serialize() - if err != nil { - return nil, err - } - - c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) - if err != nil { - return nil, err - } - - return block.NewBlockWithCid(data, c) -} - -func DecodeSectorStatus(b []byte) (*SectorStatusChange, error) { - var ss SectorStatusChange - if err := ss.UnmarshalCBOR(bytes.NewReader(b)); err != nil { - return nil, err - } - - return &ss, nil -} - const KindMinerSectorStatus = "miner_sector_status" func (s SectorStatusChange) Kind() actors.ActorStateKind { @@ -170,10 +134,10 @@ type SectorStates struct { // LoadSectorState loads all sectors from a miners partitions and returns a SectorStates structure containing individual // bitfields for all active, live, faulty and recovering sector. func LoadSectorState(ctx context.Context, state miner.State) (*SectorStates, error) { - activeSectors := []bitfield.BitField{} - liveSectors := []bitfield.BitField{} - faultySectors := []bitfield.BitField{} - recoveringSectors := []bitfield.BitField{} + var activeSectors []bitfield.BitField + var liveSectors []bitfield.BitField + var faultySectors []bitfield.BitField + var recoveringSectors []bitfield.BitField // iterate the sector states if err := state.ForEachDeadline(func(_ uint64, dl miner.Deadline) error { diff --git a/pkg/extract/actors/minerdiff/v0/sectors.go b/pkg/extract/actors/minerdiff/v0/sectors.go index 10bc1ad22..679441852 100644 --- a/pkg/extract/actors/minerdiff/v0/sectors.go +++ b/pkg/extract/actors/minerdiff/v0/sectors.go @@ -18,8 +18,6 @@ import ( var _ actors.ActorStateChange = (*SectorChangeList)(nil) -var _ abi.Keyer = (*SectorChange)(nil) - type SectorChange struct { SectorNumber uint64 `cborgen:"sector_number"` Current *typegen.Deferred `cborgen:"current_sector"` @@ -27,10 +25,6 @@ type SectorChange struct { Change core.ChangeType `cborgen:"change"` } -func (t *SectorChange) Key() string { - return abi.UIntKey(t.SectorNumber).Key() -} - type SectorChangeList []*SectorChange func (s SectorChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { @@ -39,33 +33,13 @@ func (s SectorChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { return cid.Undef, err } for _, l := range s { - if err := node.Put(l, l); err != nil { + if err := node.Put(abi.UIntKey(l.SectorNumber), l); err != nil { return cid.Undef, err } } return node.Root() } -func (s *SectorChangeList) FromAdtMap(store adt.Store, root cid.Cid, bw int) error { - sectorMap, err := adt.AsMap(store, root, bw) - if err != nil { - return err - } - - sectors := new(SectorChangeList) - sectorChange := new(SectorChange) - if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { - val := new(SectorChange) - *val = *sectorChange - *sectors = append(*sectors, val) - return nil - }); err != nil { - return err - } - *s = *sectors - return nil -} - const KindMinerSector = "miner_sector" func (s SectorChangeList) Kind() actors.ActorStateKind { diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v0/state.go index 754a41beb..ae4c9c76c 100644 --- a/pkg/extract/actors/minerdiff/v0/state.go +++ b/pkg/extract/actors/minerdiff/v0/state.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" @@ -77,157 +77,125 @@ func (s *StateDiffResult) Kind() string { return "miner" } -type StateChange struct { - // SectorStatus is the sectors whose status changed for this miner or empty. - SectorStatus *cid.Cid `cborgen:"sector_status"` - // Info is the cid of the miner change info that changed for this miner or empty. - Info *cid.Cid `cborgen:"info"` - // PreCommits is an HAMT of the pre commits that changed for this miner. - PreCommits cid.Cid `cborgen:"pre_commits"` - // Sectors is an HAMT of the sectors that changed for this miner. - Sectors cid.Cid `cborgen:"sectors"` -} - -func DecodeStateDiffResultFromStateChange(ctx context.Context, bs blockstore.Blockstore, sc *StateChange) (*StateDiffResult, error) { - out := &StateDiffResult{} - adtStore := store.WrapBlockStore(ctx, bs) +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { + out := &StateChange{} - // - // SectorChangeList - { - sectorChangeList := &SectorChangeList{} - if err := sectorChangeList.FromAdtMap(adtStore, sc.Sectors, 5); err != nil { + if sectors := sd.SectorChanges; sectors != nil { + root, err := sectors.ToAdtMap(s, 5) + if err != nil { return nil, err } - out.SectorChanges = *sectorChangeList - /* - sectorMap, err := adt.AsMap(adtStore, sc.Sectors, 5) - if err != nil { - return nil, err - } - - sectorChangeList := SectorChangeList{} - sectorChange := new(SectorChange) - if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { - val := new(SectorChange) - *val = *sectorChange - sectorChangeList = append(sectorChangeList, val) - return nil - }); err != nil { - return nil, err - } - out.SectorChanges = sectorChangeList - - */ + out.Sectors = &root } - // - // PrecommitChangeList - { - precommitChangeList := &PreCommitChangeList{} - if err := precommitChangeList.FromAdtMap(adtStore, sc.PreCommits, 5); err != nil { + if precommits := sd.PreCommitChanges; precommits != nil { + root, err := precommits.ToAdtMap(s, 5) + if err != nil { return nil, err } - out.PreCommitChanges = *precommitChangeList - /* - preCommitMap, err := adt.AsMap(adtStore, sc.PreCommits, 5) - if err != nil { - return nil, err - } - - preCommitChangeList := PreCommitChangeList{} - preCommitChange := new(PreCommitChange) - if err := preCommitMap.ForEach(preCommitChange, func(sectorNumber string) error { - val := new(PreCommitChange) - *val = *preCommitChange - preCommitChangeList = append(preCommitChangeList, val) - return nil - }); err != nil { - return nil, err - } - out.PreCommitChanges = preCommitChangeList - - */ + out.PreCommits = &root } - // - // Info - { - if sc.Info != nil { - blk, err := bs.Get(ctx, *sc.Info) - if err != nil { - return nil, err - } - info, err := DecodeInfo(blk.RawData()) - if err != nil { - return nil, err - } - out.InfoChange = info + if info := sd.InfoChange; info != nil { + c, err := s.Put(ctx, info) + if err != nil { + return nil, err } + out.Info = &c } - // - // SectorStatus - { - - if sc.SectorStatus != nil { - blk, err := bs.Get(ctx, *sc.SectorStatus) - if err != nil { - return nil, err - } - ss, err := DecodeSectorStatus(blk.RawData()) - if err != nil { - return nil, err - } - out.SectorStatusChanges = ss + if sectorstatus := sd.SectorStatusChanges; sectorstatus != nil { + c, err := s.Put(ctx, sectorstatus) + if err != nil { + return nil, err } + out.SectorStatus = &c } - return out, nil } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, bs blockstore.Blockstore) (typegen.CBORMarshaler, error) { - out := &StateChange{} - adtStore := store.WrapBlockStore(ctx, bs) +type StateChange struct { + // SectorStatus is the sectors whose status changed for this miner or empty. + SectorStatus *cid.Cid `cborgen:"sector_status"` + // Info is the cid of the miner change info that changed for this miner or empty. + Info *cid.Cid `cborgen:"info"` + // PreCommits is an HAMT of the pre commits that changed for this miner. + PreCommits *cid.Cid `cborgen:"pre_commits"` + // Sectors is an HAMT of the sectors that changed for this miner. + Sectors *cid.Cid `cborgen:"sectors"` +} - if sectors := sd.SectorChanges; sectors != nil { - root, err := sectors.ToAdtMap(adtStore, 5) +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + InfoChange: nil, + SectorStatusChanges: nil, + SectorChanges: SectorChangeList{}, + PreCommitChanges: PreCommitChangeList{}, + } + + // + // SectorChangeList + if sc.Sectors != nil { + sectorMap, err := adt.AsMap(s, *sc.Sectors, 5) if err != nil { return nil, err } - out.Sectors = root - } - if precommits := sd.PreCommitChanges; precommits != nil { - root, err := precommits.ToAdtMap(adtStore, 5) - if err != nil { + sectors := SectorChangeList{} + sectorChange := new(SectorChange) + if err := sectorMap.ForEach(sectorChange, func(sectorNumber string) error { + val := new(SectorChange) + *val = *sectorChange + sectors = append(sectors, val) + return nil + }); err != nil { return nil, err } - out.PreCommits = root + out.SectorChanges = sectors } - if info := sd.InfoChange; info != nil { - blk, err := info.ToStorageBlock() + // + // PrecommitChangeList + + if sc.PreCommits != nil { + precommitMap, err := adt.AsMap(s, *sc.PreCommits, 5) if err != nil { return nil, err } - if err := bs.Put(ctx, blk); err != nil { + + precommits := PreCommitChangeList{} + precommitChange := new(PreCommitChange) + if err := precommitMap.ForEach(precommitChange, func(sectorNumber string) error { + val := new(PreCommitChange) + *val = *precommitChange + precommits = append(precommits, val) + return nil + }); err != nil { return nil, err } - c := blk.Cid() - out.Info = &c + out.PreCommitChanges = precommits } - if sectorstatus := sd.SectorStatusChanges; sectorstatus != nil { - blk, err := sectorstatus.ToStorageBlock() - if err != nil { + // + // Info + if sc.Info != nil { + info := new(InfoChange) + if err := s.Get(ctx, *sc.Info, info); err != nil { return nil, err } - if err := bs.Put(ctx, blk); err != nil { + out.InfoChange = info + } + + // + // SectorStatus + + if sc.SectorStatus != nil { + status := new(SectorStatusChange) + if err := s.Get(ctx, *sc.SectorStatus, status); err != nil { return nil, err } - c := blk.Cid() - out.SectorStatus = &c + out.SectorStatusChanges = status } + return out, nil } diff --git a/pkg/extract/actors/minerdiff/v2/debt.go b/pkg/extract/actors/minerdiff/v2/debt.go deleted file mode 100644 index bbb3f4e83..000000000 --- a/pkg/extract/actors/minerdiff/v2/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v2/funds.go b/pkg/extract/actors/minerdiff/v2/funds.go deleted file mode 100644 index 81b65096a..000000000 --- a/pkg/extract/actors/minerdiff/v2/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v2/state.go b/pkg/extract/actors/minerdiff/v2/state.go index 9fe4be164..69c74dac0 100644 --- a/pkg/extract/actors/minerdiff/v2/state.go +++ b/pkg/extract/actors/minerdiff/v2/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v3/debt.go b/pkg/extract/actors/minerdiff/v3/debt.go deleted file mode 100644 index 03ba59b58..000000000 --- a/pkg/extract/actors/minerdiff/v3/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v3/funds.go b/pkg/extract/actors/minerdiff/v3/funds.go deleted file mode 100644 index 45fa6c116..000000000 --- a/pkg/extract/actors/minerdiff/v3/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v3/state.go b/pkg/extract/actors/minerdiff/v3/state.go index 75f47ca25..d3dd5a70b 100644 --- a/pkg/extract/actors/minerdiff/v3/state.go +++ b/pkg/extract/actors/minerdiff/v3/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v4/debt.go b/pkg/extract/actors/minerdiff/v4/debt.go deleted file mode 100644 index 8e8999f48..000000000 --- a/pkg/extract/actors/minerdiff/v4/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v4/funds.go b/pkg/extract/actors/minerdiff/v4/funds.go deleted file mode 100644 index 8fc0bb558..000000000 --- a/pkg/extract/actors/minerdiff/v4/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v4/state.go b/pkg/extract/actors/minerdiff/v4/state.go index 784c40dcd..ea583d7fe 100644 --- a/pkg/extract/actors/minerdiff/v4/state.go +++ b/pkg/extract/actors/minerdiff/v4/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v5/debt.go b/pkg/extract/actors/minerdiff/v5/debt.go deleted file mode 100644 index f7e7b6fd5..000000000 --- a/pkg/extract/actors/minerdiff/v5/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v5/funds.go b/pkg/extract/actors/minerdiff/v5/funds.go deleted file mode 100644 index 8380ed56d..000000000 --- a/pkg/extract/actors/minerdiff/v5/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v5/state.go b/pkg/extract/actors/minerdiff/v5/state.go index f5636c4ad..95b671e65 100644 --- a/pkg/extract/actors/minerdiff/v5/state.go +++ b/pkg/extract/actors/minerdiff/v5/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v6/debt.go b/pkg/extract/actors/minerdiff/v6/debt.go deleted file mode 100644 index 8b9d1e99a..000000000 --- a/pkg/extract/actors/minerdiff/v6/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v6/funds.go b/pkg/extract/actors/minerdiff/v6/funds.go deleted file mode 100644 index d3fa00306..000000000 --- a/pkg/extract/actors/minerdiff/v6/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v6/state.go b/pkg/extract/actors/minerdiff/v6/state.go index 3d7175e60..c3545513b 100644 --- a/pkg/extract/actors/minerdiff/v6/state.go +++ b/pkg/extract/actors/minerdiff/v6/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v7/debt.go b/pkg/extract/actors/minerdiff/v7/debt.go deleted file mode 100644 index 58f1f242d..000000000 --- a/pkg/extract/actors/minerdiff/v7/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v7/funds.go b/pkg/extract/actors/minerdiff/v7/funds.go deleted file mode 100644 index 7dc10c926..000000000 --- a/pkg/extract/actors/minerdiff/v7/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v7/state.go b/pkg/extract/actors/minerdiff/v7/state.go index 9b7332d07..dac14967c 100644 --- a/pkg/extract/actors/minerdiff/v7/state.go +++ b/pkg/extract/actors/minerdiff/v7/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v8/debt.go b/pkg/extract/actors/minerdiff/v8/debt.go deleted file mode 100644 index f16daccac..000000000 --- a/pkg/extract/actors/minerdiff/v8/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v8/funds.go b/pkg/extract/actors/minerdiff/v8/funds.go deleted file mode 100644 index e5f1d0189..000000000 --- a/pkg/extract/actors/minerdiff/v8/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v8/state.go b/pkg/extract/actors/minerdiff/v8/state.go index 1314b2693..d2a72308d 100644 --- a/pkg/extract/actors/minerdiff/v8/state.go +++ b/pkg/extract/actors/minerdiff/v8/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/minerdiff/v9/debt.go b/pkg/extract/actors/minerdiff/v9/debt.go deleted file mode 100644 index b286f512c..000000000 --- a/pkg/extract/actors/minerdiff/v9/debt.go +++ /dev/null @@ -1,6 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type DebtChange = v0.DebtChange -type Debt = v0.Debt diff --git a/pkg/extract/actors/minerdiff/v9/funds.go b/pkg/extract/actors/minerdiff/v9/funds.go deleted file mode 100644 index dd0f1c55e..000000000 --- a/pkg/extract/actors/minerdiff/v9/funds.go +++ /dev/null @@ -1,6 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type FundsChange = v0.FundsChange -type Funds = v0.Funds diff --git a/pkg/extract/actors/minerdiff/v9/state.go b/pkg/extract/actors/minerdiff/v9/state.go index 71b9f53b8..287cffc2c 100644 --- a/pkg/extract/actors/minerdiff/v9/state.go +++ b/pkg/extract/actors/minerdiff/v9/state.go @@ -5,5 +5,3 @@ import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" type StateDiffResult = v0.StateDiffResult type StateChange = v0.StateChange type StateDiff = v0.StateDiff - -var DecodeStateDiffResultFromStateChange = v0.DecodeStateDiffResultFromStateChange diff --git a/pkg/extract/actors/powerdiff/v0/claims.go b/pkg/extract/actors/powerdiff/v0/claims.go index 4bcabb75a..9c858452e 100644 --- a/pkg/extract/actors/powerdiff/v0/claims.go +++ b/pkg/extract/actors/powerdiff/v0/claims.go @@ -25,10 +25,6 @@ type ClaimsChange struct { Change core.ChangeType `cborgen:"change"` } -func (t *ClaimsChange) Key() string { - return core.StringKey(t.Miner).Key() -} - type ClaimsChangeList []*ClaimsChange const KindPowerClaims = "power_claims" @@ -43,7 +39,7 @@ func (p ClaimsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { return cid.Undef, err } for _, l := range p { - if err := node.Put(l, l); err != nil { + if err := node.Put(core.StringKey(l.Miner), l); err != nil { return cid.Undef, err } } diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v0/state.go index 3c9acdd04..ed559876d 100644 --- a/pkg/extract/actors/powerdiff/v0/state.go +++ b/pkg/extract/actors/powerdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" @@ -12,6 +13,32 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindPowerClaims: + stateDiff.ClaimsChanges = stateChange.(ClaimsChangeList) + default: + panic(stateChange.Kind()) + } + } + log.Infow("Extracted Power State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} + type StateDiffResult struct { ClaimsChanges ClaimsChangeList } @@ -23,7 +50,7 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store if err != nil { return nil, err } - out.Claims = root + out.Claims = &root } return out, nil } @@ -33,31 +60,29 @@ func (sd *StateDiffResult) Kind() string { } type StateChange struct { - Claims cid.Cid `cborgen:"claims"` + Claims *cid.Cid `cborgen:"claims"` } -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ClaimsChanges: ClaimsChangeList{}} -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() - var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) + if sc.Claims != nil { + claimMap, err := adt.AsMap(s, *sc.Claims, 5) if err != nil { return nil, err } - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindPowerClaims: - stateDiff.ClaimsChanges = stateChange.(ClaimsChangeList) - default: - panic(stateChange.Kind()) + + claims := ClaimsChangeList{} + claimChange := new(ClaimsChange) + if err := claimMap.ForEach(claimChange, func(key string) error { + val := new(ClaimsChange) + *val = *claimChange + claims = append(claims, val) + return nil + }); err != nil { + return nil, err } + out.ClaimsChanges = claims } - log.Infow("Extracted Power State Diff", "address", act.Address, "duration", time.Since(start)) - return stateDiff, nil + return out, nil } diff --git a/pkg/extract/actors/verifregdiff/v0/state.go b/pkg/extract/actors/verifregdiff/v0/state.go index 6676990e1..f17bd6881 100644 --- a/pkg/extract/actors/verifregdiff/v0/state.go +++ b/pkg/extract/actors/verifregdiff/v0/state.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" @@ -12,6 +13,32 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case KindVerifregClients: + stateDiff.ClientChanges = stateChange.(ClientsChangeList) + case KindVerifregVerifiers: + stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + } + } + log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} + type StateDiffResult struct { VerifierChanges VerifiersChangeList ClientChanges ClientsChangeList @@ -21,11 +48,6 @@ func (s *StateDiffResult) Kind() string { return "verifreg" } -type StateChange struct { - Verifiers *cid.Cid `cborgen:"verifiers"` - Clients *cid.Cid `cborgen:"clients"` -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { out := &StateChange{} @@ -47,28 +69,49 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store return out, nil } -type StateDiff struct { - DiffMethods []actors.ActorStateDiff +type StateChange struct { + Verifiers *cid.Cid `cborgen:"verifiers"` + Clients *cid.Cid `cborgen:"clients"` } -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() - var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + VerifierChanges: VerifiersChangeList{}, + ClientChanges: ClientsChangeList{}, + } + + if sc.Verifiers != nil { + verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) if err != nil { return nil, err } - if stateChange == nil { - continue + + verifierChange := new(VerifiersChange) + if err := verifierMap.ForEach(verifierChange, func(key string) error { + val := new(VerifiersChange) + *val = *verifierChange + out.VerifierChanges = append(out.VerifierChanges, val) + return nil + }); err != nil { + return nil, err } - switch stateChange.Kind() { - case KindVerifregClients: - stateDiff.ClientChanges = stateChange.(ClientsChangeList) - case KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + } + + if sc.Clients != nil { + clientsMap, err := adt.AsMap(s, *sc.Clients, 5) + if err != nil { + return nil, err + } + + clientChange := new(ClientsChange) + if err := clientsMap.ForEach(clientChange, func(key string) error { + val := new(ClientsChange) + *val = *clientChange + out.ClientChanges = append(out.ClientChanges, val) + return nil + }); err != nil { + return nil, err } } - log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) - return stateDiff, nil + return out, nil } diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 7e536bdc9..f906e0962 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -104,7 +104,7 @@ func main() { } if err := cbg.WriteMapEncodersToFile(IPLDActorContainerPath, IPLDActorContainerPkg, - actors.ActorIPLDContainer{}, + actors.ActorStateChangesIPLD{}, ); err != nil { panic(err) } @@ -122,7 +122,6 @@ func main() { } if err := cbg.WriteMapEncodersToFile(MessageContainerPath, MessageContainerPkg, - messages.MessageIPLDContainer{}, messages.FullBlockIPLDContainer{}, messages.ChainMessageIPLDContainer{}, messages.SignedChainMessageIPLDContainer{}, diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 5f7938d46..6ba113227 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -85,7 +85,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec } // TODO pass the adtStore not the blockstore. - actorStateContainer, err := cboractors.ProcessActorsStates(ctx, bs, actors) + actorStateContainer, err := cboractors.ProcessActorsStates(ctx, store, actors) if err != nil { return cid.Undef, err } diff --git a/pkg/transform/cbor/types.go b/pkg/transform/cbor/types.go deleted file mode 100644 index 42893b414..000000000 --- a/pkg/transform/cbor/types.go +++ /dev/null @@ -1 +0,0 @@ -package cbor diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go index bccb4c384..00b116033 100644 --- a/pkg/transform/timescale/actors/miner/v0/processor.go +++ b/pkg/transform/timescale/actors/miner/v0/processor.go @@ -61,7 +61,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, adtStore) if err != nil { return err } From f5952f66ed71170164f8c6c921631241b2a227a7 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 19 Jan 2023 12:12:28 -0800 Subject: [PATCH 27/48] compiles, need to cborgen --- pkg/extract/actors/initdiff/v0/cbor_gen.go | 83 --- pkg/extract/actors/marketdiff/v0/cbor_gen.go | 112 ---- pkg/extract/actors/minerdiff/v0/cbor_gen.go | 579 ------------------ pkg/extract/actors/powerdiff/v0/cbor_gen.go | 84 --- .../actors/verifregdiff/v0/cbor_gen.go | 113 ---- pkg/extract/actors/verifregdiff/version.go | 2 +- pkg/gen/main.go | 2 - pkg/transform/cbor/actors/cbor_gen.go | 345 ----------- pkg/transform/cbor/actors/serialize.go | 24 +- pkg/transform/cbor/messages/cbor_gen.go | 218 ------- pkg/transform/cbor/messages/serialize.go | 107 +++- pkg/transform/cbor/router.go | 44 +- .../timescale/actors/miner/router.go | 4 +- .../timescale/actors/miner/v0/processor.go | 8 +- .../timescale/actors/miner/v2/processor.go | 8 +- .../timescale/actors/miner/v3/processor.go | 8 +- .../timescale/actors/miner/v4/processor.go | 8 +- .../timescale/actors/miner/v5/processor.go | 8 +- .../timescale/actors/miner/v6/processor.go | 8 +- .../timescale/actors/miner/v7/processor.go | 8 +- .../timescale/actors/miner/v8/processor.go | 8 +- .../timescale/actors/miner/v9/processor.go | 8 +- pkg/transform/timescale/actors/processor.go | 71 ++- 23 files changed, 255 insertions(+), 1605 deletions(-) diff --git a/pkg/extract/actors/initdiff/v0/cbor_gen.go b/pkg/extract/actors/initdiff/v0/cbor_gen.go index 93611d92b..c4774a710 100644 --- a/pkg/extract/actors/initdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/initdiff/v0/cbor_gen.go @@ -211,93 +211,10 @@ func (t *AddressChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{161}); err != nil { - return err - } - - // t.Addresses (cid.Cid) (struct) - if len("addresses") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"addresses\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("addresses"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("addresses")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Addresses); err != nil { - return xerrors.Errorf("failed to write cid field t.Addresses: %w", err) - } - return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Addresses (cid.Cid) (struct) - case "addresses": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Addresses: %w", err) - } - - t.Addresses = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } return nil } diff --git a/pkg/extract/actors/marketdiff/v0/cbor_gen.go b/pkg/extract/actors/marketdiff/v0/cbor_gen.go index 5343e19c1..82eb747e1 100644 --- a/pkg/extract/actors/marketdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/marketdiff/v0/cbor_gen.go @@ -20,123 +20,11 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Deals (cid.Cid) (struct) - if len("deals") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"deals\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("deals"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("deals")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Deals); err != nil { - return xerrors.Errorf("failed to write cid field t.Deals: %w", err) - } - - // t.Proposals (cid.Cid) (struct) - if len("proposals") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"proposals\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("proposals"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("proposals")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Proposals); err != nil { - return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) - } return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Deals (cid.Cid) (struct) - case "deals": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Deals: %w", err) - } - - t.Deals = c - - } - // t.Proposals (cid.Cid) (struct) - case "proposals": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Proposals: %w", err) - } - - t.Proposals = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - return nil } func (t *ProposalChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/extract/actors/minerdiff/v0/cbor_gen.go b/pkg/extract/actors/minerdiff/v0/cbor_gen.go index e0b48e4b0..a6f58ec02 100644 --- a/pkg/extract/actors/minerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/v0/cbor_gen.go @@ -551,292 +551,6 @@ func (t *SectorChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } -func (t *FundsChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{164}); err != nil { - return err - } - - // t.VestingFunds (big.Int) (struct) - if len("vesting_funds") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"vesting_funds\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("vesting_funds"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("vesting_funds")); err != nil { - return err - } - - if err := t.VestingFunds.MarshalCBOR(cw); err != nil { - return err - } - - // t.InitialPledgeRequirement (big.Int) (struct) - if len("initial_pledge_requirement") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"initial_pledge_requirement\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("initial_pledge_requirement"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("initial_pledge_requirement")); err != nil { - return err - } - - if err := t.InitialPledgeRequirement.MarshalCBOR(cw); err != nil { - return err - } - - // t.PreCommitDeposit (big.Int) (struct) - if len("pre_commit_deposit") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"pre_commit_deposit\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commit_deposit"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("pre_commit_deposit")); err != nil { - return err - } - - if err := t.PreCommitDeposit.MarshalCBOR(cw); err != nil { - return err - } - - // t.Change (core.ChangeType) (uint8) - if len("change") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"change\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("change")); err != nil { - return err - } - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { - return err - } - return nil -} - -func (t *FundsChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = FundsChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("FundsChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.VestingFunds (big.Int) (struct) - case "vesting_funds": - - { - - if err := t.VestingFunds.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.VestingFunds: %w", err) - } - - } - // t.InitialPledgeRequirement (big.Int) (struct) - case "initial_pledge_requirement": - - { - - if err := t.InitialPledgeRequirement.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.InitialPledgeRequirement: %w", err) - } - - } - // t.PreCommitDeposit (big.Int) (struct) - case "pre_commit_deposit": - - { - - if err := t.PreCommitDeposit.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.PreCommitDeposit: %w", err) - } - - } - // t.Change (core.ChangeType) (uint8) - case "change": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint8 field") - } - if extra > math.MaxUint8 { - return fmt.Errorf("integer in input was too large for uint8 field") - } - t.Change = core.ChangeType(extra) - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} -func (t *DebtChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.FeeDebt (big.Int) (struct) - if len("fee_debt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"fee_debt\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fee_debt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("fee_debt")); err != nil { - return err - } - - if err := t.FeeDebt.MarshalCBOR(cw); err != nil { - return err - } - - // t.Change (core.ChangeType) (uint8) - if len("change") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"change\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("change")); err != nil { - return err - } - - if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { - return err - } - return nil -} - -func (t *DebtChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = DebtChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("DebtChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.FeeDebt (big.Int) (struct) - case "fee_debt": - - { - - if err := t.FeeDebt.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.FeeDebt: %w", err) - } - - } - // t.Change (core.ChangeType) (uint8) - case "change": - - maj, extra, err = cr.ReadHeader() - if err != nil { - return err - } - if maj != cbg.MajUnsignedInt { - return fmt.Errorf("wrong type for uint8 field") - } - if extra > math.MaxUint8 { - return fmt.Errorf("integer in input was too large for uint8 field") - } - t.Change = core.ChangeType(extra) - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} func (t *InfoChange) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) @@ -956,302 +670,9 @@ func (t *InfoChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{166}); err != nil { - return err - } - - // t.Funds (cid.Cid) (struct) - if len("funds") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"funds\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("funds"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("funds")); err != nil { - return err - } - - if t.Funds == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Funds); err != nil { - return xerrors.Errorf("failed to write cid field t.Funds: %w", err) - } - } - - // t.Debt (cid.Cid) (struct) - if len("debt") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"debt\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("debt"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("debt")); err != nil { - return err - } - - if t.Debt == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Debt); err != nil { - return xerrors.Errorf("failed to write cid field t.Debt: %w", err) - } - } - - // t.SectorStatus (cid.Cid) (struct) - if len("sector_status") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sector_status\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("sector_status")); err != nil { - return err - } - - if t.SectorStatus == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { - return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) - } - } - - // t.Info (cid.Cid) (struct) - if len("info") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"info\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("info")); err != nil { - return err - } - - if t.Info == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.Info); err != nil { - return xerrors.Errorf("failed to write cid field t.Info: %w", err) - } - } - - // t.PreCommits (cid.Cid) (struct) - if len("pre_commits") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"pre_commits\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("pre_commits")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.PreCommits); err != nil { - return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) - } - - // t.Sectors (cid.Cid) (struct) - if len("sectors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"sectors\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("sectors")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Sectors); err != nil { - return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) - } - return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Funds (cid.Cid) (struct) - case "funds": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Funds: %w", err) - } - - t.Funds = &c - } - - } - // t.Debt (cid.Cid) (struct) - case "debt": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Debt: %w", err) - } - - t.Debt = &c - } - - } - // t.SectorStatus (cid.Cid) (struct) - case "sector_status": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) - } - - t.SectorStatus = &c - } - - } - // t.Info (cid.Cid) (struct) - case "info": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Info: %w", err) - } - - t.Info = &c - } - - } - // t.PreCommits (cid.Cid) (struct) - case "pre_commits": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) - } - - t.PreCommits = c - - } - // t.Sectors (cid.Cid) (struct) - case "sectors": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) - } - - t.Sectors = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - return nil } diff --git a/pkg/extract/actors/powerdiff/v0/cbor_gen.go b/pkg/extract/actors/powerdiff/v0/cbor_gen.go index 1d1a4d7fa..6051372e3 100644 --- a/pkg/extract/actors/powerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/powerdiff/v0/cbor_gen.go @@ -20,94 +20,10 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{161}); err != nil { - return err - } - - // t.Claims (cid.Cid) (struct) - if len("claims") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"claims\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("claims")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Claims); err != nil { - return xerrors.Errorf("failed to write cid field t.Claims: %w", err) - } - return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Claims (cid.Cid) (struct) - case "claims": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Claims: %w", err) - } - - t.Claims = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - return nil } func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go index a2b46c898..cfcdbe94f 100644 --- a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go @@ -20,123 +20,10 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{162}); err != nil { - return err - } - - // t.Verifiers (cid.Cid) (struct) - if len("verifiers") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"verifiers\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("verifiers")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Verifiers); err != nil { - return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) - } - - // t.Clients (cid.Cid) (struct) - if len("clients") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"clients\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("clients"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("clients")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.Clients); err != nil { - return xerrors.Errorf("failed to write cid field t.Clients: %w", err) - } - return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { - *t = StateChange{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("StateChange: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.Verifiers (cid.Cid) (struct) - case "verifiers": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) - } - - t.Verifiers = c - - } - // t.Clients (cid.Cid) (struct) - case "clients": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.Clients: %w", err) - } - - t.Clients = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - return nil } func (t *ClientsChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go index fcaea63e1..c54a034c2 100644 --- a/pkg/extract/actors/verifregdiff/version.go +++ b/pkg/extract/actors/verifregdiff/version.go @@ -72,7 +72,7 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { DiffMethods: []actors.ActorStateDiff{ v9.Verifiers{}, v9.Claims{}, - // TODO + // TODO implement this //v9.Allocations{}, }}, nil case actortypes.Version10: diff --git a/pkg/gen/main.go b/pkg/gen/main.go index f906e0962..26dc684bf 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -57,8 +57,6 @@ func main() { minerV0.SectorStatusChange{}, minerV0.PreCommitChange{}, minerV0.SectorChange{}, - minerV0.FundsChange{}, - minerV0.DebtChange{}, minerV0.InfoChange{}, minerV0.StateChange{}, ); err != nil { diff --git a/pkg/transform/cbor/actors/cbor_gen.go b/pkg/transform/cbor/actors/cbor_gen.go index f1e2904eb..cdc4e982a 100644 --- a/pkg/transform/cbor/actors/cbor_gen.go +++ b/pkg/transform/cbor/actors/cbor_gen.go @@ -3,14 +3,10 @@ package actors import ( - "fmt" - "io" "math" "sort" - types "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" - cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -18,344 +14,3 @@ var _ = xerrors.Errorf var _ = cid.Undef var _ = math.E var _ = sort.Sort - -func (t *ActorIPLDContainer) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{168}); err != nil { - return err - } - - // t.CurrentTipSet (types.TipSet) (struct) - if len("current") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"current\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("current")); err != nil { - return err - } - - if err := t.CurrentTipSet.MarshalCBOR(cw); err != nil { - return err - } - - // t.ExecutedTipSet (types.TipSet) (struct) - if len("executed") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"executed\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("executed"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("executed")); err != nil { - return err - } - - if err := t.ExecutedTipSet.MarshalCBOR(cw); err != nil { - return err - } - - // t.MinerActors (cid.Cid) (struct) - if len("miners") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"miners\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miners"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("miners")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.MinerActors); err != nil { - return xerrors.Errorf("failed to write cid field t.MinerActors: %w", err) - } - - // t.VerifregActor (cid.Cid) (struct) - if len("verifreg") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"verifreg\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifreg"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("verifreg")); err != nil { - return err - } - - if t.VerifregActor == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.VerifregActor); err != nil { - return xerrors.Errorf("failed to write cid field t.VerifregActor: %w", err) - } - } - - // t.ActorStates (cid.Cid) (struct) - if len("actors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"actors\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("actors")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.ActorStates); err != nil { - return xerrors.Errorf("failed to write cid field t.ActorStates: %w", err) - } - - // t.InitActor (cid.Cid) (struct) - if len("init") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"init\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("init"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("init")); err != nil { - return err - } - - if t.InitActor == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if err := cbg.WriteCid(cw, *t.InitActor); err != nil { - return xerrors.Errorf("failed to write cid field t.InitActor: %w", err) - } - } - - // t.MarketActor (cid.Cid) (struct) - if len("market") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"market\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("market"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("market")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.MarketActor); err != nil { - return xerrors.Errorf("failed to write cid field t.MarketActor: %w", err) - } - - // t.PowerActor (cid.Cid) (struct) - if len("power") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"power\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("power"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("power")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.PowerActor); err != nil { - return xerrors.Errorf("failed to write cid field t.PowerActor: %w", err) - } - - return nil -} - -func (t *ActorIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { - *t = ActorIPLDContainer{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("ActorIPLDContainer: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.CurrentTipSet (types.TipSet) (struct) - case "current": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.CurrentTipSet = new(types.TipSet) - if err := t.CurrentTipSet.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.CurrentTipSet pointer: %w", err) - } - } - - } - // t.ExecutedTipSet (types.TipSet) (struct) - case "executed": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.ExecutedTipSet = new(types.TipSet) - if err := t.ExecutedTipSet.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.ExecutedTipSet pointer: %w", err) - } - } - - } - // t.MinerActors (cid.Cid) (struct) - case "miners": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.MinerActors: %w", err) - } - - t.MinerActors = c - - } - // t.VerifregActor (cid.Cid) (struct) - case "verifreg": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.VerifregActor: %w", err) - } - - t.VerifregActor = &c - } - - } - // t.ActorStates (cid.Cid) (struct) - case "actors": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ActorStates: %w", err) - } - - t.ActorStates = c - - } - // t.InitActor (cid.Cid) (struct) - case "init": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.InitActor: %w", err) - } - - t.InitActor = &c - } - - } - // t.MarketActor (cid.Cid) (struct) - case "market": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.MarketActor: %w", err) - } - - t.MarketActor = c - - } - // t.PowerActor (cid.Cid) (struct) - case "power": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.PowerActor: %w", err) - } - - t.PowerActor = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 9f0f150aa..a21a8acfd 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -26,9 +26,29 @@ type ActorStateChangesIPLD struct { } func (a *ActorStateChangesIPLD) Attributes() []attribute.KeyValue { - return []attribute.KeyValue{ - // TODO + var out []attribute.KeyValue + if a.DataCapActor != nil { + out = append(out, attribute.String("data_cap_root", a.DataCapActor.String())) } + if a.InitActor != nil { + out = append(out, attribute.String("init_root", a.InitActor.String())) + } + if a.MarketActor != nil { + out = append(out, attribute.String("market_root", a.MarketActor.String())) + } + if a.MinerActors != nil { + out = append(out, attribute.String("miner_root", a.MinerActors.String())) + } + if a.PowerActor != nil { + out = append(out, attribute.String("power_root", a.PowerActor.String())) + } + if a.RawActors != nil { + out = append(out, attribute.String("raw_root", a.RawActors.String())) + } + if a.VerifregActor != nil { + out = append(out, attribute.String("verifreg_root", a.VerifregActor.String())) + } + return out } func (a *ActorStateChangesIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { diff --git a/pkg/transform/cbor/messages/cbor_gen.go b/pkg/transform/cbor/messages/cbor_gen.go index c886fdde1..41f77bac1 100644 --- a/pkg/transform/cbor/messages/cbor_gen.go +++ b/pkg/transform/cbor/messages/cbor_gen.go @@ -20,224 +20,6 @@ var _ = cid.Undef var _ = math.E var _ = sort.Sort -func (t *MessageIPLDContainer) MarshalCBOR(w io.Writer) error { - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - - cw := cbg.NewCborWriter(w) - - if _, err := cw.Write([]byte{165}); err != nil { - return err - } - - // t.CurrentTipSet (types.TipSet) (struct) - if len("current") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"current\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("current")); err != nil { - return err - } - - if err := t.CurrentTipSet.MarshalCBOR(cw); err != nil { - return err - } - - // t.ExecutedTipSet (types.TipSet) (struct) - if len("executed") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"executed\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("executed"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("executed")); err != nil { - return err - } - - if err := t.ExecutedTipSet.MarshalCBOR(cw); err != nil { - return err - } - - // t.BaseFee (big.Int) (struct) - if len("base_fee") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"base_fee\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("base_fee"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("base_fee")); err != nil { - return err - } - - if err := t.BaseFee.MarshalCBOR(cw); err != nil { - return err - } - - // t.FullBlocks (cid.Cid) (struct) - if len("full_blocks") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"full_blocks\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("full_blocks"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("full_blocks")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.FullBlocks); err != nil { - return xerrors.Errorf("failed to write cid field t.FullBlocks: %w", err) - } - - // t.ImplicitMessages (cid.Cid) (struct) - if len("implicit_messages") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"implicit_messages\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("implicit_messages"))); err != nil { - return err - } - if _, err := io.WriteString(w, string("implicit_messages")); err != nil { - return err - } - - if err := cbg.WriteCid(cw, t.ImplicitMessages); err != nil { - return xerrors.Errorf("failed to write cid field t.ImplicitMessages: %w", err) - } - - return nil -} - -func (t *MessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { - *t = MessageIPLDContainer{} - - cr := cbg.NewCborReader(r) - - maj, extra, err := cr.ReadHeader() - if err != nil { - return err - } - defer func() { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } - }() - - if maj != cbg.MajMap { - return fmt.Errorf("cbor input should be of type map") - } - - if extra > cbg.MaxLength { - return fmt.Errorf("MessageIPLDContainer: map struct too large (%d)", extra) - } - - var name string - n := extra - - for i := uint64(0); i < n; i++ { - - { - sval, err := cbg.ReadString(cr) - if err != nil { - return err - } - - name = string(sval) - } - - switch name { - // t.CurrentTipSet (types.TipSet) (struct) - case "current": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.CurrentTipSet = new(types.TipSet) - if err := t.CurrentTipSet.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.CurrentTipSet pointer: %w", err) - } - } - - } - // t.ExecutedTipSet (types.TipSet) (struct) - case "executed": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.ExecutedTipSet = new(types.TipSet) - if err := t.ExecutedTipSet.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.ExecutedTipSet pointer: %w", err) - } - } - - } - // t.BaseFee (big.Int) (struct) - case "base_fee": - - { - - if err := t.BaseFee.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.BaseFee: %w", err) - } - - } - // t.FullBlocks (cid.Cid) (struct) - case "full_blocks": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.FullBlocks: %w", err) - } - - t.FullBlocks = c - - } - // t.ImplicitMessages (cid.Cid) (struct) - case "implicit_messages": - - { - - c, err := cbg.ReadCid(cr) - if err != nil { - return xerrors.Errorf("failed to read cid field t.ImplicitMessages: %w", err) - } - - t.ImplicitMessages = c - - } - - default: - // Field doesn't exist on this type, so ignore it - cbg.ScanForLinks(r, func(cid.Cid) {}) - } - } - - return nil -} func (t *FullBlockIPLDContainer) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go index 27844db74..a7362d538 100644 --- a/pkg/transform/cbor/messages/serialize.go +++ b/pkg/transform/cbor/messages/serialize.go @@ -47,6 +47,37 @@ func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[c return fullBlkHamt.Root() } +func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) (map[cid.Cid]*processor.FullBlock, error) { + fullBlkHamt, err := adt2.AsMap(store, root, 5) + if err != nil { + return nil, nil + } + out := make(map[cid.Cid]*processor.FullBlock) + fbc := new(FullBlockIPLDContainer) + if err := fullBlkHamt.ForEach(fbc, func(key string) error { + chainMessages, err := DecodeChainMessagesHAMT(ctx, store, fbc.BlsMessages) + if err != nil { + return err + } + signedChainMessages, err := DecodeSignedChainMessagesHAMT(ctx, store, fbc.SecpMessages) + if err != nil { + return err + } + bh := new(types.BlockHeader) + *bh = *fbc.BlockHeader + // TODO assert key == bh.Cid + out[bh.Cid()] = &processor.FullBlock{ + Block: bh, + SecpMessages: signedChainMessages, + BlsMessages: chainMessages, + } + return nil + }); err != nil { + return nil, err + } + return out, nil +} + type ChainMessageIPLDContainer struct { Message *types.Message `cborgen:"message"` Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` @@ -75,6 +106,30 @@ func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages [ return messageHamt.Root() } +func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ChainMessage, error) { + messagesHamt, err := adt2.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*processor.ChainMessage + mc := new(ChainMessageIPLDContainer) + if err := messagesHamt.ForEach(mc, func(key string) error { + val := new(processor.ChainMessage) + *val.Receipt = *mc.Receipt + *val.Message = *mc.Message + val.VmMessages, err = processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, val) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + type SignedChainMessageIPLDContainer struct { Message *types.SignedMessage `cborgen:"message"` Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` @@ -103,14 +158,38 @@ func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, mess return messageHamt.Root() } +func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.SignedChainMessage, error) { + messagesHamt, err := adt2.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*processor.SignedChainMessage + mc := new(SignedChainMessageIPLDContainer) + if err := messagesHamt.ForEach(mc, func(key string) error { + val := new(processor.SignedChainMessage) + *val.Receipt = *mc.Receipt + *val.Message = *mc.Message + val.VmMessages, err = processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, val) + return nil + }); err != nil { + return nil, err + } + return out, nil +} + type ImplicitMessageIPLDContainer struct { Message *types.Message `cborgen:"message"` Receipt *processor.ImplicitMessageReceipt `cborgen:"receipt"` VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -// MakeImplicitMessagesAMT returns the root of a hamt node containing the set of implicit messages -func MakeImplicitMessagesAMT(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { +// MakeImplicitMessagesHAMT returns the root of a hamt node containing the set of implicit messages +func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { messageHamt, err := adt2.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -131,3 +210,27 @@ func MakeImplicitMessagesAMT(ctx context.Context, store adtstore.Store, messages } return messageHamt.Root() } + +func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ImplicitMessage, error) { + messagesHamt, err := adt2.AsMap(store, root, 5) + if err != nil { + return nil, err + } + + var out []*processor.ImplicitMessage + msg := new(ImplicitMessageIPLDContainer) + if err := messagesHamt.ForEach(msg, func(key string) error { + val := new(processor.ImplicitMessage) + *val.Message = *msg.Message + *val.Receipt = *msg.Receipt + val.VmMessages, err = processor.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) + if err != nil { + return err + } + out = append(out, val) + return nil + }); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 6ba113227..7f74b0719 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -13,6 +13,8 @@ import ( logging "github.com/ipfs/go-log/v2" v1car "github.com/ipld/go-car" "github.com/ipld/go-car/util" + "go.opentelemetry.io/otel/attribute" + "go.uber.org/zap/zapcore" "github.com/filecoin-project/lily/pkg/extract/processor" cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" @@ -24,7 +26,26 @@ var log = logging.Logger("lily/transform/cbor") type RootStateIPLD struct { StateVersion uint64 - State cid.Cid + NetworkName string + NetworkVersion uint64 + + State cid.Cid // StateExtractionIPLD +} + +func (r *RootStateIPLD) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.Int64("state_version", int64(r.StateVersion)), + attribute.String("state_root", r.State.String()), + attribute.String("network_name", r.NetworkName), + attribute.Int64("network_version", int64(r.NetworkVersion)), + } +} + +func (r *RootStateIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range r.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil } type StateExtractionIPLD struct { @@ -38,6 +59,24 @@ type StateExtractionIPLD struct { Actors cid.Cid } +func (s *StateExtractionIPLD) Attributes() []attribute.KeyValue { + return []attribute.KeyValue{ + attribute.String("current_tipset", s.Current.Key().String()), + attribute.String("parent_tipset", s.Parent.Key().String()), + attribute.String("base_fee", s.BaseFee.String()), + attribute.String("full_block_root", s.FullBlocks.String()), + attribute.String("implicit_message_root", s.ImplicitMessages.String()), + attribute.String("actors_root", s.Actors.String()), + } +} + +func (s *StateExtractionIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { + for _, a := range s.Attributes() { + enc.AddString(string(a.Key), a.Value.Emit()) + } + return nil +} + func WriteCar(ctx context.Context, root cid.Cid, carVersion uint64, bs blockstore.Blockstore, w io.Writer) error { if err := v1car.WriteHeader(&v1car.CarHeader{ Roots: []cid.Cid{root}, @@ -74,7 +113,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec return cid.Undef, fmt.Errorf("actor and message executed tipset does not match") } - implicitMsgsAMT, err := messages2.MakeImplicitMessagesAMT(ctx, store, messages.ImplicitMessages) + implicitMsgsAMT, err := messages2.MakeImplicitMessagesHAMT(ctx, store, messages.ImplicitMessages) if err != nil { return cid.Undef, err } @@ -84,7 +123,6 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec return cid.Undef, err } - // TODO pass the adtStore not the blockstore. actorStateContainer, err := cboractors.ProcessActorsStates(ctx, store, actors) if err != nil { return cid.Undef, err diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index 3a134ff1f..64b346112 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -6,7 +6,7 @@ import ( "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" - "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -50,7 +50,7 @@ func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr addr } } -type MinerHandler = func(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) +type MinerHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) func MakeMinerProcessor(av actortypes.Version) (MinerHandler, error) { switch av { diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go index 00b116033..2e89a75f3 100644 --- a/pkg/transform/timescale/actors/miner/v0/processor.go +++ b/pkg/transform/timescale/actors/miner/v0/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, adtStore) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v2/processor.go b/pkg/transform/timescale/actors/miner/v2/processor.go index d3684d3ba..3a36f5925 100644 --- a/pkg/transform/timescale/actors/miner/v2/processor.go +++ b/pkg/transform/timescale/actors/miner/v2/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v3/processor.go b/pkg/transform/timescale/actors/miner/v3/processor.go index 4f411088c..7aec27b13 100644 --- a/pkg/transform/timescale/actors/miner/v3/processor.go +++ b/pkg/transform/timescale/actors/miner/v3/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v4/processor.go b/pkg/transform/timescale/actors/miner/v4/processor.go index 0c487b400..a3376a10a 100644 --- a/pkg/transform/timescale/actors/miner/v4/processor.go +++ b/pkg/transform/timescale/actors/miner/v4/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v5/processor.go b/pkg/transform/timescale/actors/miner/v5/processor.go index 716492b6d..c0c1239c9 100644 --- a/pkg/transform/timescale/actors/miner/v5/processor.go +++ b/pkg/transform/timescale/actors/miner/v5/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v6/processor.go b/pkg/transform/timescale/actors/miner/v6/processor.go index fb1c3155e..0275a99ae 100644 --- a/pkg/transform/timescale/actors/miner/v6/processor.go +++ b/pkg/transform/timescale/actors/miner/v6/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v7/processor.go b/pkg/transform/timescale/actors/miner/v7/processor.go index 06c69f294..a329c0734 100644 --- a/pkg/transform/timescale/actors/miner/v7/processor.go +++ b/pkg/transform/timescale/actors/miner/v7/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v8/processor.go b/pkg/transform/timescale/actors/miner/v8/processor.go index 35511ffb8..488234489 100644 --- a/pkg/transform/timescale/actors/miner/v8/processor.go +++ b/pkg/transform/timescale/actors/miner/v8/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/miner/v9/processor.go b/pkg/transform/timescale/actors/miner/v9/processor.go index a6683990a..1fa9ca84c 100644 --- a/pkg/transform/timescale/actors/miner/v9/processor.go +++ b/pkg/transform/timescale/actors/miner/v9/processor.go @@ -5,7 +5,6 @@ import ( "github.com/filecoin-project/go-address" store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" @@ -43,12 +42,11 @@ func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, add return ExtractMinerStateChanges(ctx, current, executed, addr, change) } -func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { +func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { out := model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) // a map of all miners whose state has changes - minerMap, err := adt.AsMap(adtStore, minerMapRoot, 5) + minerMap, err := adt.AsMap(s, minerMapRoot, 5) if err != nil { return nil, err } @@ -61,7 +59,7 @@ func MinerHandler(ctx context.Context, bs blockstore.Blockstore, current, execut return err } // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerdiff.DecodeStateDiffResultFromStateChange(ctx, bs, minerState) + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/actors/processor.go index 1009ae6e4..613d694da 100644 --- a/pkg/transform/timescale/actors/processor.go +++ b/pkg/transform/timescale/actors/processor.go @@ -21,7 +21,9 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" - cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/actors" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" @@ -43,42 +45,83 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe adtStore := store.WrapBlockStore(ctx, bs) - actorIPLDContainer := new(cboractors.ActorIPLDContainer) - if err := adtStore.Get(ctx, header.Roots[0], actorIPLDContainer); err != nil { + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := adtStore.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { return err } + log.Infow("open root", "car_root", header.Roots[0], zap.Inline(rootIPLDContainer)) - log.Infow("Open Delta", "root", header.Roots[0], zap.Inline(actorIPLDContainer)) + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := adtStore.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + log.Infow("open state extraction", zap.Inline(stateExtractionIPLDContainer)) + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent - current := actorIPLDContainer.CurrentTipSet - executed := actorIPLDContainer.ExecutedTipSet av, err := core.ActorVersionForTipSet(ctx, current, nvg) if err != nil { return err } - var models model.PersistableList - actorModels, err := ProcessActorStates(ctx, bs, current, executed, av, actorIPLDContainer.ActorStates) + _, err = HandleFullBlocks(ctx, adtStore, current, parent, stateExtractionIPLDContainer.FullBlocks) + if err != nil { + return err + } + + _, err = HandleImplicitMessages(ctx, adtStore, current, parent, stateExtractionIPLDContainer.ImplicitMessages) if err != nil { return err } - models = append(models, actorModels) - minerModels, err := ProcessMiners(ctx, bs, current, executed, av, actorIPLDContainer.MinerActors) + _, err = HandleActorStateChanges(ctx, adtStore, current, parent, av, stateExtractionIPLDContainer.Actors) if err != nil { return err } - models = append(models, minerModels) - return strg.PersistBatch(ctx, models...) + return nil } -func ProcessMiners(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { +func ProcessMiners(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { minerHandler, err := miner.MakeMinerProcessor(av) if err != nil { return nil, err } - return minerHandler(ctx, bs, current, executed, root) + return minerHandler(ctx, s, current, executed, root) +} + +func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (interface{}, error) { + actorIPLDContainer := new(actors.ActorStateChangesIPLD) + if err := s.Get(ctx, root, actorIPLDContainer); err != nil { + return nil, err + } + log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) + if actorIPLDContainer.MinerActors != nil { + minerModels, err := ProcessMiners(ctx, s, current, parent, av, *actorIPLDContainer.MinerActors) + if err != nil { + return nil, err + } + _ = minerModels + } + return nil, nil +} + +func HandleFullBlocks(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (interface{}, error) { + fullBlockMap, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return nil, err + } + _ = fullBlockMap + return nil, nil +} + +func HandleImplicitMessages(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (interface{}, error) { + implicitMessages, err := messages.DecodeImplicitMessagesHAMT(ctx, s, root) + if err != nil { + return nil, err + } + _ = implicitMessages + return nil, nil } func ProcessActorStates(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, av actorstypes.Version, actorMapRoot cid.Cid) (model.Persistable, error) { From 443cfd2f7936e575909388ba3136a780d0b78251 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 19 Jan 2023 12:34:17 -0800 Subject: [PATCH 28/48] can encode and decode car file, produces miner models --- pkg/extract/actors/initdiff/v0/cbor_gen.go | 99 ++++ pkg/extract/actors/marketdiff/v0/cbor_gen.go | 144 ++++++ pkg/extract/actors/minerdiff/v0/cbor_gen.go | 235 ++++++++++ pkg/extract/actors/powerdiff/v0/cbor_gen.go | 100 ++++ .../actors/verifregdiff/v0/cbor_gen.go | 145 ++++++ pkg/gen/main.go | 11 + pkg/transform/cbor/actors/cbor_gen.go | 381 +++++++++++++++ pkg/transform/cbor/cbor_gen.go | 434 ++++++++++++++++++ pkg/transform/cbor/messages/serialize.go | 59 ++- pkg/transform/cbor/router.go | 20 +- 10 files changed, 1593 insertions(+), 35 deletions(-) create mode 100644 pkg/transform/cbor/cbor_gen.go diff --git a/pkg/extract/actors/initdiff/v0/cbor_gen.go b/pkg/extract/actors/initdiff/v0/cbor_gen.go index c4774a710..a6c29c139 100644 --- a/pkg/extract/actors/initdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/initdiff/v0/cbor_gen.go @@ -211,10 +211,109 @@ func (t *AddressChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Addresses (cid.Cid) (struct) + if len("addresses") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"addresses\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("addresses"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("addresses")); err != nil { + return err + } + + if t.Addresses == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Addresses); err != nil { + return xerrors.Errorf("failed to write cid field t.Addresses: %w", err) + } + } + return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Addresses (cid.Cid) (struct) + case "addresses": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Addresses: %w", err) + } + + t.Addresses = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } return nil } diff --git a/pkg/extract/actors/marketdiff/v0/cbor_gen.go b/pkg/extract/actors/marketdiff/v0/cbor_gen.go index 82eb747e1..e11c76bfa 100644 --- a/pkg/extract/actors/marketdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/marketdiff/v0/cbor_gen.go @@ -20,11 +20,155 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Deals (cid.Cid) (struct) + if len("deals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"deals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("deals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("deals")); err != nil { + return err + } + + if t.Deals == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Deals); err != nil { + return xerrors.Errorf("failed to write cid field t.Deals: %w", err) + } + } + + // t.Proposals (cid.Cid) (struct) + if len("proposals") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"proposals\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("proposals"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("proposals")); err != nil { + return err + } + + if t.Proposals == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Proposals); err != nil { + return xerrors.Errorf("failed to write cid field t.Proposals: %w", err) + } + } return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Deals (cid.Cid) (struct) + case "deals": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Deals: %w", err) + } + + t.Deals = &c + } + + } + // t.Proposals (cid.Cid) (struct) + case "proposals": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Proposals: %w", err) + } + + t.Proposals = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + return nil } func (t *ProposalChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/extract/actors/minerdiff/v0/cbor_gen.go b/pkg/extract/actors/minerdiff/v0/cbor_gen.go index a6f58ec02..208a34a87 100644 --- a/pkg/extract/actors/minerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/v0/cbor_gen.go @@ -670,9 +670,244 @@ func (t *InfoChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.SectorStatus (cid.Cid) (struct) + if len("sector_status") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sector_status\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sector_status"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sector_status")); err != nil { + return err + } + + if t.SectorStatus == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.SectorStatus); err != nil { + return xerrors.Errorf("failed to write cid field t.SectorStatus: %w", err) + } + } + + // t.Info (cid.Cid) (struct) + if len("info") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"info\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("info"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("info")); err != nil { + return err + } + + if t.Info == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Info); err != nil { + return xerrors.Errorf("failed to write cid field t.Info: %w", err) + } + } + + // t.PreCommits (cid.Cid) (struct) + if len("pre_commits") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"pre_commits\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("pre_commits"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("pre_commits")); err != nil { + return err + } + + if t.PreCommits == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PreCommits); err != nil { + return xerrors.Errorf("failed to write cid field t.PreCommits: %w", err) + } + } + + // t.Sectors (cid.Cid) (struct) + if len("sectors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"sectors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sectors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("sectors")); err != nil { + return err + } + + if t.Sectors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Sectors); err != nil { + return xerrors.Errorf("failed to write cid field t.Sectors: %w", err) + } + } + return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.SectorStatus (cid.Cid) (struct) + case "sector_status": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.SectorStatus: %w", err) + } + + t.SectorStatus = &c + } + + } + // t.Info (cid.Cid) (struct) + case "info": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Info: %w", err) + } + + t.Info = &c + } + + } + // t.PreCommits (cid.Cid) (struct) + case "pre_commits": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PreCommits: %w", err) + } + + t.PreCommits = &c + } + + } + // t.Sectors (cid.Cid) (struct) + case "sectors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Sectors: %w", err) + } + + t.Sectors = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + return nil } diff --git a/pkg/extract/actors/powerdiff/v0/cbor_gen.go b/pkg/extract/actors/powerdiff/v0/cbor_gen.go index 6051372e3..f27bf776c 100644 --- a/pkg/extract/actors/powerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/powerdiff/v0/cbor_gen.go @@ -20,10 +20,110 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{161}); err != nil { + return err + } + + // t.Claims (cid.Cid) (struct) + if len("claims") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"claims\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("claims"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("claims")); err != nil { + return err + } + + if t.Claims == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Claims); err != nil { + return xerrors.Errorf("failed to write cid field t.Claims: %w", err) + } + } + return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Claims (cid.Cid) (struct) + case "claims": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Claims: %w", err) + } + + t.Claims = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + return nil } func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go index cfcdbe94f..714d557be 100644 --- a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v0/cbor_gen.go @@ -20,10 +20,155 @@ var _ = math.E var _ = sort.Sort func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Verifiers (cid.Cid) (struct) + if len("verifiers") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifiers\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifiers"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("verifiers")); err != nil { + return err + } + + if t.Verifiers == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Verifiers); err != nil { + return xerrors.Errorf("failed to write cid field t.Verifiers: %w", err) + } + } + + // t.Clients (cid.Cid) (struct) + if len("clients") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"clients\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("clients"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("clients")); err != nil { + return err + } + + if t.Clients == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Clients); err != nil { + return xerrors.Errorf("failed to write cid field t.Clients: %w", err) + } + } + return nil } func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Verifiers (cid.Cid) (struct) + case "verifiers": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Verifiers: %w", err) + } + + t.Verifiers = &c + } + + } + // t.Clients (cid.Cid) (struct) + case "clients": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Clients: %w", err) + } + + t.Clients = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + return nil } func (t *ClientsChange) MarshalCBOR(w io.Writer) error { diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 26dc684bf..51bcb8523 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -11,6 +11,7 @@ import ( verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/actors" "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) @@ -45,6 +46,9 @@ const MessageStatePkg = "processor" const MessageContainerPath = "pkg/transform/cbor/messages/cbor_gen.go" const MessageContainerPkg = "messages" +const RootStatePath = "pkg/transform/cbor/cbor_gen.go" +const RootStatePkg = "cbor" + func main() { if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, actordiff.ActorChange{}, @@ -127,4 +131,11 @@ func main() { ); err != nil { panic(err) } + + if err := cbg.WriteMapEncodersToFile(RootStatePath, RootStatePkg, + cbor.RootStateIPLD{}, + cbor.StateExtractionIPLD{}, + ); err != nil { + panic(err) + } } diff --git a/pkg/transform/cbor/actors/cbor_gen.go b/pkg/transform/cbor/actors/cbor_gen.go index cdc4e982a..a237d6ea5 100644 --- a/pkg/transform/cbor/actors/cbor_gen.go +++ b/pkg/transform/cbor/actors/cbor_gen.go @@ -3,10 +3,13 @@ package actors import ( + "fmt" + "io" "math" "sort" cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" ) @@ -14,3 +17,381 @@ var _ = xerrors.Errorf var _ = cid.Undef var _ = math.E var _ = sort.Sort + +func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{167}); err != nil { + return err + } + + // t.DataCapActor (cid.Cid) (struct) + if len("DataCapActor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"DataCapActor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DataCapActor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("DataCapActor")); err != nil { + return err + } + + if t.DataCapActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.DataCapActor); err != nil { + return xerrors.Errorf("failed to write cid field t.DataCapActor: %w", err) + } + } + + // t.InitActor (cid.Cid) (struct) + if len("InitActor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"InitActor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("InitActor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("InitActor")); err != nil { + return err + } + + if t.InitActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.InitActor); err != nil { + return xerrors.Errorf("failed to write cid field t.InitActor: %w", err) + } + } + + // t.MarketActor (cid.Cid) (struct) + if len("MarketActor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"MarketActor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MarketActor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("MarketActor")); err != nil { + return err + } + + if t.MarketActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.MarketActor); err != nil { + return xerrors.Errorf("failed to write cid field t.MarketActor: %w", err) + } + } + + // t.MinerActors (cid.Cid) (struct) + if len("MinerActors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"MinerActors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MinerActors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("MinerActors")); err != nil { + return err + } + + if t.MinerActors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.MinerActors); err != nil { + return xerrors.Errorf("failed to write cid field t.MinerActors: %w", err) + } + } + + // t.PowerActor (cid.Cid) (struct) + if len("PowerActor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"PowerActor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PowerActor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("PowerActor")); err != nil { + return err + } + + if t.PowerActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.PowerActor); err != nil { + return xerrors.Errorf("failed to write cid field t.PowerActor: %w", err) + } + } + + // t.RawActors (cid.Cid) (struct) + if len("RawActors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"RawActors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RawActors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("RawActors")); err != nil { + return err + } + + if t.RawActors == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.RawActors); err != nil { + return xerrors.Errorf("failed to write cid field t.RawActors: %w", err) + } + } + + // t.VerifregActor (cid.Cid) (struct) + if len("VerifregActor") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"VerifregActor\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("VerifregActor"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("VerifregActor")); err != nil { + return err + } + + if t.VerifregActor == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.VerifregActor); err != nil { + return xerrors.Errorf("failed to write cid field t.VerifregActor: %w", err) + } + } + + return nil +} + +func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = ActorStateChangesIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("ActorStateChangesIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.DataCapActor (cid.Cid) (struct) + case "DataCapActor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.DataCapActor: %w", err) + } + + t.DataCapActor = &c + } + + } + // t.InitActor (cid.Cid) (struct) + case "InitActor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.InitActor: %w", err) + } + + t.InitActor = &c + } + + } + // t.MarketActor (cid.Cid) (struct) + case "MarketActor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MarketActor: %w", err) + } + + t.MarketActor = &c + } + + } + // t.MinerActors (cid.Cid) (struct) + case "MinerActors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.MinerActors: %w", err) + } + + t.MinerActors = &c + } + + } + // t.PowerActor (cid.Cid) (struct) + case "PowerActor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.PowerActor: %w", err) + } + + t.PowerActor = &c + } + + } + // t.RawActors (cid.Cid) (struct) + case "RawActors": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.RawActors: %w", err) + } + + t.RawActors = &c + } + + } + // t.VerifregActor (cid.Cid) (struct) + case "VerifregActor": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.VerifregActor: %w", err) + } + + t.VerifregActor = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go new file mode 100644 index 000000000..90c7f3f9e --- /dev/null +++ b/pkg/transform/cbor/cbor_gen.go @@ -0,0 +1,434 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package cbor + +import ( + "fmt" + "io" + "math" + "sort" + + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *RootStateIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.StateVersion (uint64) (uint64) + if len("stateversion") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"stateversion\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("stateversion"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("stateversion")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.StateVersion)); err != nil { + return err + } + + // t.NetworkName (string) (string) + if len("networkname") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"networkname\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("networkname"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("networkname")); err != nil { + return err + } + + if len(t.NetworkName) > cbg.MaxLength { + return xerrors.Errorf("Value in field t.NetworkName was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.NetworkName))); err != nil { + return err + } + if _, err := io.WriteString(w, string(t.NetworkName)); err != nil { + return err + } + + // t.NetworkVersion (uint64) (uint64) + if len("networkversion") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"networkversion\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("networkversion"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("networkversion")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.NetworkVersion)); err != nil { + return err + } + + // t.State (cid.Cid) (struct) + if len("state") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"state\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("state"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("state")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.State); err != nil { + return xerrors.Errorf("failed to write cid field t.State: %w", err) + } + + return nil +} + +func (t *RootStateIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = RootStateIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("RootStateIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.StateVersion (uint64) (uint64) + case "stateversion": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.StateVersion = uint64(extra) + + } + // t.NetworkName (string) (string) + case "networkname": + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + t.NetworkName = string(sval) + } + // t.NetworkVersion (uint64) (uint64) + case "networkversion": + + { + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint64 field") + } + t.NetworkVersion = uint64(extra) + + } + // t.State (cid.Cid) (struct) + case "state": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.State: %w", err) + } + + t.State = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *StateExtractionIPLD) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{166}); err != nil { + return err + } + + // t.Current (types.TipSet) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Parent (types.TipSet) (struct) + if len("parent") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"parent\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("parent"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("parent")); err != nil { + return err + } + + if err := t.Parent.MarshalCBOR(cw); err != nil { + return err + } + + // t.BaseFee (big.Int) (struct) + if len("basefee") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"basefee\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("basefee"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("basefee")); err != nil { + return err + } + + if err := t.BaseFee.MarshalCBOR(cw); err != nil { + return err + } + + // t.FullBlocks (cid.Cid) (struct) + if len("fullblocks") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fullblocks\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fullblocks"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fullblocks")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.FullBlocks); err != nil { + return xerrors.Errorf("failed to write cid field t.FullBlocks: %w", err) + } + + // t.ImplicitMessages (cid.Cid) (struct) + if len("implicitmessages") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"implicitmessages\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("implicitmessages"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("implicitmessages")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.ImplicitMessages); err != nil { + return xerrors.Errorf("failed to write cid field t.ImplicitMessages: %w", err) + } + + // t.Actors (cid.Cid) (struct) + if len("actors") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"actors\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("actors"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("actors")); err != nil { + return err + } + + if err := cbg.WriteCid(cw, t.Actors); err != nil { + return xerrors.Errorf("failed to write cid field t.Actors: %w", err) + } + + return nil +} + +func (t *StateExtractionIPLD) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateExtractionIPLD{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateExtractionIPLD: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Current (types.TipSet) (struct) + case "current": + + { + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Current: %w", err) + } + + } + // t.Parent (types.TipSet) (struct) + case "parent": + + { + + if err := t.Parent.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Parent: %w", err) + } + + } + // t.BaseFee (big.Int) (struct) + case "basefee": + + { + + if err := t.BaseFee.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.BaseFee: %w", err) + } + + } + // t.FullBlocks (cid.Cid) (struct) + case "fullblocks": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.FullBlocks: %w", err) + } + + t.FullBlocks = c + + } + // t.ImplicitMessages (cid.Cid) (struct) + case "implicitmessages": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.ImplicitMessages: %w", err) + } + + t.ImplicitMessages = c + + } + // t.Actors (cid.Cid) (struct) + case "actors": + + { + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Actors: %w", err) + } + + t.Actors = c + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go index a7362d538..6c2747fbf 100644 --- a/pkg/transform/cbor/messages/serialize.go +++ b/pkg/transform/cbor/messages/serialize.go @@ -4,9 +4,9 @@ import ( "context" "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" adtstore "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/pkg/extract/processor" @@ -19,7 +19,7 @@ type FullBlockIPLDContainer struct { } func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[cid.Cid]*processor.FullBlock) (cid.Cid, error) { - fullBlkHamt, err := adt2.MakeEmptyMap(store, 5) + fullBlkHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err } @@ -48,9 +48,9 @@ func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[c } func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) (map[cid.Cid]*processor.FullBlock, error) { - fullBlkHamt, err := adt2.AsMap(store, root, 5) + fullBlkHamt, err := adt.AsMap(store, root, 5) if err != nil { - return nil, nil + return nil, err } out := make(map[cid.Cid]*processor.FullBlock) fbc := new(FullBlockIPLDContainer) @@ -85,7 +85,7 @@ type ChainMessageIPLDContainer struct { } func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { - messageHamt, err := adt2.MakeEmptyMap(store, 5) + messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err } @@ -107,7 +107,7 @@ func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages [ } func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ChainMessage, error) { - messagesHamt, err := adt2.AsMap(store, root, 5) + messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } @@ -115,14 +115,17 @@ func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid var out []*processor.ChainMessage mc := new(ChainMessageIPLDContainer) if err := messagesHamt.ForEach(mc, func(key string) error { - val := new(processor.ChainMessage) - *val.Receipt = *mc.Receipt - *val.Message = *mc.Message - val.VmMessages, err = processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + msgs := *mc.Message + rcpts := *mc.Receipt + vmMessages, err := processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, val) + out = append(out, &processor.ChainMessage{ + Message: &msgs, + Receipt: &rcpts, + VmMessages: vmMessages, + }) return nil }); err != nil { return nil, err @@ -137,7 +140,7 @@ type SignedChainMessageIPLDContainer struct { } func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { - messageHamt, err := adt2.MakeEmptyMap(store, 5) + messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err } @@ -159,7 +162,7 @@ func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, mess } func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.SignedChainMessage, error) { - messagesHamt, err := adt2.AsMap(store, root, 5) + messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } @@ -167,14 +170,17 @@ func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, ro var out []*processor.SignedChainMessage mc := new(SignedChainMessageIPLDContainer) if err := messagesHamt.ForEach(mc, func(key string) error { - val := new(processor.SignedChainMessage) - *val.Receipt = *mc.Receipt - *val.Message = *mc.Message - val.VmMessages, err = processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + msg := *mc.Message + recet := *mc.Receipt + vmMessages, err := processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, val) + out = append(out, &processor.SignedChainMessage{ + Message: &msg, + Receipt: &recet, + VmMessages: vmMessages, + }) return nil }); err != nil { return nil, err @@ -190,7 +196,7 @@ type ImplicitMessageIPLDContainer struct { // MakeImplicitMessagesHAMT returns the root of a hamt node containing the set of implicit messages func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { - messageHamt, err := adt2.MakeEmptyMap(store, 5) + messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err } @@ -212,7 +218,7 @@ func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, message } func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ImplicitMessage, error) { - messagesHamt, err := adt2.AsMap(store, root, 5) + messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } @@ -220,14 +226,17 @@ func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root var out []*processor.ImplicitMessage msg := new(ImplicitMessageIPLDContainer) if err := messagesHamt.ForEach(msg, func(key string) error { - val := new(processor.ImplicitMessage) - *val.Message = *msg.Message - *val.Receipt = *msg.Receipt - val.VmMessages, err = processor.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) + m := *msg.Message + rect := *msg.Receipt + vmMessages, err := processor.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, val) + out = append(out, &processor.ImplicitMessage{ + Message: &m, + Receipt: &rect, + VmMessages: vmMessages, + }) return nil }); err != nil { return nil, err diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 7f74b0719..ea2daaa8c 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -24,12 +24,12 @@ import ( var log = logging.Logger("lily/transform/cbor") type RootStateIPLD struct { - StateVersion uint64 + StateVersion uint64 `cborgen:"stateversion"` - NetworkName string - NetworkVersion uint64 + NetworkName string `cborgen:"networkname"` + NetworkVersion uint64 `cborgen:"networkversion"` - State cid.Cid // StateExtractionIPLD + State cid.Cid `cborgen:"state"` // StateExtractionIPLD } func (r *RootStateIPLD) Attributes() []attribute.KeyValue { @@ -49,14 +49,14 @@ func (r *RootStateIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error { } type StateExtractionIPLD struct { - Current types.TipSet - Parent types.TipSet + Current types.TipSet `cborgen:"current"` + Parent types.TipSet `cborgen:"parent"` - BaseFee abi.TokenAmount + BaseFee abi.TokenAmount `cborgen:"basefee"` - FullBlocks cid.Cid - ImplicitMessages cid.Cid - Actors cid.Cid + FullBlocks cid.Cid `cborgen:"fullblocks"` + ImplicitMessages cid.Cid `cborgen:"implicitmessages"` + Actors cid.Cid `cborgen:"actors"` } func (s *StateExtractionIPLD) Attributes() []attribute.KeyValue { From b57eb41bd9d280f5261f090e6034598b824b9bf1 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 19 Jan 2023 16:45:33 -0800 Subject: [PATCH 29/48] works --- lens/lily/impl.go | 4 +- .../actors/{actordiff => rawdiff}/actor.go | 2 +- .../actors/{actordiff => rawdiff}/cbor_gen.go | 2 +- .../actors/{actordiff => rawdiff}/state.go | 2 +- pkg/extract/actors/verifregdiff/v9/state.go | 71 +++++++---- pkg/extract/processor/actors.go | 6 +- pkg/extract/processor/messages.go | 93 +++++++-------- pkg/gen/main.go | 6 +- .../timescale/actors/init/addresses.go | 44 +++++++ .../timescale/actors/init/processor.go | 26 ++++ .../timescale/actors/market/router.go | 28 +++++ .../timescale/actors/market/util/label.go | 24 ++++ .../timescale/actors/market/v0/deals.go | 55 +++++++++ .../timescale/actors/market/v0/processor.go | 53 +++++++++ .../timescale/actors/market/v0/proposals.go | 68 +++++++++++ .../timescale/actors/market/v9/deals.go | 55 +++++++++ .../timescale/actors/market/v9/processor.go | 53 +++++++++ .../timescale/actors/market/v9/proposals.go | 92 +++++++++++++++ .../timescale/actors/miner/router.go | 4 +- .../timescale/actors/miner/v0/processor.go | 4 +- .../timescale/actors/miner/v0/state.go | 4 +- .../timescale/actors/miner/v2/processor.go | 4 +- .../timescale/actors/miner/v2/state.go | 4 +- .../timescale/actors/miner/v3/processor.go | 4 +- .../timescale/actors/miner/v3/state.go | 4 +- .../timescale/actors/miner/v4/processor.go | 4 +- .../timescale/actors/miner/v4/state.go | 4 +- .../timescale/actors/miner/v5/processor.go | 4 +- .../timescale/actors/miner/v5/state.go | 4 +- .../timescale/actors/miner/v6/processor.go | 4 +- .../timescale/actors/miner/v6/state.go | 4 +- .../timescale/actors/miner/v7/processor.go | 4 +- .../timescale/actors/miner/v7/state.go | 4 +- .../timescale/actors/miner/v8/processor.go | 4 +- .../timescale/actors/miner/v8/state.go | 4 +- .../timescale/actors/miner/v9/processor.go | 4 +- .../timescale/actors/miner/v9/state.go | 4 +- pkg/transform/timescale/actors/raw/state.go | 4 +- .../timescale/actors/reward/router.go | 4 +- .../timescale/actors/reward/v0/state.go | 4 +- .../timescale/actors/reward/v2/state.go | 4 +- .../timescale/actors/reward/v3/state.go | 4 +- .../timescale/actors/reward/v4/state.go | 4 +- .../timescale/actors/reward/v5/state.go | 4 +- .../timescale/actors/reward/v6/state.go | 4 +- .../timescale/actors/reward/v7/state.go | 4 +- .../timescale/actors/reward/v8/state.go | 4 +- .../timescale/actors/reward/v9/state.go | 4 +- pkg/transform/timescale/actors/types.go | 1 - .../timescale/actors/verifreg/router.go | 28 +++++ .../timescale/actors/verifreg/v0/clients.go | 62 ++++++++++ .../timescale/actors/verifreg/v0/processor.go | 53 +++++++++ .../timescale/actors/verifreg/v0/verifiers.go | 62 ++++++++++ .../actors/verifreg/v9/allocations.go | 16 +++ .../timescale/actors/verifreg/v9/claims.go | 16 +++ .../timescale/actors/verifreg/v9/processor.go | 54 +++++++++ .../timescale/actors/verifreg/v9/verifiers.go | 62 ++++++++++ pkg/transform/timescale/fullblock/blocks.go | 43 +++++++ pkg/transform/timescale/fullblock/messages.go | 93 +++++++++++++++ .../timescale/{actors => }/processor.go | 111 +++++++++++++++--- 60 files changed, 1250 insertions(+), 155 deletions(-) rename pkg/extract/actors/{actordiff => rawdiff}/actor.go (99%) rename pkg/extract/actors/{actordiff => rawdiff}/cbor_gen.go (99%) rename pkg/extract/actors/{actordiff => rawdiff}/state.go (98%) create mode 100644 pkg/transform/timescale/actors/init/addresses.go create mode 100644 pkg/transform/timescale/actors/init/processor.go create mode 100644 pkg/transform/timescale/actors/market/router.go create mode 100644 pkg/transform/timescale/actors/market/util/label.go create mode 100644 pkg/transform/timescale/actors/market/v0/deals.go create mode 100644 pkg/transform/timescale/actors/market/v0/processor.go create mode 100644 pkg/transform/timescale/actors/market/v0/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v9/deals.go create mode 100644 pkg/transform/timescale/actors/market/v9/processor.go create mode 100644 pkg/transform/timescale/actors/market/v9/proposals.go delete mode 100644 pkg/transform/timescale/actors/types.go create mode 100644 pkg/transform/timescale/actors/verifreg/router.go create mode 100644 pkg/transform/timescale/actors/verifreg/v0/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v0/processor.go create mode 100644 pkg/transform/timescale/actors/verifreg/v0/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v9/allocations.go create mode 100644 pkg/transform/timescale/actors/verifreg/v9/claims.go create mode 100644 pkg/transform/timescale/actors/verifreg/v9/processor.go create mode 100644 pkg/transform/timescale/actors/verifreg/v9/verifiers.go create mode 100644 pkg/transform/timescale/fullblock/blocks.go create mode 100644 pkg/transform/timescale/fullblock/messages.go rename pkg/transform/timescale/{actors => }/processor.go (56%) diff --git a/lens/lily/impl.go b/lens/lily/impl.go index e070f2de9..96bed2968 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -46,7 +46,7 @@ import ( "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract/processor" "github.com/filecoin-project/lily/pkg/transform/cbor" - "github.com/filecoin-project/lily/pkg/transform/timescale/actors" + "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" ) @@ -221,7 +221,7 @@ func (m *LilyNodeAPI) LilyIndexIPLDFile(_ context.Context, cfg *LilyIndexIPLDFil return false, err } - if err := actors.Process(ctx, f, strg, m.StateManager.GetNetworkVersion); err != nil { + if err := timescale.Process(ctx, f, strg, m.StateManager.GetNetworkVersion); err != nil { return false, err } diff --git a/pkg/extract/actors/actordiff/actor.go b/pkg/extract/actors/rawdiff/actor.go similarity index 99% rename from pkg/extract/actors/actordiff/actor.go rename to pkg/extract/actors/rawdiff/actor.go index 9801b640e..4b77610f7 100644 --- a/pkg/extract/actors/actordiff/actor.go +++ b/pkg/extract/actors/rawdiff/actor.go @@ -1,4 +1,4 @@ -package actordiff +package rawdiff import ( "bytes" diff --git a/pkg/extract/actors/actordiff/cbor_gen.go b/pkg/extract/actors/rawdiff/cbor_gen.go similarity index 99% rename from pkg/extract/actors/actordiff/cbor_gen.go rename to pkg/extract/actors/rawdiff/cbor_gen.go index 0ffe1f633..cc097b1bd 100644 --- a/pkg/extract/actors/actordiff/cbor_gen.go +++ b/pkg/extract/actors/rawdiff/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package actordiff +package rawdiff import ( "fmt" diff --git a/pkg/extract/actors/actordiff/state.go b/pkg/extract/actors/rawdiff/state.go similarity index 98% rename from pkg/extract/actors/actordiff/state.go rename to pkg/extract/actors/rawdiff/state.go index 203ebdd94..6c7c5b38e 100644 --- a/pkg/extract/actors/actordiff/state.go +++ b/pkg/extract/actors/rawdiff/state.go @@ -1,4 +1,4 @@ -package actordiff +package rawdiff import ( "context" diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index c35beed30..7a440c965 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -4,7 +4,8 @@ import ( "context" "time" - adtstore "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" @@ -17,6 +18,34 @@ import ( var log = logging.Logger("extract/actors/verifreg") +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + start := time.Now() + var stateDiff = new(StateDiffResult) + for _, f := range s.DiffMethods { + stateChange, err := f.Diff(ctx, api, act) + if err != nil { + return nil, err + } + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case v0.KindVerifregVerifiers: + stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + case KindVerifregClaims: + stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) + case KindVerifregAllocations: + stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) + } + } + log.Infow("Extracted Verified Registry State Diff", "address", act.Address, "duration", time.Since(start)) + return stateDiff, nil +} + type StateDiffResult struct { VerifierChanges v8.VerifiersChangeList ClaimChanges ClaimsChangeMap @@ -27,7 +56,7 @@ func (sd *StateDiffResult) Kind() string { return "verifreg" } -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store adtstore.Store) (cbg.CBORMarshaler, error) { +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} if verifiers := sd.VerifierChanges; verifiers != nil { @@ -62,30 +91,28 @@ type StateChange struct { Allocations *cid.Cid `cborgen:"allocations"` } -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + VerifierChanges: VerifiersChangeList{}, + } -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() - var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) + if sc.Verifiers != nil { + verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) if err != nil { return nil, err } - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case v0.KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) - case KindVerifregClaims: - stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) - case KindVerifregAllocations: - stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) + + verifierChange := new(VerifiersChange) + if err := verifierMap.ForEach(verifierChange, func(key string) error { + val := new(VerifiersChange) + *val = *verifierChange + out.VerifierChanges = append(out.VerifierChanges, val) + return nil + }); err != nil { + return nil, err } } - log.Infow("Extracted Verified Registry State Diff", "address", act.Address, "duration", time.Since(start)) - return stateDiff, nil + + panic("TODO") + return out, nil } diff --git a/pkg/extract/processor/actors.go b/pkg/extract/processor/actors.go index 2dbdf26c4..9a688c22c 100644 --- a/pkg/extract/processor/actors.go +++ b/pkg/extract/processor/actors.go @@ -15,11 +15,11 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" "github.com/filecoin-project/lily/pkg/extract/statetree" "github.com/filecoin-project/lily/tasks" @@ -89,8 +89,8 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. Type: change.ChangeType, } grp.Go(func() error { - actorDiff := &actordiff.StateDiff{ - DiffMethods: []actors.ActorStateDiff{actordiff.Actor{}}, + actorDiff := &rawdiff.StateDiff{ + DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, } actorStateChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { diff --git a/pkg/extract/processor/messages.go b/pkg/extract/processor/messages.go index 7ac16387b..a3f029fff 100644 --- a/pkg/extract/processor/messages.go +++ b/pkg/extract/processor/messages.go @@ -12,61 +12,62 @@ import ( "github.com/filecoin-project/lily/tasks" ) -type RootState struct { - // StateVersion contains the version of the State data structure. - // If changes are made to its structure, this version will increment. - StateVersion uint64 +/* + type RootState struct { + // StateVersion contains the version of the State data structure. + // If changes are made to its structure, this version will increment. + StateVersion uint64 - // State contains a single StateExtraction. - State cid.Cid -} - -type StateExtraction struct { - // Current and Parent are the TipSets whose state is diffed to produce Actors. - // BaseFee is calculated from Parent. - // FullBlocks contains the blocks and messages from Parent and receipts from Current. - // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. - // Actors contains actors whos state changed between Parent and Current. + // State contains a single StateExtraction. + State cid.Cid + } - // Current contains the tipset whose state root is the result of Parent's execution. - Current *types.TipSet - // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. - Parent *types.TipSet - // BaseFee contains the basefee during Parent's execution. - BaseFee abi.TokenAmount - // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. - FullBlocks map[cid.Cid]*FullBlock - // ImplicitMessages contains a list of all implicit messages executed at Parent. - ImplicitMessages []*ImplicitMessage - // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. - Actors ActorStateChanges -} + type StateExtraction struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. -type StateExtractionIPLD struct { - // Current and Parent are the TipSets whose state is diffed to produce Actors. - // BaseFee is calculated from Parent. - // FullBlocks contains the blocks and messages from Parent and the messages respective receipts from Current. - // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. - // Actors contains actors whos state changed between Parent and Current. + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks map[cid.Cid]*FullBlock + // ImplicitMessages contains a list of all implicit messages executed at Parent. + ImplicitMessages []*ImplicitMessage + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors ActorStateChanges + } - // Current contains the tipset whose state root is the result of Parent's execution. - Current *types.TipSet - // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. - Parent *types.TipSet + type StateExtractionIPLD struct { + // Current and Parent are the TipSets whose state is diffed to produce Actors. + // BaseFee is calculated from Parent. + // FullBlocks contains the blocks and messages from Parent and the messages respective receipts from Current. + // ImplicitMessages contains implicit messages applied in Parent. These messages and their respective receipts to no appear on chain as the name suggests. + // Actors contains actors whos state changed between Parent and Current. - // BaseFee contains the basefee during Parent's execution. - BaseFee abi.TokenAmount + // Current contains the tipset whose state root is the result of Parent's execution. + Current *types.TipSet + // Parent contains the parent tipset of Current. Execution of Parent's state produces Current's state root. + Parent *types.TipSet - // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. - FullBlocks cid.Cid // HAMT[BlockHeaderCID]FullBlock + // BaseFee contains the basefee during Parent's execution. + BaseFee abi.TokenAmount - // ImplicitMessages contains a map of all implicit messages executed at Parent. - ImplicitMessages cid.Cid // HAMT[MessageCID]ImplicitMessage + // FullBlocks contains a map of a BlockHeader CID to a FullBlock. Together these BlockHeaders form the Parent TipSet. + FullBlocks cid.Cid // HAMT[BlockHeaderCID]FullBlock - // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. - Actors cid.Cid // ActorStateChanges -} + // ImplicitMessages contains a map of all implicit messages executed at Parent. + ImplicitMessages cid.Cid // HAMT[MessageCID]ImplicitMessage + // Actors contains the actors whose state changed while executing Parent. Their current state is contained in Current's state root. + Actors cid.Cid // ActorStateChanges + } +*/ type MessageStateChanges struct { Current *types.TipSet Executed *types.TipSet diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 51bcb8523..a09759a0e 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -3,11 +3,11 @@ package main import ( cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" "github.com/filecoin-project/lily/pkg/extract/processor" @@ -51,8 +51,8 @@ const RootStatePkg = "cbor" func main() { if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, - actordiff.ActorChange{}, - actordiff.StateChange{}, + rawdiff.ActorChange{}, + rawdiff.StateChange{}, ); err != nil { panic(err) } diff --git a/pkg/transform/timescale/actors/init/addresses.go b/pkg/transform/timescale/actors/init/addresses.go new file mode 100644 index 000000000..dc8cbccf1 --- /dev/null +++ b/pkg/transform/timescale/actors/init/addresses.go @@ -0,0 +1,44 @@ +package init + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + cbg "github.com/whyrusleeping/cbor-gen" + + "github.com/filecoin-project/lily/model" + initmodel "github.com/filecoin-project/lily/model/actors/init" + "github.com/filecoin-project/lily/pkg/core" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" +) + +type Addresses struct{} + +func (a Addresses) Extract(ctx context.Context, current, executed *types.TipSet, changes *initdiff.StateDiffResult) (model.Persistable, error) { + out := initmodel.IDAddressList{} + for _, change := range changes.AddressesChanges { + if change.Change == core.ChangeTypeAdd || change.Change == core.ChangeTypeModify { + robustAddr, err := address.NewFromBytes(change.Address) + if err != nil { + return nil, err + } + var actorID cbg.CborInt + if err := actorID.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + idAddr, err := address.NewIDAddress(uint64(actorID)) + if err != nil { + return nil, err + } + out = append(out, &initmodel.IDAddress{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + ID: idAddr.String(), + Address: robustAddr.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/init/processor.go b/pkg/transform/timescale/actors/init/processor.go new file mode 100644 index 000000000..f92561aeb --- /dev/null +++ b/pkg/transform/timescale/actors/init/processor.go @@ -0,0 +1,26 @@ +package init + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" +) + +func InitHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, initMapRoot cid.Cid) (model.Persistable, error) { + initState := new(initdiff.StateChange) + if err := s.Get(ctx, initMapRoot, initState); err != nil { + return nil, err + } + + initStateDiff, err := initState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + return Addresses{}.Extract(ctx, current, executed, initStateDiff) +} diff --git a/pkg/transform/timescale/actors/market/router.go b/pkg/transform/timescale/actors/market/router.go new file mode 100644 index 000000000..0e7dd3b6b --- /dev/null +++ b/pkg/transform/timescale/actors/market/router.go @@ -0,0 +1,28 @@ +package market + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v0" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v9" +) + +type MarketHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) + +func MakeMarketProcessor(av actortypes.Version) (MarketHandler, error) { + switch av { + case actortypes.Version0: + return v0.MarketHandler, nil + case actortypes.Version9: + return v9.MarketHandler, nil + default: + return nil, fmt.Errorf("unsupported market actor version: %d", av) + } +} diff --git a/pkg/transform/timescale/actors/market/util/label.go b/pkg/transform/timescale/actors/market/util/label.go new file mode 100644 index 000000000..70c932fcf --- /dev/null +++ b/pkg/transform/timescale/actors/market/util/label.go @@ -0,0 +1,24 @@ +package util + +import ( + "strings" + "unicode/utf8" + + "golang.org/x/text/runes" +) + +// SanitizeLabel ensures: +// - s is a valid utf8 string by removing any ill formed bytes. +// - s does not contain any nil (\x00) bytes because postgres doesn't support storing NULL (\0x00) characters in text fields. +func SanitizeLabel(s string) string { + if s == "" { + return s + } + s = strings.Replace(s, "\000", "", -1) + if utf8.ValidString(s) { + return s + } + + tr := runes.ReplaceIllFormed() + return tr.String(s) +} diff --git a/pkg/transform/timescale/actors/market/v0/deals.go b/pkg/transform/timescale/actors/market/v0/deals.go new file mode 100644 index 000000000..24a4247ab --- /dev/null +++ b/pkg/transform/timescale/actors/market/v0/deals.go @@ -0,0 +1,55 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" +) + +type Deals struct{} + +func (Deals) Extract(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v0/processor.go b/pkg/transform/timescale/actors/market/v0/processor.go new file mode 100644 index 000000000..35b7dff0d --- /dev/null +++ b/pkg/transform/timescale/actors/market/v0/processor.go @@ -0,0 +1,53 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" +) + +type Extractor interface { + Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + Extractors []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.Extractors)) + for _, e := range se.Extractors { + m, err := e.Extract(ctx, current, executed, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MarketHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) { + marketState := new(marketdiff.StateChange) + if err := s.Get(ctx, marketRoot, marketState); err != nil { + return nil, err + } + marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + stateExtractor := &StateExtract{ + Extractors: []Extractor{ + Deals{}, + Proposals{}, + }, + } + return stateExtractor.Process(ctx, current, executed, marketStateDiff) +} diff --git a/pkg/transform/timescale/actors/market/v0/proposals.go b/pkg/transform/timescale/actors/market/v0/proposals.go new file mode 100644 index 000000000..6eaf67bbd --- /dev/null +++ b/pkg/transform/timescale/actors/market/v0/proposals.go @@ -0,0 +1,68 @@ +package v0 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v9/deals.go b/pkg/transform/timescale/actors/market/v9/deals.go new file mode 100644 index 000000000..aab7ecdb9 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v9/deals.go @@ -0,0 +1,55 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" +) + +type Deals struct{} + +func (Deals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range change.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v9/processor.go b/pkg/transform/timescale/actors/market/v9/processor.go new file mode 100644 index 000000000..7c367bf6e --- /dev/null +++ b/pkg/transform/timescale/actors/market/v9/processor.go @@ -0,0 +1,53 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" +) + +type Extractor interface { + Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + Extractors []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.Extractors)) + for _, e := range se.Extractors { + m, err := e.Extract(ctx, current, executed, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func MarketHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) { + marketState := new(marketdiff.StateChange) + if err := s.Get(ctx, marketRoot, marketState); err != nil { + return nil, err + } + marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + stateExtractor := &StateExtract{ + Extractors: []Extractor{ + Deals{}, + Proposals{}, + }, + } + return stateExtractor.Process(ctx, current, executed, marketStateDiff) +} diff --git a/pkg/transform/timescale/actors/market/v9/proposals.go b/pkg/transform/timescale/actors/market/v9/proposals.go new file mode 100644 index 000000000..66406124c --- /dev/null +++ b/pkg/transform/timescale/actors/market/v9/proposals.go @@ -0,0 +1,92 @@ +package v9 + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v9/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + var isString bool + var base64Label string + if prop.State.Label.IsString() { + labelString, err := prop.State.Label.ToString() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not a string despite claiming it is (developer error?)", prop.DealID) + } + + isString = true + base64Label = base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(labelString))) + + } else if prop.State.Label.IsBytes() { + labelBytes, err := prop.State.Label.ToBytes() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not bytes despit claiming it is (developer error?)", prop.DealID) + } + + isString = false + base64Label = base64.StdEncoding.EncodeToString(labelBytes) + + } else { + // TODO this should never happen, but if it does it indicates a logic. + return nil, fmt.Errorf("deal proposal (ID: %d) label is neither bytes nor string (DEVELOPER ERROR)", prop.DealID) + } + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: base64Label, + IsString: isString, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index 64b346112..5ecb4ca82 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v3" @@ -23,7 +23,7 @@ import ( v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v9" ) -func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange, version actortypes.Version) (model.Persistable, error) { +func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { switch version { case actortypes.Version0: return v0.MinerStateHandler(ctx, current, executed, addr, change) diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go index 2e89a75f3..1c9e68642 100644 --- a/pkg/transform/timescale/actors/miner/v0/processor.go +++ b/pkg/transform/timescale/actors/miner/v0/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v0/state.go b/pkg/transform/timescale/actors/miner/v0/state.go index 7a815f589..ea943db69 100644 --- a/pkg/transform/timescale/actors/miner/v0/state.go +++ b/pkg/transform/timescale/actors/miner/v0/state.go @@ -11,12 +11,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v2/processor.go b/pkg/transform/timescale/actors/miner/v2/processor.go index 3a36f5925..fe8216265 100644 --- a/pkg/transform/timescale/actors/miner/v2/processor.go +++ b/pkg/transform/timescale/actors/miner/v2/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v2/state.go b/pkg/transform/timescale/actors/miner/v2/state.go index f1f98bb18..955b26818 100644 --- a/pkg/transform/timescale/actors/miner/v2/state.go +++ b/pkg/transform/timescale/actors/miner/v2/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v3/processor.go b/pkg/transform/timescale/actors/miner/v3/processor.go index 7aec27b13..c3ef08d0c 100644 --- a/pkg/transform/timescale/actors/miner/v3/processor.go +++ b/pkg/transform/timescale/actors/miner/v3/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v3/state.go b/pkg/transform/timescale/actors/miner/v3/state.go index b790b8c68..ece3578da 100644 --- a/pkg/transform/timescale/actors/miner/v3/state.go +++ b/pkg/transform/timescale/actors/miner/v3/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v4/processor.go b/pkg/transform/timescale/actors/miner/v4/processor.go index a3376a10a..785aa5ddc 100644 --- a/pkg/transform/timescale/actors/miner/v4/processor.go +++ b/pkg/transform/timescale/actors/miner/v4/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v4/state.go b/pkg/transform/timescale/actors/miner/v4/state.go index deb1c9004..24524696e 100644 --- a/pkg/transform/timescale/actors/miner/v4/state.go +++ b/pkg/transform/timescale/actors/miner/v4/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v5/processor.go b/pkg/transform/timescale/actors/miner/v5/processor.go index c0c1239c9..08d167e12 100644 --- a/pkg/transform/timescale/actors/miner/v5/processor.go +++ b/pkg/transform/timescale/actors/miner/v5/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v5/state.go b/pkg/transform/timescale/actors/miner/v5/state.go index 156348997..d539cf6ba 100644 --- a/pkg/transform/timescale/actors/miner/v5/state.go +++ b/pkg/transform/timescale/actors/miner/v5/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v6/processor.go b/pkg/transform/timescale/actors/miner/v6/processor.go index 0275a99ae..c2816bc17 100644 --- a/pkg/transform/timescale/actors/miner/v6/processor.go +++ b/pkg/transform/timescale/actors/miner/v6/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v6/state.go b/pkg/transform/timescale/actors/miner/v6/state.go index fa7d57f20..45aba28c8 100644 --- a/pkg/transform/timescale/actors/miner/v6/state.go +++ b/pkg/transform/timescale/actors/miner/v6/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v7/processor.go b/pkg/transform/timescale/actors/miner/v7/processor.go index a329c0734..233cf3f1d 100644 --- a/pkg/transform/timescale/actors/miner/v7/processor.go +++ b/pkg/transform/timescale/actors/miner/v7/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v7/state.go b/pkg/transform/timescale/actors/miner/v7/state.go index c54b8bfc2..16541595d 100644 --- a/pkg/transform/timescale/actors/miner/v7/state.go +++ b/pkg/transform/timescale/actors/miner/v7/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v8/processor.go b/pkg/transform/timescale/actors/miner/v8/processor.go index 488234489..fe26bfc0b 100644 --- a/pkg/transform/timescale/actors/miner/v8/processor.go +++ b/pkg/transform/timescale/actors/miner/v8/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v8/state.go b/pkg/transform/timescale/actors/miner/v8/state.go index 21cfbfe6c..0f1ad68d9 100644 --- a/pkg/transform/timescale/actors/miner/v8/state.go +++ b/pkg/transform/timescale/actors/miner/v8/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/miner/v9/processor.go b/pkg/transform/timescale/actors/miner/v9/processor.go index 1fa9ca84c..6d1ba61d4 100644 --- a/pkg/transform/timescale/actors/miner/v9/processor.go +++ b/pkg/transform/timescale/actors/miner/v9/processor.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" ) @@ -38,7 +38,7 @@ func (se *StateExtract) Process(ctx context.Context, current, executed *types.Ti return out, nil } -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { return ExtractMinerStateChanges(ctx, current, executed, addr, change) } diff --git a/pkg/transform/timescale/actors/miner/v9/state.go b/pkg/transform/timescale/actors/miner/v9/state.go index 96fc33311..fbc5e6df3 100644 --- a/pkg/transform/timescale/actors/miner/v9/state.go +++ b/pkg/transform/timescale/actors/miner/v9/state.go @@ -10,12 +10,12 @@ import ( "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { return nil, nil } diff --git a/pkg/transform/timescale/actors/raw/state.go b/pkg/transform/timescale/actors/raw/state.go index 3c2073f5c..e78e8cd3f 100644 --- a/pkg/transform/timescale/actors/raw/state.go +++ b/pkg/transform/timescale/actors/raw/state.go @@ -12,10 +12,10 @@ import ( "github.com/filecoin-project/lily/model" commonmodel "github.com/filecoin-project/lily/model/actors/common" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RawActorHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RawActorHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { // noop when actor is removed if change.Change == core.ChangeTypeRemove { return nil, nil diff --git a/pkg/transform/timescale/actors/reward/router.go b/pkg/transform/timescale/actors/reward/router.go index ef6abf967..a17a9e2f7 100644 --- a/pkg/transform/timescale/actors/reward/router.go +++ b/pkg/transform/timescale/actors/reward/router.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v0" v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v2" v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v3" @@ -21,7 +21,7 @@ import ( v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v9" ) -func HandleReward(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange, version actortypes.Version) (model.Persistable, error) { +func HandleReward(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { switch version { case actortypes.Version0: return v0.RewardHandler(ctx, current, executed, addr, change) diff --git a/pkg/transform/timescale/actors/reward/v0/state.go b/pkg/transform/timescale/actors/reward/v0/state.go index b3d78b0c9..1030c7b2a 100644 --- a/pkg/transform/timescale/actors/reward/v0/state.go +++ b/pkg/transform/timescale/actors/reward/v0/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v2/state.go b/pkg/transform/timescale/actors/reward/v2/state.go index 93f251fb8..fc4bd7968 100644 --- a/pkg/transform/timescale/actors/reward/v2/state.go +++ b/pkg/transform/timescale/actors/reward/v2/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v3/state.go b/pkg/transform/timescale/actors/reward/v3/state.go index 6f383b8e4..ac9de4e7e 100644 --- a/pkg/transform/timescale/actors/reward/v3/state.go +++ b/pkg/transform/timescale/actors/reward/v3/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v4/state.go b/pkg/transform/timescale/actors/reward/v4/state.go index f6f36a988..2240e9d5b 100644 --- a/pkg/transform/timescale/actors/reward/v4/state.go +++ b/pkg/transform/timescale/actors/reward/v4/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v5/state.go b/pkg/transform/timescale/actors/reward/v5/state.go index d4610446f..5d53eefc3 100644 --- a/pkg/transform/timescale/actors/reward/v5/state.go +++ b/pkg/transform/timescale/actors/reward/v5/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v6/state.go b/pkg/transform/timescale/actors/reward/v6/state.go index ca9c0ed1f..2c8b693b3 100644 --- a/pkg/transform/timescale/actors/reward/v6/state.go +++ b/pkg/transform/timescale/actors/reward/v6/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v7/state.go b/pkg/transform/timescale/actors/reward/v7/state.go index e6707cab1..4573d90b7 100644 --- a/pkg/transform/timescale/actors/reward/v7/state.go +++ b/pkg/transform/timescale/actors/reward/v7/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v8/state.go b/pkg/transform/timescale/actors/reward/v8/state.go index ae8963f2f..c928f5c14 100644 --- a/pkg/transform/timescale/actors/reward/v8/state.go +++ b/pkg/transform/timescale/actors/reward/v8/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/reward/v9/state.go b/pkg/transform/timescale/actors/reward/v9/state.go index 28aa84c3c..0d0d8990f 100644 --- a/pkg/transform/timescale/actors/reward/v9/state.go +++ b/pkg/transform/timescale/actors/reward/v9/state.go @@ -11,10 +11,10 @@ import ( "github.com/filecoin-project/lily/model" rewardmodel "github.com/filecoin-project/lily/model/actors/reward" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" ) -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *actordiff.ActorChange) (model.Persistable, error) { +func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { if change.Change == core.ChangeTypeRemove { panic("reward is a singleton actor and cannot be removed") } diff --git a/pkg/transform/timescale/actors/types.go b/pkg/transform/timescale/actors/types.go deleted file mode 100644 index 1256222bc..000000000 --- a/pkg/transform/timescale/actors/types.go +++ /dev/null @@ -1 +0,0 @@ -package actors diff --git a/pkg/transform/timescale/actors/verifreg/router.go b/pkg/transform/timescale/actors/verifreg/router.go new file mode 100644 index 000000000..95e0f54cc --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/router.go @@ -0,0 +1,28 @@ +package verifreg + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v0" + v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v9" +) + +type VerifregHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) + +func MakeVerifregProcessor(av actortypes.Version) (VerifregHandler, error) { + switch av { + case actortypes.Version0: + return v0.VerifregHandler, nil + case actortypes.Version9: + return v9.VerifregHandler, nil + default: + return nil, fmt.Errorf("unsupported verifreg actor version: %d", av) + } +} diff --git a/pkg/transform/timescale/actors/verifreg/v0/clients.go b/pkg/transform/timescale/actors/verifreg/v0/clients.go new file mode 100644 index 000000000..655594c0d --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v0/clients.go @@ -0,0 +1,62 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +) + +type Clients struct{} + +func (Clients) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v0/processor.go b/pkg/transform/timescale/actors/verifreg/v0/processor.go new file mode 100644 index 000000000..dae0d545f --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v0/processor.go @@ -0,0 +1,53 @@ +package v0 + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +) + +type Extractor interface { + Extract(ctx context.Context, current, executed *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + Extractors []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.Extractors)) + for _, e := range se.Extractors { + m, err := e.Extract(ctx, current, executed, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func VerifregHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) { + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, verifregRoot, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + stateExtractor := &StateExtract{ + Extractors: []Extractor{ + Clients{}, + Verifiers{}, + }, + } + return stateExtractor.Process(ctx, current, executed, verifrefStateDiff) +} diff --git a/pkg/transform/timescale/actors/verifreg/v0/verifiers.go b/pkg/transform/timescale/actors/verifreg/v0/verifiers.go new file mode 100644 index 000000000..9520f3595 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v0/verifiers.go @@ -0,0 +1,62 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +) + +type Verifiers struct{} + +func (Verifiers) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v9/allocations.go b/pkg/transform/timescale/actors/verifreg/v9/allocations.go new file mode 100644 index 000000000..39bc88671 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v9/allocations.go @@ -0,0 +1,16 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" +) + +type Allocations struct{} + +func (Allocations) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + panic("TODO") +} diff --git a/pkg/transform/timescale/actors/verifreg/v9/claims.go b/pkg/transform/timescale/actors/verifreg/v9/claims.go new file mode 100644 index 000000000..ce6a3a42f --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v9/claims.go @@ -0,0 +1,16 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" +) + +type Claims struct{} + +func (Claims) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + panic("TODO") +} diff --git a/pkg/transform/timescale/actors/verifreg/v9/processor.go b/pkg/transform/timescale/actors/verifreg/v9/processor.go new file mode 100644 index 000000000..64d2ddb16 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v9/processor.go @@ -0,0 +1,54 @@ +package v9 + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" +) + +type Extractor interface { + Extract(ctx context.Context, current, executed *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +type StateExtract struct { + Extractors []Extractor +} + +func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.PersistableList, error) { + out := make(model.PersistableList, 0, len(se.Extractors)) + for _, e := range se.Extractors { + m, err := e.Extract(ctx, current, executed, changes) + if err != nil { + return nil, err + } + if m != nil { + out = append(out, m) + } + } + return out, nil +} + +func VerifregHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) { + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, verifregRoot, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + + stateExtractor := &StateExtract{ + Extractors: []Extractor{ + Verifiers{}, + Claims{}, + Allocations{}, + }, + } + return stateExtractor.Process(ctx, current, executed, verifrefStateDiff) +} diff --git a/pkg/transform/timescale/actors/verifreg/v9/verifiers.go b/pkg/transform/timescale/actors/verifreg/v9/verifiers.go new file mode 100644 index 000000000..a3c3b3d13 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v9/verifiers.go @@ -0,0 +1,62 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v9/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" +) + +type Verifiers struct{} + +func (Verifiers) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/fullblock/blocks.go b/pkg/transform/timescale/fullblock/blocks.go new file mode 100644 index 000000000..e952858e8 --- /dev/null +++ b/pkg/transform/timescale/fullblock/blocks.go @@ -0,0 +1,43 @@ +package fullblock + +import ( + "context" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/model/blocks" + "github.com/filecoin-project/lily/pkg/extract/processor" +) + +func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { + out := blocks.BlockHeaders{} + for _, fb := range fullBlocks { + out = append(out, &blocks.BlockHeader{ + Height: int64(fb.Block.Height), + Cid: fb.Block.Cid().String(), + Miner: fb.Block.Miner.String(), + ParentWeight: fb.Block.ParentWeight.String(), + ParentBaseFee: fb.Block.ParentBaseFee.String(), + ParentStateRoot: fb.Block.ParentStateRoot.String(), + WinCount: fb.Block.ElectionProof.WinCount, + Timestamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + }) + } + return out, nil +} + +func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { + out := blocks.BlockParents{} + for _, fb := range fullBlocks { + for _, p := range fb.Block.Parents { + out = append(out, &blocks.BlockParent{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Parent: p.String(), + }) + } + } + return out, nil +} diff --git a/pkg/transform/timescale/fullblock/messages.go b/pkg/transform/timescale/fullblock/messages.go new file mode 100644 index 000000000..4528bd723 --- /dev/null +++ b/pkg/transform/timescale/fullblock/messages.go @@ -0,0 +1,93 @@ +package fullblock + +import ( + "context" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/model/messages" + "github.com/filecoin-project/lily/pkg/extract/processor" +) + +func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { + out := messages.Messages{} + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + out = append(out, &messages.Message{ + Height: int64(fb.Block.Height), + Cid: msg.Message.Cid().String(), + From: msg.Message.Message.From.String(), + To: msg.Message.Message.To.String(), + Value: msg.Message.Message.Value.String(), + GasFeeCap: msg.Message.Message.GasFeeCap.String(), + GasPremium: msg.Message.Message.GasPremium.String(), + GasLimit: msg.Message.Message.GasLimit, + SizeBytes: msg.Message.ChainLength(), + Nonce: msg.Message.Message.Nonce, + Method: uint64(msg.Message.Message.Method), + }) + } + for _, msg := range fb.BlsMessages { + out = append(out, &messages.Message{ + Height: int64(fb.Block.Height), + Cid: msg.Message.Cid().String(), + From: msg.Message.From.String(), + To: msg.Message.To.String(), + Value: msg.Message.Value.String(), + GasFeeCap: msg.Message.GasFeeCap.String(), + GasPremium: msg.Message.GasPremium.String(), + GasLimit: msg.Message.GasLimit, + SizeBytes: msg.Message.ChainLength(), + Nonce: msg.Message.Nonce, + Method: uint64(msg.Message.Method), + }) + } + } + return out, nil +} + +func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { + out := messages.VMMessageList{} + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + out = append(out, &messages.VMMessage{ + Height: int64(fb.Block.Height), + StateRoot: fb.Block.ParentStateRoot.String(), + Cid: vm.Message.Cid().String(), + Source: vm.Source.String(), + From: vm.Message.From.String(), + To: vm.Message.To.String(), + Value: vm.Message.Value.String(), + Method: uint64(vm.Message.Method), + ActorCode: "TODO", + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Params: "", //vm.Message.Params + Returns: "", //vm.Receipt.Return + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + out = append(out, &messages.VMMessage{ + Height: int64(fb.Block.Height), + StateRoot: fb.Block.ParentStateRoot.String(), + Cid: vm.Message.Cid().String(), + Source: vm.Source.String(), + From: vm.Message.From.String(), + To: vm.Message.To.String(), + Value: vm.Message.Value.String(), + Method: uint64(vm.Message.Method), + ActorCode: "TODO", + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Params: "", //vm.Message.Params + Returns: "", //vm.Receipt.Return + }) + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/processor.go b/pkg/transform/timescale/processor.go similarity index 56% rename from pkg/transform/timescale/actors/processor.go rename to pkg/transform/timescale/processor.go index 613d694da..0e1e922ab 100644 --- a/pkg/transform/timescale/actors/processor.go +++ b/pkg/transform/timescale/processor.go @@ -1,4 +1,4 @@ -package actors +package timescale import ( "context" @@ -20,13 +20,17 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/actordiff" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/actors" "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + init_ "github.com/filecoin-project/lily/pkg/transform/timescale/actors/init" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg" + "github.com/filecoin-project/lily/pkg/transform/timescale/fullblock" ) var log = logging.Logger("/lily/transform/timescale") @@ -64,22 +68,27 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe return err } - _, err = HandleFullBlocks(ctx, adtStore, current, parent, stateExtractionIPLDContainer.FullBlocks) + toStorage := model.PersistableList{} + fbModels, err := HandleFullBlocks(ctx, adtStore, current, parent, stateExtractionIPLDContainer.FullBlocks) if err != nil { return err } + toStorage = append(toStorage, fbModels) + _, err = HandleImplicitMessages(ctx, adtStore, current, parent, stateExtractionIPLDContainer.ImplicitMessages) if err != nil { return err } - _, err = HandleActorStateChanges(ctx, adtStore, current, parent, av, stateExtractionIPLDContainer.Actors) + actorModels, err := HandleActorStateChanges(ctx, adtStore, current, parent, av, stateExtractionIPLDContainer.Actors) if err != nil { return err } - return nil + toStorage = append(toStorage, actorModels) + + return strg.PersistBatch(ctx, toStorage) } func ProcessMiners(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { @@ -90,29 +99,102 @@ func ProcessMiners(ctx context.Context, s store.Store, current, executed *types. return minerHandler(ctx, s, current, executed, root) } -func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (interface{}, error) { +func ProcessInitAddresses(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { + return init_.InitHandler(ctx, s, current, executed, root) +} + +func ProcessMarketActor(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { + marketHandler, err := market.MakeMarketProcessor(av) + if err != nil { + return nil, err + } + return marketHandler(ctx, s, current, executed, root) +} + +func ProcessVerifregActor(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { + verifregHandler, err := verifreg.MakeVerifregProcessor(av) + if err != nil { + return nil, err + } + return verifregHandler(ctx, s, current, executed, root) +} + +func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { actorIPLDContainer := new(actors.ActorStateChangesIPLD) if err := s.Get(ctx, root, actorIPLDContainer); err != nil { return nil, err } log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) + out := model.PersistableList{} + if actorIPLDContainer.MarketActor != nil { + marketModels, err := ProcessMarketActor(ctx, s, current, parent, av, *actorIPLDContainer.MarketActor) + if err != nil { + return nil, err + } + out = append(out, marketModels) + } if actorIPLDContainer.MinerActors != nil { minerModels, err := ProcessMiners(ctx, s, current, parent, av, *actorIPLDContainer.MinerActors) if err != nil { return nil, err } - _ = minerModels + out = append(out, minerModels) } - return nil, nil + if actorIPLDContainer.InitActor != nil { + initModels, err := ProcessInitAddresses(ctx, s, current, parent, av, *actorIPLDContainer.InitActor) + if err != nil { + return nil, err + } + out = append(out, initModels) + } + if actorIPLDContainer.RawActors != nil { + rawModels, err := ProcessActorStates(ctx, s, current, parent, av, *actorIPLDContainer.RawActors) + if err != nil { + return nil, err + } + out = append(out, rawModels) + } + /* + if actorIPLDContainer.VerifregActor != nil { + verifregModels, err := ProcessVerifregActor(ctx, s, current, parent, av, *actorIPLDContainer.VerifregActor) + if err != nil { + return nil, err + } + out = append(out, verifregModels) + } + + */ + return out, nil } -func HandleFullBlocks(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (interface{}, error) { +func HandleFullBlocks(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (model.PersistableList, error) { + out := model.PersistableList{} fullBlockMap, err := messages.DecodeFullBlockHAMT(ctx, s, root) if err != nil { return nil, err } - _ = fullBlockMap - return nil, nil + bh, err := fullblock.ExtractBlockHeaders(ctx, fullBlockMap) + if err != nil { + return nil, err + } + out = append(out, bh) + bp, err := fullblock.ExtractBlockParents(ctx, fullBlockMap) + if err != nil { + return nil, err + } + out = append(out, bp) + msgs, err := fullblock.ExtractMessages(ctx, fullBlockMap) + if err != nil { + return nil, err + } + out = append(out, msgs) + vm, err := fullblock.ExtractVmMessages(ctx, fullBlockMap) + if err != nil { + return nil, err + } + out = append(out, vm) + + return out, nil } func HandleImplicitMessages(ctx context.Context, s store.Store, current, parent *types.TipSet, root cid.Cid) (interface{}, error) { @@ -124,14 +206,13 @@ func HandleImplicitMessages(ctx context.Context, s store.Store, current, parent return nil, nil } -func ProcessActorStates(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, av actorstypes.Version, actorMapRoot cid.Cid) (model.Persistable, error) { +func ProcessActorStates(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, actorMapRoot cid.Cid) (model.Persistable, error) { var out = model.PersistableList{} - adtStore := store.WrapBlockStore(ctx, bs) - actorMap, err := adt.AsMap(adtStore, actorMapRoot, 5) + actorMap, err := adt.AsMap(s, actorMapRoot, 5) if err != nil { return nil, err } - actorState := new(actordiff.ActorChange) + actorState := new(rawdiff.ActorChange) if err := actorMap.ForEach(actorState, func(key string) error { addr, err := address.NewFromBytes([]byte(key)) if err != nil { From b2f33ef9e9782c8d19de368f36f7ac2642347249 Mon Sep 17 00:00:00 2001 From: frrist Date: Mon, 23 Jan 2023 10:53:56 -0800 Subject: [PATCH 30/48] wire up allocation for verifreg --- .../actors/builtin/verifreg/actor.go.template | 5 + .../actors/builtin/verifreg/state.go.template | 42 ++++++++ chain/actors/builtin/verifreg/v0.go | 27 +++++ chain/actors/builtin/verifreg/v2.go | 27 +++++ chain/actors/builtin/verifreg/v3.go | 27 +++++ chain/actors/builtin/verifreg/v4.go | 27 +++++ chain/actors/builtin/verifreg/v5.go | 27 +++++ chain/actors/builtin/verifreg/v6.go | 27 +++++ chain/actors/builtin/verifreg/v7.go | 27 +++++ chain/actors/builtin/verifreg/v8.go | 27 +++++ chain/actors/builtin/verifreg/v9.go | 31 ++++++ chain/actors/builtin/verifreg/verifreg.go | 5 + chain/datasource/datasource.go | 16 +++ chain/indexer/tasktype/table_tasks.go | 2 +- lens/interface.go | 2 + lens/lily/impl.go | 34 ++---- model/actors/market/dealproposal.go | 2 +- pkg/extract/{processor => }/actors.go | 102 ++++++------------ pkg/extract/actors/datacap/v9/allowances.go | 1 + pkg/extract/actors/datacap/v9/balances.go | 60 +++++++++++ pkg/extract/actors/datacap/v9/load.go | 24 +++++ pkg/extract/actors/datacap/v9/state.go | 1 + pkg/extract/actors/executor.go | 31 ++++++ pkg/extract/actors/generic/types.go | 2 + pkg/extract/actors/initdiff/v0/state.go | 15 ++- pkg/extract/actors/marketdiff/v0/state.go | 15 ++- pkg/extract/actors/minerdiff/v0/state.go | 33 ++---- pkg/extract/actors/powerdiff/v0/state.go | 15 ++- pkg/extract/actors/rawdiff/state.go | 41 +++++-- pkg/extract/actors/types.go | 2 - pkg/extract/actors/verifregdiff/v0/state.go | 16 +-- .../actors/verifregdiff/v9/allocations.go | 84 ++++++++++++++- .../actors/verifregdiff/v9/cbor_gen.go | 26 ++--- pkg/extract/actors/verifregdiff/v9/claims.go | 9 +- pkg/extract/actors/verifregdiff/v9/load.go | 14 ++- pkg/extract/actors/verifregdiff/v9/state.go | 16 +-- pkg/extract/actors/verifregdiff/version.go | 3 +- pkg/extract/chain.go | 39 +++++++ pkg/extract/{processor => chain}/cbor_gen.go | 2 +- pkg/extract/{processor => chain}/messages.go | 38 ++----- pkg/extract/{processor => chain}/vm.go | 2 +- pkg/extract/extract.go | 67 ++++++++++++ pkg/gen/main.go | 16 +-- pkg/transform/cbor/actors/serialize.go | 4 +- pkg/transform/cbor/actors/verifreg/state.go | 10 +- pkg/transform/cbor/messages/cbor_gen.go | 3 +- pkg/transform/cbor/messages/serialize.go | 82 +++++++------- pkg/transform/cbor/router.go | 26 ++--- pkg/transform/timescale/fullblock/blocks.go | 6 +- pkg/transform/timescale/fullblock/messages.go | 6 +- schemas/v1/base.go | 2 +- tasks/api.go | 3 + tasks/messages/message_test.go | 2 +- 53 files changed, 874 insertions(+), 299 deletions(-) rename pkg/extract/{processor => }/actors.go (66%) create mode 100644 pkg/extract/actors/datacap/v9/allowances.go create mode 100644 pkg/extract/actors/datacap/v9/balances.go create mode 100644 pkg/extract/actors/datacap/v9/load.go create mode 100644 pkg/extract/actors/datacap/v9/state.go create mode 100644 pkg/extract/actors/executor.go create mode 100644 pkg/extract/chain.go rename pkg/extract/{processor => chain}/cbor_gen.go (99%) rename pkg/extract/{processor => chain}/messages.go (92%) rename pkg/extract/{processor => chain}/vm.go (99%) create mode 100644 pkg/extract/extract.go diff --git a/chain/actors/builtin/verifreg/actor.go.template b/chain/actors/builtin/verifreg/actor.go.template index 49da54270..dfffe00fe 100644 --- a/chain/actors/builtin/verifreg/actor.go.template +++ b/chain/actors/builtin/verifreg/actor.go.template @@ -77,6 +77,11 @@ type State interface { ClaimsMapBitWidth() int ClaimsMapHashFunction() func(input []byte) []byte + AllocationsMap() (adt.Map, error) + AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) + AllocationsMapBitWidth() int + AllocationsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/actors/builtin/verifreg/state.go.template b/chain/actors/builtin/verifreg/state.go.template index 442558e1b..c6c8eab33 100644 --- a/chain/actors/builtin/verifreg/state.go.template +++ b/chain/actors/builtin/verifreg/state.go.template @@ -130,6 +130,48 @@ func (s *state{{.v}}) VerifiersMapHashFunction() func(input []byte) []byte { {{end}} } +func (s *state{{.v}}) AllocationsMap() (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + return adt{{.v}}.AsMap(s.store, s.Allocations{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}}) +{{end}} +} + +func (s *state{{.v}}) AllocationsMapBitWidth() int { +{{if (ge .v 3)}} + return builtin{{.v}}.DefaultHamtBitwidth +{{else}} + return 5 +{{end}} +} + +func (s *state{{.v}}) AllocationsMapHashFunction() func(input []byte) []byte { +{{if (le .v 1)}} + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } +{{else}} + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +{{end}} +} + +func (s *state{{.v}}) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { +{{if (le .v 8)}} + return nil, fmt.Errorf("unsupported in actors v{{.v}}") +{{else}} + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(clientIdAddr), s.Allocations, builtin{{.v}}.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, builtin{{.v}}.DefaultHamtBitwidth) +{{end}} +} + func (s *state{{.v}}) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v0.go b/chain/actors/builtin/verifreg/v0.go index 1791dd447..26f760c7b 100644 --- a/chain/actors/builtin/verifreg/v0.go +++ b/chain/actors/builtin/verifreg/v0.go @@ -93,6 +93,33 @@ func (s *state0) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state0) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + +func (s *state0) AllocationsMapBitWidth() int { + + return 5 + +} + +func (s *state0) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256simd.Sum256(input) + return res[:] + } + +} + +func (s *state0) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v0") + +} + func (s *state0) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v2.go b/chain/actors/builtin/verifreg/v2.go index 54ad540f2..f505b5bb9 100644 --- a/chain/actors/builtin/verifreg/v2.go +++ b/chain/actors/builtin/verifreg/v2.go @@ -93,6 +93,33 @@ func (s *state2) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state2) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + +func (s *state2) AllocationsMapBitWidth() int { + + return 5 + +} + +func (s *state2) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state2) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v2") + +} + func (s *state2) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v3.go b/chain/actors/builtin/verifreg/v3.go index f6dce7c04..7e3c15178 100644 --- a/chain/actors/builtin/verifreg/v3.go +++ b/chain/actors/builtin/verifreg/v3.go @@ -95,6 +95,33 @@ func (s *state3) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state3) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + +func (s *state3) AllocationsMapBitWidth() int { + + return builtin3.DefaultHamtBitwidth + +} + +func (s *state3) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state3) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v3") + +} + func (s *state3) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v4.go b/chain/actors/builtin/verifreg/v4.go index 748e3bffd..0e7ab3fb6 100644 --- a/chain/actors/builtin/verifreg/v4.go +++ b/chain/actors/builtin/verifreg/v4.go @@ -95,6 +95,33 @@ func (s *state4) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state4) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + +func (s *state4) AllocationsMapBitWidth() int { + + return builtin4.DefaultHamtBitwidth + +} + +func (s *state4) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state4) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v4") + +} + func (s *state4) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v5.go b/chain/actors/builtin/verifreg/v5.go index 325ffcce0..f701c09ff 100644 --- a/chain/actors/builtin/verifreg/v5.go +++ b/chain/actors/builtin/verifreg/v5.go @@ -95,6 +95,33 @@ func (s *state5) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state5) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + +func (s *state5) AllocationsMapBitWidth() int { + + return builtin5.DefaultHamtBitwidth + +} + +func (s *state5) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state5) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v5") + +} + func (s *state5) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v6.go b/chain/actors/builtin/verifreg/v6.go index bf78ed5da..851cda196 100644 --- a/chain/actors/builtin/verifreg/v6.go +++ b/chain/actors/builtin/verifreg/v6.go @@ -95,6 +95,33 @@ func (s *state6) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state6) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + +func (s *state6) AllocationsMapBitWidth() int { + + return builtin6.DefaultHamtBitwidth + +} + +func (s *state6) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state6) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v6") + +} + func (s *state6) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v7.go b/chain/actors/builtin/verifreg/v7.go index ef43ae36c..4e53e548b 100644 --- a/chain/actors/builtin/verifreg/v7.go +++ b/chain/actors/builtin/verifreg/v7.go @@ -95,6 +95,33 @@ func (s *state7) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state7) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + +func (s *state7) AllocationsMapBitWidth() int { + + return builtin7.DefaultHamtBitwidth + +} + +func (s *state7) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state7) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v7") + +} + func (s *state7) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v8.go b/chain/actors/builtin/verifreg/v8.go index 7bcf54765..3e0b4d483 100644 --- a/chain/actors/builtin/verifreg/v8.go +++ b/chain/actors/builtin/verifreg/v8.go @@ -94,6 +94,33 @@ func (s *state8) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state8) AllocationsMap() (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + +func (s *state8) AllocationsMapBitWidth() int { + + return builtin8.DefaultHamtBitwidth + +} + +func (s *state8) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state8) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + return nil, fmt.Errorf("unsupported in actors v8") + +} + func (s *state8) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/v9.go b/chain/actors/builtin/verifreg/v9.go index 58e7e5365..9bafc6003 100644 --- a/chain/actors/builtin/verifreg/v9.go +++ b/chain/actors/builtin/verifreg/v9.go @@ -96,6 +96,37 @@ func (s *state9) VerifiersMapHashFunction() func(input []byte) []byte { } +func (s *state9) AllocationsMap() (adt.Map, error) { + + return adt9.AsMap(s.store, s.Allocations, builtin9.DefaultHamtBitwidth) + +} + +func (s *state9) AllocationsMapBitWidth() int { + + return builtin9.DefaultHamtBitwidth + +} + +func (s *state9) AllocationsMapHashFunction() func(input []byte) []byte { + + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } + +} + +func (s *state9) AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) { + + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(clientIdAddr), s.Allocations, builtin9.DefaultHamtBitwidth) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, builtin9.DefaultHamtBitwidth) + +} + func (s *state9) RootKey() (address.Address, error) { return s.State.RootKey, nil } diff --git a/chain/actors/builtin/verifreg/verifreg.go b/chain/actors/builtin/verifreg/verifreg.go index 31792b0b0..ef2684a07 100644 --- a/chain/actors/builtin/verifreg/verifreg.go +++ b/chain/actors/builtin/verifreg/verifreg.go @@ -104,6 +104,11 @@ type State interface { ClaimsMapBitWidth() int ClaimsMapHashFunction() func(input []byte) []byte + AllocationsMap() (adt.Map, error) + AllocationMapForClient(clientIdAddr address.Address) (adt.Map, error) + AllocationsMapBitWidth() int + AllocationsMapHashFunction() func(input []byte) []byte + RootKey() (address.Address, error) VerifiedClientDataCap(address.Address) (bool, abi.StoragePower, error) VerifierDataCap(address.Address) (bool, abi.StoragePower, error) diff --git a/chain/datasource/datasource.go b/chain/datasource/datasource.go index fddc18b33..733337bfc 100644 --- a/chain/datasource/datasource.go +++ b/chain/datasource/datasource.go @@ -138,6 +138,22 @@ type DataSource struct { diffPreCommitGroup singleflight.Group } +func (t *DataSource) NetworkName(ctx context.Context) (string, error) { + name, err := t.node.StateNetworkName(ctx) + if err != nil { + return "", err + } + return string(name), nil +} + +func (t *DataSource) NetworkVersion(ctx context.Context, tsk types.TipSetKey) (uint, error) { + v, err := t.node.StateNetworkVersion(ctx, tsk) + if err != nil { + return 0, err + } + return uint(v), nil +} + func (t *DataSource) ComputeBaseFee(ctx context.Context, ts *types.TipSet) (abi.TokenAmount, error) { return t.node.ComputeBaseFee(ctx, ts) } diff --git a/chain/indexer/tasktype/table_tasks.go b/chain/indexer/tasktype/table_tasks.go index 309dfe341..407cbbb78 100644 --- a/chain/indexer/tasktype/table_tasks.go +++ b/chain/indexer/tasktype/table_tasks.go @@ -197,7 +197,7 @@ var TableFieldComments = map[string]map[string]string{ "Label": "An arbitrary client chosen label to apply to the deal. The value is base64 encoded before persisting.", "PaddedPieceSize": "The piece size in bytes with padding.", "PieceCID": "CID of a sector piece. A Piece is an object that represents a whole or part of a File.", - "ProviderCollateral": "The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires.", + "ProviderCollateral": "The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires.", "ProviderID": "Address of the actor providing the services.", "StartEpoch": "The epoch at which this deal with begin. Storage deal must appear in a sealed (proven) sector no later than start_epoch, otherwise it is invalid.", "StateRoot": "CID of the parent state root for this deal.", diff --git a/lens/interface.go b/lens/interface.go index e9fd91d17..6e139d16b 100644 --- a/lens/interface.go +++ b/lens/interface.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/filecoin-project/go-state-types/exitcode" + "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/filecoin-project/go-address" @@ -64,6 +65,7 @@ type StateAPI interface { StateGetReceipt(ctx context.Context, bcid cid.Cid, tsk types.TipSetKey) (*types.MessageReceipt, error) CirculatingSupply(context.Context, types.TipSetKey) (api.CirculatingSupply, error) StateNetworkName(context.Context) (dtypes.NetworkName, error) + StateNetworkVersion(ctx context.Context, key types.TipSetKey) (network.Version, error) } type ShouldBurnFn func(ctx context.Context, msg *types.Message, errcode exitcode.ExitCode) (bool, error) diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 96bed2968..9f0a58900 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -28,7 +28,6 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -44,7 +43,7 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" @@ -142,7 +141,7 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) // the context's passed to these methods live for the duration of the clients request, so make a new one. ctx := context.Background() - // TODO add buffering + // TODO add file buffering f, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) defer f.Close() if err != nil { @@ -164,38 +163,17 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } - grp, grpCtx := errgroup.WithContext(ctx) - // TODO parallelize the below - var ( - messageChanges *processor.MessageStateChanges - actorChanges *processor.ActorStateChanges - ) - grp.Go(func() error { - messageChanges, err = processor.Messages(grpCtx, taskAPI, currentTs, executedTs) - if err != nil { - return err - } - return nil - }) - grp.Go(func() error { - actorChanges, err = processor.Actors(grpCtx, taskAPI, currentTs, executedTs, m.StateManager.GetNetworkVersion) - if err != nil { - return err - } - - return nil - }) - - if err := grp.Wait(); err != nil { + chainState, err := extract.State(ctx, taskAPI, currentTs, executedTs) + if err != nil { return false, err } bs := blockstore.NewMemorySync() - root, err := cbor.PersistToStore(ctx, bs, currentTs, executedTs, messageChanges, actorChanges) + root, err := cbor.PersistToStore(ctx, bs, currentTs, executedTs, chainState) if err != nil { return false, err } - if err := cbor.WriteCar(ctx, root, 1, bs, f); err != nil { + if err := cbor.WriteCarV1(ctx, root, bs, f); err != nil { return false, err } return true, nil diff --git a/model/actors/market/dealproposal.go b/model/actors/market/dealproposal.go index 829da0b0b..c1d025b5f 100644 --- a/model/actors/market/dealproposal.go +++ b/model/actors/market/dealproposal.go @@ -37,7 +37,7 @@ type MarketDealProposal struct { ProviderID string `pg:",notnull"` // The amount of FIL (in attoFIL) the client has pledged as collateral. ClientCollateral string `pg:",notnull"` - // The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires. + // The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires. ProviderCollateral string `pg:",notnull"` // The amount of FIL (in attoFIL) that will be transferred from the client to the provider every epoch this deal is active for. StoragePricePerEpoch string `pg:",notnull"` diff --git a/pkg/extract/processor/actors.go b/pkg/extract/actors.go similarity index 66% rename from pkg/extract/processor/actors.go rename to pkg/extract/actors.go index 9a688c22c..59c637141 100644 --- a/pkg/extract/processor/actors.go +++ b/pkg/extract/actors.go @@ -1,4 +1,4 @@ -package processor +package extract import ( "context" @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" + actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" logging "github.com/ipfs/go-log/v2" @@ -13,6 +14,7 @@ import ( "go.uber.org/zap/zapcore" "golang.org/x/sync/errgroup" + "github.com/filecoin-project/lily/chain/actors/builtin" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" @@ -60,7 +62,26 @@ type StateDiffResult struct { type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version -func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, nvg NetworkVersionGetter) (*ActorStateChanges, error) { +func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Version, act *actors.ActorChange, diffLoader func(av actortypes.Version) (actors.ActorDiff, error)) (*StateDiffResult, error) { + // construct the state differ required by this actor version + diff, err := diffLoader(version) + if err != nil { + return nil, err + } + start := time.Now() + // diff the actors state and collect changes + diffRes, err := diff.State(ctx, api, act) + if err != nil { + return nil, err + } + log.Infow("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "duration", time.Since(start)) + return &StateDiffResult{ + ActorDiff: diffRes, + Address: act.Address, + }, nil +} + +func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { return nil, err @@ -72,11 +93,6 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } - actorVersion, err := core.ActorVersionForTipSet(ctx, current, nvg) - if err != nil { - return nil, err - } - grp, grpCtx := errgroup.WithContext(ctx) results := make(chan *StateDiffResult, len(actorChanges)) for addr, change := range actorChanges { @@ -88,6 +104,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. Current: change.Current, Type: change.ChangeType, } + grp.Go(func() error { actorDiff := &rawdiff.StateDiff{ DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, @@ -98,91 +115,42 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. } results <- &StateDiffResult{ ActorDiff: actorStateChanges, - Address: addr, + Address: act.Address, } if core.MinerCodes.Has(change.Current.Code) { - start := time.Now() - // construct the state differ required by this actor version - actorDiff, err := minerdiff.StateDiffFor(actorVersion) + res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) if err != nil { return err } - // diff the actors state and collect changes - minerChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - log.Infow("Extract Miner", "address", addr, "duration", time.Since(start)) - results <- &StateDiffResult{ - ActorDiff: minerChanges, - Address: addr, - } + results <- res } if core.VerifregCodes.Has(change.Current.Code) { - start := time.Now() - // construct the state differ required by this actor version - actorDiff, err := verifregdiff.StateDiffFor(actorVersion) + res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) if err != nil { return err } - // diff the actors state and collect changes - verifregChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - log.Infow("Extract VerifiedRegistry", "address", addr, "duration", time.Since(start)) - results <- &StateDiffResult{ - ActorDiff: verifregChanges, - Address: addr, - } + results <- res } if core.InitCodes.Has(change.Current.Code) { - start := time.Now() - actorDiff, err := initdiff.StateDiffFor(actorVersion) + res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) if err != nil { return err } - initChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - log.Infow("Extracted Init", "address", addr, "duration", time.Since(start)) - results <- &StateDiffResult{ - ActorDiff: initChanges, - Address: addr, - } + results <- res } if core.PowerCodes.Has(change.Current.Code) { - start := time.Now() - actorDiff, err := powerdiff.StateDiffFor(actorVersion) + res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) if err != nil { return err } - powerChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - log.Infow("Extracted Power", "address", addr, "duration", time.Since(start)) - results <- &StateDiffResult{ - ActorDiff: powerChanges, - Address: addr, - } + results <- res } if core.MarketCodes.Has(change.Current.Code) { - start := time.Now() - actorDiff, err := marketdiff.StateDiffFor(actorVersion) + res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) if err != nil { return err } - marketChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - log.Infow("Extracted Market", "address", addr, "duration", time.Since(start)) - results <- &StateDiffResult{ - ActorDiff: marketChanges, - Address: addr, - } + results <- res } return nil }) diff --git a/pkg/extract/actors/datacap/v9/allowances.go b/pkg/extract/actors/datacap/v9/allowances.go new file mode 100644 index 000000000..be1c6b564 --- /dev/null +++ b/pkg/extract/actors/datacap/v9/allowances.go @@ -0,0 +1 @@ +package v9 diff --git a/pkg/extract/actors/datacap/v9/balances.go b/pkg/extract/actors/datacap/v9/balances.go new file mode 100644 index 000000000..eb975372a --- /dev/null +++ b/pkg/extract/actors/datacap/v9/balances.go @@ -0,0 +1,60 @@ +package v9 + +import ( + "context" + "time" + + logging "github.com/ipfs/go-log/v2" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("lily/extract/actors/balance/v9") + +type BalanceChange struct { + Client []byte + Current *typegen.Deferred + Previous *typegen.Deferred + Change core.ChangeType +} + +type BalanceChangeList []*BalanceChange + +const KindDataCapBalance = "datacap_balance" + +func (b BalanceChangeList) Kind() actors.ActorStateKind { + return KindDataCapBalance +} + +type Balance struct{} + +func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindDataCapBalance, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffBalances(ctx, api, act) +} + +func DiffBalances(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (BalanceChangeList, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapBalancesMapLoader) + if err != nil { + return nil, err + } + + out := make(BalanceChangeList, len(mapChange)) + for i, change := range mapChange { + out[i] = &BalanceChange{ + Client: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + } + } + return out, nil +} diff --git a/pkg/extract/actors/datacap/v9/load.go b/pkg/extract/actors/datacap/v9/load.go new file mode 100644 index 000000000..d44beb5af --- /dev/null +++ b/pkg/extract/actors/datacap/v9/load.go @@ -0,0 +1,24 @@ +package v9 + +import ( + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/datacap" +) + +var DatacapStateLoader = func(s adt.Store, act *types.Actor) (interface{}, error) { + return datacap.Load(s, act) +} + +var DatacapBalancesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := m.(datacap.State) + balanceMap, err := datacapState.VerifiedClients() + if err != nil { + return nil, nil, err + } + return balanceMap, &adt.MapOpts{ + Bitwidth: datacapState.VerifiedClientsMapBitWidth(), + HashFunc: datacapState.VerifiedClientsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/datacap/v9/state.go b/pkg/extract/actors/datacap/v9/state.go new file mode 100644 index 000000000..be1c6b564 --- /dev/null +++ b/pkg/extract/actors/datacap/v9/state.go @@ -0,0 +1 @@ +package v9 diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go new file mode 100644 index 000000000..ff2966c75 --- /dev/null +++ b/pkg/extract/actors/executor.go @@ -0,0 +1,31 @@ +package actors + +import ( + "context" + + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/tasks" +) + +func ExecuteStateDiff(ctx context.Context, grp *errgroup.Group, api tasks.DataSource, act *ActorChange, fns ...ActorStateDiff) ([]ActorStateChange, error) { + out := make([]ActorStateChange, len(fns)) + for i, fn := range fns { + fn := fn + i := i + + grp.Go(func() error { + res, err := fn.Diff(ctx, api, act) + if err != nil { + return err + } + out[i] = res + return nil + }) + } + + if err := grp.Wait(); err != nil { + return nil, err + } + return out, nil +} diff --git a/pkg/extract/actors/generic/types.go b/pkg/extract/actors/generic/types.go index 5414dc6a7..8d44e13de 100644 --- a/pkg/extract/actors/generic/types.go +++ b/pkg/extract/actors/generic/types.go @@ -9,6 +9,8 @@ import ( var log = logging.Logger("lily/extract/diff") +// TODO use go-state-types store + type ActorStateArrayLoader = func(interface{}) (adt.Array, int, error) type ActorStateLoader = func(adt.Store, *types.Actor) (interface{}, error) type ActorStateMapLoader = func(interface{}) (adt.Map, *adt.MapOpts, error) diff --git a/pkg/extract/actors/initdiff/v0/state.go b/pkg/extract/actors/initdiff/v0/state.go index 17b2e5e53..ac7b3bd76 100644 --- a/pkg/extract/actors/initdiff/v0/state.go +++ b/pkg/extract/actors/initdiff/v0/state.go @@ -2,12 +2,12 @@ package v0 import ( "context" - "time" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -18,13 +18,13 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -33,7 +33,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.AddressesChanges = stateChange.(AddressChangeList) } } - log.Infow("Extracted Init State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/marketdiff/v0/state.go b/pkg/extract/actors/marketdiff/v0/state.go index e4b4f8679..d9b3dbb4d 100644 --- a/pkg/extract/actors/marketdiff/v0/state.go +++ b/pkg/extract/actors/marketdiff/v0/state.go @@ -2,13 +2,13 @@ package v0 import ( "context" - "time" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -19,13 +19,13 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpCtx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpCtx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -38,7 +38,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors panic(stateChange.Kind()) } } - log.Infow("Extracted Market State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v0/state.go index ae4c9c76c..2d3294f29 100644 --- a/pkg/extract/actors/minerdiff/v0/state.go +++ b/pkg/extract/actors/minerdiff/v0/state.go @@ -22,33 +22,18 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpCtx := errgroup.WithContext(ctx) - results := make(chan actors.ActorStateChange, len(s.DiffMethods)) - - for _, f := range s.DiffMethods { - f := f - grp.Go(func() error { - stateChange, err := f.Diff(grpCtx, api, act) - if err != nil { - return err - } - - // TODO maybe this method should also return a bool to indicate if anything actually changed, instead of two null values. - if stateChange != nil { - results <- stateChange - } - return nil - }) + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err } - go func() { - if err := grp.Wait(); err != nil { - log.Error(err) - } - close(results) - }() var stateDiff = new(StateDiffResult) - for stateChange := range results { + for _, stateChange := range results { + // some results may be nil, skip those + if stateChange == nil { + continue + } switch stateChange.Kind() { case KindMinerInfo: stateDiff.InfoChange = stateChange.(*InfoChange) diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v0/state.go index ed559876d..6e517611b 100644 --- a/pkg/extract/actors/powerdiff/v0/state.go +++ b/pkg/extract/actors/powerdiff/v0/state.go @@ -2,12 +2,12 @@ package v0 import ( "context" - "time" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -18,13 +18,13 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpCtx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpCtx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -35,7 +35,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors panic(stateChange.Kind()) } } - log.Infow("Extracted Power State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/rawdiff/state.go b/pkg/extract/actors/rawdiff/state.go index 6c7c5b38e..686fd10f1 100644 --- a/pkg/extract/actors/rawdiff/state.go +++ b/pkg/extract/actors/rawdiff/state.go @@ -2,11 +2,11 @@ package rawdiff import ( "context" - "time" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -17,13 +17,13 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -32,7 +32,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.ActorStateChanges = stateChange.(*ActorChange) } } - log.Infow("Extracted Raw Actor State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } @@ -51,3 +50,29 @@ func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Stor type StateChange struct { ActorState cid.Cid `cborgen:"actors"` } + +func GenericStateDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns []actors.ActorStateDiff) ([]actors.ActorStateChange, error) { + grp, grpCtx := errgroup.WithContext(ctx) + out := make([]actors.ActorStateChange, 0, len(diffFns)) + results := make(chan actors.ActorStateChange, len(diffFns)) + for _, diff := range diffFns { + diff := diff + grp.Go(func() error { + res, err := diff.Diff(grpCtx, api, act) + if err != nil { + return err + } + results <- res + return nil + }) + } + if err := grp.Wait(); err != nil { + close(results) + return nil, err + } + close(results) + for res := range results { + out = append(out, res) + } + return out, nil +} diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index 40013aeff..f7da04f5b 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -55,5 +55,3 @@ func (a ActorChange) MarshalLogObject(enc zapcore.ObjectEncoder) error { } return nil } - -type ActorChanges []*ActorChange diff --git a/pkg/extract/actors/verifregdiff/v0/state.go b/pkg/extract/actors/verifregdiff/v0/state.go index f17bd6881..7a572728f 100644 --- a/pkg/extract/actors/verifregdiff/v0/state.go +++ b/pkg/extract/actors/verifregdiff/v0/state.go @@ -2,12 +2,12 @@ package v0 import ( "context" - "time" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/tasks" @@ -18,13 +18,14 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } + var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -35,7 +36,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) } } - log.Infow("Extracted Verifid Registry State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/verifregdiff/v9/allocations.go b/pkg/extract/actors/verifregdiff/v9/allocations.go index e3f4c5c82..2a57da891 100644 --- a/pkg/extract/actors/verifregdiff/v9/allocations.go +++ b/pkg/extract/actors/verifregdiff/v9/allocations.go @@ -5,19 +5,25 @@ import ( "time" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" "github.com/filecoin-project/lily/tasks" ) // TODO add cbor gen tags type AllocationsChange struct { - Provider []byte `cborgen:"provider"` + Client []byte `cborgen:"provider"` ClaimID []byte `cborgen:"claimID"` Current *typegen.Deferred `cborgen:"current"` Previous *typegen.Deferred `cborgen:"previous"` @@ -33,7 +39,29 @@ func (c AllocationsChangeMap) Kind() actors.ActorStateKind { } func (c AllocationsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { - panic("NYI") + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for client, changes := range c { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.ClaimID), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(client), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() } type Allocations struct{} @@ -47,7 +75,53 @@ func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.A } func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - // TODO this will look a lot like the DiffClaims method, except diffing allocations - // - need to add method to the actor that exposes the allocations HAMT and its sub HAMT AllocationMapForProvider() - panic("NYI") + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregAllocationMapLoader) + if err != nil { + return nil, err + } + out := make(AllocationsChangeMap) + for _, change := range mapChange { + clientId, err := abi.ParseUIntKey(string(change.Key)) + if err != nil { + return nil, err + } + clientAddress, err := address.NewIDAddress(clientId) + if err != nil { + return nil, err + } + clientMapChanges, err := diffClientMap(ctx, api, act, clientAddress, change.Key) + if err != nil { + return nil, err + } + out[clientAddress] = clientMapChanges + } + return out, nil +} + +func diffClientMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, clientAddress address.Address, clientKey []byte) ([]*AllocationsChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := i.(verifreg.State) + clientAllocationMap, err := verifregState.AllocationMapForClient(clientAddress) + if err != nil { + return nil, nil, err + } + return clientAllocationMap, &adt.MapOpts{ + Bitwidth: verifregState.AllocationsMapBitWidth(), + HashFunc: verifregState.AllocationsMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*AllocationsChange, 0, len(mapChange)) + for _, change := range mapChange { + out = append(out, &AllocationsChange{ + Client: clientKey, + ClaimID: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + }) + } + return out, nil } diff --git a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go index a5b4c64e6..6caf28fa1 100644 --- a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go @@ -228,7 +228,7 @@ func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { return err } - // t.Provider ([]uint8) (slice) + // t.Client ([]uint8) (slice) if len("provider") > cbg.MaxLength { return xerrors.Errorf("Value in field \"provider\" was too long") } @@ -241,7 +241,7 @@ func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { } if len(t.Provider) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Provider was too long") + return xerrors.Errorf("Byte array in field t.Client was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { @@ -364,7 +364,7 @@ func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.Provider ([]uint8) (slice) + // t.Client ([]uint8) (slice) case "provider": maj, extra, err = cr.ReadHeader() @@ -373,7 +373,7 @@ func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Provider: byte array too large (%d)", extra) + return fmt.Errorf("t.Client: byte array too large (%d)", extra) } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") @@ -465,7 +465,7 @@ func (t *AllocationsChange) MarshalCBOR(w io.Writer) error { return err } - // t.Provider ([]uint8) (slice) + // t.Client ([]uint8) (slice) if len("provider") > cbg.MaxLength { return xerrors.Errorf("Value in field \"provider\" was too long") } @@ -477,15 +477,15 @@ func (t *AllocationsChange) MarshalCBOR(w io.Writer) error { return err } - if len(t.Provider) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Provider was too long") + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { return err } - if _, err := cw.Write(t.Provider[:]); err != nil { + if _, err := cw.Write(t.Client[:]); err != nil { return err } @@ -601,7 +601,7 @@ func (t *AllocationsChange) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.Provider ([]uint8) (slice) + // t.Client ([]uint8) (slice) case "provider": maj, extra, err = cr.ReadHeader() @@ -610,17 +610,17 @@ func (t *AllocationsChange) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Provider: byte array too large (%d)", extra) + return fmt.Errorf("t.Client: byte array too large (%d)", extra) } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") } if extra > 0 { - t.Provider = make([]uint8, extra) + t.Client = make([]uint8, extra) } - if _, err := io.ReadFull(cr, t.Provider[:]); err != nil { + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { return err } // t.ClaimID ([]uint8) (slice) diff --git a/pkg/extract/actors/verifregdiff/v9/claims.go b/pkg/extract/actors/verifregdiff/v9/claims.go index 14d9a0c4c..ae6ca0594 100644 --- a/pkg/extract/actors/verifregdiff/v9/claims.go +++ b/pkg/extract/actors/verifregdiff/v9/claims.go @@ -6,7 +6,6 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" @@ -14,6 +13,8 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -76,7 +77,7 @@ func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorC } func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { - mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifiiregClaimsMapLoader) + mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregClaimsMapLoader) if err != nil { return nil, err } @@ -91,7 +92,7 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan if err != nil { return nil, err } - subMapChanges, err := diffSubMap(ctx, api, act, providerAddress, change.Key) + subMapChanges, err := diffProviderMap(ctx, api, act, providerAddress, change.Key) if err != nil { return nil, err } @@ -100,7 +101,7 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan return out, nil } -func diffSubMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { +func diffProviderMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { verifregState := i.(verifreg.State) providerClaimMap, err := verifregState.ClaimMapForProvider(providerAddress) diff --git a/pkg/extract/actors/verifregdiff/v9/load.go b/pkg/extract/actors/verifregdiff/v9/load.go index b5ddea1f5..46cb8e10c 100644 --- a/pkg/extract/actors/verifregdiff/v9/load.go +++ b/pkg/extract/actors/verifregdiff/v9/load.go @@ -5,7 +5,7 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" ) -var VerifiiregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { +var VerifregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { verifregState := m.(verifreg.State) claimsMap, err := verifregState.ClaimsMap() if err != nil { @@ -16,3 +16,15 @@ var VerifiiregClaimsMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, erro HashFunc: verifregState.ClaimsMapHashFunction(), }, nil } + +var VerifregAllocationMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + verifregState := m.(verifreg.State) + allocationMap, err := verifregState.AllocationsMap() + if err != nil { + return nil, nil, err + } + return allocationMap, &adt.MapOpts{ + Bitwidth: verifregState.AllocationsMapBitWidth(), + HashFunc: verifregState.AllocationsMapHashFunction(), + }, nil +} diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index 7a440c965..744987a8f 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -2,13 +2,13 @@ package v9 import ( "context" - "time" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" @@ -23,13 +23,14 @@ type StateDiff struct { } func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - start := time.Now() + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } + var stateDiff = new(StateDiffResult) - for _, f := range s.DiffMethods { - stateChange, err := f.Diff(ctx, api, act) - if err != nil { - return nil, err - } + for _, stateChange := range results { if stateChange == nil { continue } @@ -42,7 +43,6 @@ func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) } } - log.Infow("Extracted Verified Registry State Diff", "address", act.Address, "duration", time.Since(start)) return stateDiff, nil } diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go index c54a034c2..1f9c19e8b 100644 --- a/pkg/extract/actors/verifregdiff/version.go +++ b/pkg/extract/actors/verifregdiff/version.go @@ -72,8 +72,7 @@ func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { DiffMethods: []actors.ActorStateDiff{ v9.Verifiers{}, v9.Claims{}, - // TODO implement this - //v9.Allocations{}, + v9.Allocations{}, }}, nil case actortypes.Version10: panic("Not yet implemented") diff --git a/pkg/extract/chain.go b/pkg/extract/chain.go new file mode 100644 index 000000000..0b37b4570 --- /dev/null +++ b/pkg/extract/chain.go @@ -0,0 +1,39 @@ +package extract + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/tasks" +) + +type MessageStateChanges struct { + Current *types.TipSet + Executed *types.TipSet + BaseFee abi.TokenAmount + FullBlocks map[cid.Cid]*chain.FullBlock + ImplicitMessages []*chain.ImplicitMessage +} + +func FullBlockMessages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*MessageStateChanges, error) { + baseFee, err := api.ComputeBaseFee(ctx, executed) + if err != nil { + return nil, err + } + fullBlocks, implicitMessages, err := chain.Messages(ctx, api, current, executed) + if err != nil { + return nil, err + } + + return &MessageStateChanges{ + Current: current, + Executed: executed, + BaseFee: baseFee, + FullBlocks: fullBlocks, + ImplicitMessages: implicitMessages, + }, nil +} diff --git a/pkg/extract/processor/cbor_gen.go b/pkg/extract/chain/cbor_gen.go similarity index 99% rename from pkg/extract/processor/cbor_gen.go rename to pkg/extract/chain/cbor_gen.go index 1ca287217..e6f7c5997 100644 --- a/pkg/extract/processor/cbor_gen.go +++ b/pkg/extract/chain/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package processor +package chain import ( "fmt" diff --git a/pkg/extract/processor/messages.go b/pkg/extract/chain/messages.go similarity index 92% rename from pkg/extract/processor/messages.go rename to pkg/extract/chain/messages.go index a3f029fff..8126a1096 100644 --- a/pkg/extract/processor/messages.go +++ b/pkg/extract/chain/messages.go @@ -1,4 +1,4 @@ -package processor +package chain import ( "context" @@ -68,13 +68,6 @@ import ( Actors cid.Cid // ActorStateChanges } */ -type MessageStateChanges struct { - Current *types.TipSet - Executed *types.TipSet - BaseFee abi.TokenAmount - FullBlocks map[cid.Cid]*FullBlock - ImplicitMessages []*ImplicitMessage -} type FullBlock struct { Block *types.BlockHeader @@ -135,11 +128,11 @@ type ActorError struct { Error string `cborgen:"error"` } -func Messages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*MessageStateChanges, error) { +func Messages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (map[cid.Cid]*FullBlock, []*ImplicitMessage, error) { // fist get all messages included in the executed tipset, not all of these messages will have receipts since some were not executed. blkMsgs, err := api.TipSetBlockMessages(ctx, executed) if err != nil { - return nil, err + return nil, nil, err } // build two maps containing all signed (secpMsgs) and unsigned (blsMsgs) messages. secpBlkMsgs := make(map[cid.Cid][]*SignedChainMessage) @@ -162,12 +155,12 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type exeBlkMsgs, err := api.TipSetMessageReceipts(ctx, current, executed) if err != nil { - return nil, err + return nil, nil, err } for _, ebm := range exeBlkMsgs { itr, err := ebm.Iterator() if err != nil { - return nil, err + return nil, nil, err } for itr.HasNext() { msg, recIdx, rec := itr.Next() @@ -189,14 +182,14 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type msgExe, err := api.MessageExecutionsV2(ctx, current, executed) if err != nil { - return nil, err + return nil, nil, err } var im []*ImplicitMessage for _, emsg := range msgExe { vmMsgs, err := ProcessVmMessages(ctx, emsg) if err != nil { - return nil, err + return nil, nil, err } if emsg.Implicit { im = append(im, &ImplicitMessage{ @@ -223,26 +216,15 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type } } - baseFee, err := api.ComputeBaseFee(ctx, executed) - if err != nil { - return nil, err - } - - out := &MessageStateChanges{ - Current: current, - Executed: executed, - BaseFee: baseFee, - FullBlocks: make(map[cid.Cid]*FullBlock), - ImplicitMessages: im, - } + fullBlocks := make(map[cid.Cid]*FullBlock) for _, blk := range blkMsgs { - out.FullBlocks[blk.Block.Cid()] = &FullBlock{ + fullBlocks[blk.Block.Cid()] = &FullBlock{ Block: blk.Block, SecpMessages: secpBlkMsgs[blk.Block.Cid()], BlsMessages: blsBlkMsgs[blk.Block.Cid()], } } - return out, nil + return fullBlocks, im, nil } func GetMessageGasOutputs(msg *lens.MessageExecutionV2) *MessageGasOutputs { diff --git a/pkg/extract/processor/vm.go b/pkg/extract/chain/vm.go similarity index 99% rename from pkg/extract/processor/vm.go rename to pkg/extract/chain/vm.go index b66cd3d2a..54815fa5e 100644 --- a/pkg/extract/processor/vm.go +++ b/pkg/extract/chain/vm.go @@ -1,4 +1,4 @@ -package processor +package chain import ( "context" diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go new file mode 100644 index 000000000..563f98d4f --- /dev/null +++ b/pkg/extract/extract.go @@ -0,0 +1,67 @@ +package extract + +import ( + "context" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/network" + "github.com/filecoin-project/lotus/chain/types" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/tasks" +) + +type ChainState struct { + NetworkName string + NetworkVersion uint64 + ActorVersion uint64 + Message *MessageStateChanges + Actors *ActorStateChanges +} + +func State(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*ChainState, error) { + var ( + blockmessages *MessageStateChanges + actorChanges *ActorStateChanges + err error + ) + + networkName, err := api.NetworkName(ctx) + if err != nil { + return nil, err + } + + networkVersion, err := api.NetworkVersion(ctx, current.Key()) + if err != nil { + return nil, err + } + + actorVersion, err := actortypes.VersionForNetwork(network.Version(networkVersion)) + if err != nil { + return nil, err + } + + grp, grpCtx := errgroup.WithContext(ctx) + grp.Go(func() error { + // all blocks, messages, implicit messages, from executed and receipts from current + blockmessages, err = FullBlockMessages(grpCtx, api, current, executed) + return err + }) + grp.Go(func() error { + // all actor changes between current and parent, actor state exists in current. + actorChanges, err = Actors(grpCtx, api, current, executed, actorVersion) + return err + }) + + if err := grp.Wait(); err != nil { + return nil, err + } + + return &ChainState{ + NetworkName: networkName, + NetworkVersion: uint64(networkVersion), + ActorVersion: uint64(actorVersion), + Message: blockmessages, + Actors: actorChanges, + }, nil +} diff --git a/pkg/gen/main.go b/pkg/gen/main.go index a09759a0e..5f8205aab 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract/chain" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/actors" "github.com/filecoin-project/lily/pkg/transform/cbor/messages" @@ -112,13 +112,13 @@ func main() { } if err := cbg.WriteMapEncodersToFile(MessageStatePath, MessageStatePkg, - processor.ChainMessageReceipt{}, - processor.ImplicitMessageReceipt{}, - processor.MessageGasOutputs{}, - processor.ActorError{}, - processor.VmMessage{}, - processor.VmMessageGasTrace{}, - processor.Loc{}, + chain.ChainMessageReceipt{}, + chain.ImplicitMessageReceipt{}, + chain.MessageGasOutputs{}, + chain.ActorError{}, + chain.VmMessage{}, + chain.VmMessageGasTrace{}, + chain.Loc{}, ); err != nil { panic(err) } diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index a21a8acfd..416cf7241 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -11,8 +11,8 @@ import ( "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" + "github.com/filecoin-project/lily/pkg/extract" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/pkg/extract/processor" ) type ActorStateChangesIPLD struct { @@ -58,7 +58,7 @@ func (a *ActorStateChangesIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) erro return nil } -func ProcessActorsStates(ctx context.Context, s store.Store, changes *processor.ActorStateChanges) (*ActorStateChangesIPLD, error) { +func ProcessActorsStates(ctx context.Context, s store.Store, changes *extract.ActorStateChanges) (*ActorStateChangesIPLD, error) { out := &ActorStateChangesIPLD{} // TODO DataCap diff --git a/pkg/transform/cbor/actors/verifreg/state.go b/pkg/transform/cbor/actors/verifreg/state.go index 3ee098d9a..032a022a4 100644 --- a/pkg/transform/cbor/actors/verifreg/state.go +++ b/pkg/transform/cbor/actors/verifreg/state.go @@ -3,18 +3,16 @@ package verifreg import ( "context" - "github.com/filecoin-project/lotus/blockstore" - adt2 "github.com/filecoin-project/specs-actors/v3/actors/util/adt" + "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/pkg/extract/actors" ) -func HandleChanges(ctx context.Context, bs blockstore.Blockstore, verifreg actors.ActorDiffResult) (cid.Cid, error) { - store := adt2.WrapBlockStore(ctx, bs) - vsc, err := verifreg.MarshalStateChange(ctx, bs) +func HandleChanges(ctx context.Context, s store.Store, verifreg actors.ActorDiffResult) (cid.Cid, error) { + vsc, err := verifreg.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err } - return store.Put(ctx, vsc) + return s.Put(ctx, vsc) } diff --git a/pkg/transform/cbor/messages/cbor_gen.go b/pkg/transform/cbor/messages/cbor_gen.go index 41f77bac1..5ee42ca6a 100644 --- a/pkg/transform/cbor/messages/cbor_gen.go +++ b/pkg/transform/cbor/messages/cbor_gen.go @@ -8,11 +8,12 @@ import ( "math" "sort" - processor "github.com/filecoin-project/lily/pkg/extract/processor" types "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" + + processor "github.com/filecoin-project/lily/pkg/extract/chain" ) var _ = xerrors.Errorf diff --git a/pkg/transform/cbor/messages/serialize.go b/pkg/transform/cbor/messages/serialize.go index 6c2747fbf..fe6c762fb 100644 --- a/pkg/transform/cbor/messages/serialize.go +++ b/pkg/transform/cbor/messages/serialize.go @@ -9,7 +9,7 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract/chain" ) type FullBlockIPLDContainer struct { @@ -18,7 +18,7 @@ type FullBlockIPLDContainer struct { BlsMessages cid.Cid `cborgen:"bls_messages"` } -func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[cid.Cid]*processor.FullBlock) (cid.Cid, error) { +func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[cid.Cid]*chain.FullBlock) (cid.Cid, error) { fullBlkHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -47,12 +47,12 @@ func MakeFullBlockHAMT(ctx context.Context, store adtstore.Store, fullBlks map[c return fullBlkHamt.Root() } -func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) (map[cid.Cid]*processor.FullBlock, error) { +func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) (map[cid.Cid]*chain.FullBlock, error) { fullBlkHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } - out := make(map[cid.Cid]*processor.FullBlock) + out := make(map[cid.Cid]*chain.FullBlock) fbc := new(FullBlockIPLDContainer) if err := fullBlkHamt.ForEach(fbc, func(key string) error { chainMessages, err := DecodeChainMessagesHAMT(ctx, store, fbc.BlsMessages) @@ -66,7 +66,7 @@ func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid bh := new(types.BlockHeader) *bh = *fbc.BlockHeader // TODO assert key == bh.Cid - out[bh.Cid()] = &processor.FullBlock{ + out[bh.Cid()] = &chain.FullBlock{ Block: bh, SecpMessages: signedChainMessages, BlsMessages: chainMessages, @@ -79,12 +79,12 @@ func DecodeFullBlockHAMT(ctx context.Context, store adtstore.Store, root cid.Cid } type ChainMessageIPLDContainer struct { - Message *types.Message `cborgen:"message"` - Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` - VmMessagesAmt cid.Cid `cborgen:"vm_messages"` + Message *types.Message `cborgen:"message"` + Receipt *chain.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ChainMessage) (cid.Cid, error) { +func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.ChainMessage) (cid.Cid, error) { messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -106,24 +106,29 @@ func MakeChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages [ return messageHamt.Root() } -func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ChainMessage, error) { +func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.ChainMessage, error) { messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } - var out []*processor.ChainMessage + var out []*chain.ChainMessage mc := new(ChainMessageIPLDContainer) if err := messagesHamt.ForEach(mc, func(key string) error { - msgs := *mc.Message - rcpts := *mc.Receipt - vmMessages, err := processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + var msg types.Message + var rec chain.ChainMessageReceipt + + msg = *mc.Message + if mc.Receipt != nil { + rec = *mc.Receipt + } + vmMessages, err := chain.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, &processor.ChainMessage{ - Message: &msgs, - Receipt: &rcpts, + out = append(out, &chain.ChainMessage{ + Message: &msg, + Receipt: &rec, VmMessages: vmMessages, }) return nil @@ -134,12 +139,12 @@ func DecodeChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid } type SignedChainMessageIPLDContainer struct { - Message *types.SignedMessage `cborgen:"message"` - Receipt *processor.ChainMessageReceipt `cborgen:"receipt"` - VmMessagesAmt cid.Cid `cborgen:"vm_messages"` + Message *types.SignedMessage `cborgen:"message"` + Receipt *chain.ChainMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } -func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.SignedChainMessage) (cid.Cid, error) { +func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.SignedChainMessage) (cid.Cid, error) { messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -161,24 +166,29 @@ func MakeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, mess return messageHamt.Root() } -func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.SignedChainMessage, error) { +func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.SignedChainMessage, error) { messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } - var out []*processor.SignedChainMessage + var out []*chain.SignedChainMessage mc := new(SignedChainMessageIPLDContainer) if err := messagesHamt.ForEach(mc, func(key string) error { - msg := *mc.Message - recet := *mc.Receipt - vmMessages, err := processor.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) + var msg types.SignedMessage + var rec chain.ChainMessageReceipt + + msg = *mc.Message + if mc.Receipt != nil { + rec = *mc.Receipt + } + vmMessages, err := chain.VmMessageListFromAdtArray(store, mc.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, &processor.SignedChainMessage{ + out = append(out, &chain.SignedChainMessage{ Message: &msg, - Receipt: &recet, + Receipt: &rec, VmMessages: vmMessages, }) return nil @@ -189,13 +199,13 @@ func DecodeSignedChainMessagesHAMT(ctx context.Context, store adtstore.Store, ro } type ImplicitMessageIPLDContainer struct { - Message *types.Message `cborgen:"message"` - Receipt *processor.ImplicitMessageReceipt `cborgen:"receipt"` - VmMessagesAmt cid.Cid `cborgen:"vm_messages"` + Message *types.Message `cborgen:"message"` + Receipt *chain.ImplicitMessageReceipt `cborgen:"receipt"` + VmMessagesAmt cid.Cid `cborgen:"vm_messages"` } // MakeImplicitMessagesHAMT returns the root of a hamt node containing the set of implicit messages -func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*processor.ImplicitMessage) (cid.Cid, error) { +func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, messages []*chain.ImplicitMessage) (cid.Cid, error) { messageHamt, err := adt.MakeEmptyMap(store, 5) if err != nil { return cid.Undef, err @@ -217,22 +227,22 @@ func MakeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, message return messageHamt.Root() } -func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*processor.ImplicitMessage, error) { +func DecodeImplicitMessagesHAMT(ctx context.Context, store adtstore.Store, root cid.Cid) ([]*chain.ImplicitMessage, error) { messagesHamt, err := adt.AsMap(store, root, 5) if err != nil { return nil, err } - var out []*processor.ImplicitMessage + var out []*chain.ImplicitMessage msg := new(ImplicitMessageIPLDContainer) if err := messagesHamt.ForEach(msg, func(key string) error { m := *msg.Message rect := *msg.Receipt - vmMessages, err := processor.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) + vmMessages, err := chain.VmMessageListFromAdtArray(store, msg.VmMessagesAmt, 5) if err != nil { return err } - out = append(out, &processor.ImplicitMessage{ + out = append(out, &chain.ImplicitMessage{ Message: &m, Receipt: &rect, VmMessages: vmMessages, diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index ea2daaa8c..2f2be3816 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -16,7 +16,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract" cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" messages2 "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) @@ -77,10 +77,10 @@ func (s *StateExtractionIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error return nil } -func WriteCar(ctx context.Context, root cid.Cid, carVersion uint64, bs blockstore.Blockstore, w io.Writer) error { +func WriteCarV1(ctx context.Context, root cid.Cid, bs blockstore.Blockstore, w io.Writer) error { if err := v1car.WriteHeader(&v1car.CarHeader{ Roots: []cid.Cid{root}, - Version: carVersion, + Version: 1, }, w); err != nil { return err } @@ -102,28 +102,28 @@ func WriteCar(ctx context.Context, root cid.Cid, carVersion uint64, bs blockstor return nil } -func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, messages *processor.MessageStateChanges, actors *processor.ActorStateChanges) (cid.Cid, error) { +func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, chainState *extract.ChainState) (cid.Cid, error) { store := adtStore.WrapBlockStore(ctx, bs) // sanity check - if !messages.Current.Equals(actors.Current) { + if !chainState.Message.Current.Equals(chainState.Actors.Current) { return cid.Undef, fmt.Errorf("actor and message current tipset does not match") } - if !messages.Executed.Equals(actors.Executed) { + if !chainState.Message.Executed.Equals(chainState.Actors.Executed) { return cid.Undef, fmt.Errorf("actor and message executed tipset does not match") } - implicitMsgsAMT, err := messages2.MakeImplicitMessagesHAMT(ctx, store, messages.ImplicitMessages) + implicitMsgsAMT, err := messages2.MakeImplicitMessagesHAMT(ctx, store, chainState.Message.ImplicitMessages) if err != nil { return cid.Undef, err } - fullBlkHAMT, err := messages2.MakeFullBlockHAMT(ctx, store, messages.FullBlocks) + fullBlkHAMT, err := messages2.MakeFullBlockHAMT(ctx, store, chainState.Message.FullBlocks) if err != nil { return cid.Undef, err } - actorStateContainer, err := cboractors.ProcessActorsStates(ctx, store, actors) + actorStateContainer, err := cboractors.ProcessActorsStates(ctx, store, chainState.Actors) if err != nil { return cid.Undef, err } @@ -136,7 +136,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec extractedState := &StateExtractionIPLD{ Current: *current, Parent: *executed, - BaseFee: messages.BaseFee, + BaseFee: chainState.Message.BaseFee, FullBlocks: fullBlkHAMT, ImplicitMessages: implicitMsgsAMT, Actors: actorStatesRoot, @@ -148,8 +148,10 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec } rootState := &RootStateIPLD{ - StateVersion: 0, - State: extractedStateRoot, + NetworkVersion: chainState.NetworkVersion, + NetworkName: chainState.NetworkName, + StateVersion: 0, + State: extractedStateRoot, } root, err := store.Put(ctx, rootState) diff --git a/pkg/transform/timescale/fullblock/blocks.go b/pkg/transform/timescale/fullblock/blocks.go index e952858e8..ea5d5d04c 100644 --- a/pkg/transform/timescale/fullblock/blocks.go +++ b/pkg/transform/timescale/fullblock/blocks.go @@ -7,10 +7,10 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/model/blocks" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract/chain" ) -func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { +func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { out := blocks.BlockHeaders{} for _, fb := range fullBlocks { out = append(out, &blocks.BlockHeader{ @@ -28,7 +28,7 @@ func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*processor. return out, nil } -func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { +func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { out := blocks.BlockParents{} for _, fb := range fullBlocks { for _, p := range fb.Block.Parents { diff --git a/pkg/transform/timescale/fullblock/messages.go b/pkg/transform/timescale/fullblock/messages.go index 4528bd723..e071e9737 100644 --- a/pkg/transform/timescale/fullblock/messages.go +++ b/pkg/transform/timescale/fullblock/messages.go @@ -7,10 +7,10 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/model/messages" - "github.com/filecoin-project/lily/pkg/extract/processor" + "github.com/filecoin-project/lily/pkg/extract/chain" ) -func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { +func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { out := messages.Messages{} for _, fb := range fullBlocks { for _, msg := range fb.SecpMessages { @@ -47,7 +47,7 @@ func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*processor.Full return out, nil } -func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*processor.FullBlock) (model.Persistable, error) { +func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { out := messages.VMMessageList{} for _, fb := range fullBlocks { for _, msg := range fb.SecpMessages { diff --git a/schemas/v1/base.go b/schemas/v1/base.go index 4845d3cf9..f8e61fffe 100644 --- a/schemas/v1/base.go +++ b/schemas/v1/base.go @@ -596,7 +596,7 @@ COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.prov COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.start_epoch IS 'The epoch at which this deal with begin. Storage deal must appear in a sealed (proven) sector no later than start_epoch, otherwise it is invalid.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.end_epoch IS 'The epoch at which this deal with end.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.storage_price_per_epoch IS 'The amount of FIL (in attoFIL) that will be transferred from the client to the provider every epoch this deal is active for.'; -COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.provider_collateral IS 'The amount of FIL (in attoFIL) the provider has pledged as collateral. The Provider deal collateral is only slashed when a sector is terminated before the deal expires.'; +COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.provider_collateral IS 'The amount of FIL (in attoFIL) the provider has pledged as collateral. The Client deal collateral is only slashed when a sector is terminated before the deal expires.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.client_collateral IS 'The amount of FIL (in attoFIL) the client has pledged as collateral.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.label IS 'An arbitrary client chosen label to apply to the deal.'; COMMENT ON COLUMN {{ .SchemaName | default "public"}}.market_deal_proposals.height IS 'Epoch at which this deal proposal was added or changed.'; diff --git a/tasks/api.go b/tasks/api.go index 21a5460d8..e8da0ddcf 100644 --- a/tasks/api.go +++ b/tasks/api.go @@ -70,4 +70,7 @@ type DataSource interface { ShouldBurnFn(ctx context.Context, ts *types.TipSet) (lens.ShouldBurnFn, error) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error) + + NetworkName(ctx context.Context) (string, error) + NetworkVersion(ctx context.Context, tsk types.TipSetKey) (uint, error) } diff --git a/tasks/messages/message_test.go b/tasks/messages/message_test.go index 436af4c2e..0a531baf4 100644 --- a/tasks/messages/message_test.go +++ b/tasks/messages/message_test.go @@ -174,7 +174,7 @@ func TestParseMessageParams(t *testing.T) { }, }), wantMethod: "PublishStorageDeals", - wantEncoded: `{"Deals":[{"Proposal":{"PieceCID":{"/":"baga6ea4seaqgqzxo27ongakwwef5x3cihl6fgritvgeq5akvjqij6lpgofsogiq"},"PieceSize":1310720,"VerifiedDeal":false,"Client":"f1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i","Provider":"f08178",` + + wantEncoded: `{"Deals":[{"Proposal":{"PieceCID":{"/":"baga6ea4seaqgqzxo27ongakwwef5x3cihl6fgritvgeq5akvjqij6lpgofsogiq"},"PieceSize":1310720,"VerifiedDeal":false,"Client":"f1nslxql4pck5pq7hddlzym3orxlx35wkepzjkm3i","Client":"f08178",` + `"Label":"\\ufffdepcids\\ufffd\\ufffd*X'\\u0000\\u0001U\\ufffd\\ufffd\\u0002 \\u0011;\\u0012\\ufffd\\ufffd0\\ufffd3\\ufffdMA\\ufffd\\ufffd}b\\ufffd\\rf\\ufffdmX\\u001b\u003e\\ufffd\\ufffdm\\ufffdÛ¬\\ufffd\\ufffd\\ufffd",` + `"StartEpoch":475750,"EndEpoch":750173,"StoragePricePerEpoch":{"Int":61035},"ProviderCollateral":{"Int":0},"ClientCollateral":{"Int":0}},"ClientSignature":{"Type":1,"Data":"9a8sdvutVlu0fizD0JmqZjKJaQLj3W3ZtJ2yTReIry8kZ8cDa33V3Pe0sdZzSjz9mRdM/KPm1jL/PZhqpDeYNwE="}}]}`, wantErr: false, From 588752bd373e1918ed34da6873b94cfff889fa76 Mon Sep 17 00:00:00 2001 From: frrist Date: Mon, 23 Jan 2023 10:54:40 -0800 Subject: [PATCH 31/48] remove unused actor seralize handlers --- pkg/transform/cbor/actors/init_/state.go | 19 ------------ pkg/transform/cbor/actors/market/state.go | 19 ------------ pkg/transform/cbor/actors/miner/state.go | 31 ------------------- pkg/transform/cbor/actors/power/state.go | 18 ----------- pkg/transform/cbor/actors/raw/state.go | 33 --------------------- pkg/transform/cbor/actors/verifreg/state.go | 18 ----------- 6 files changed, 138 deletions(-) delete mode 100644 pkg/transform/cbor/actors/init_/state.go delete mode 100644 pkg/transform/cbor/actors/market/state.go delete mode 100644 pkg/transform/cbor/actors/miner/state.go delete mode 100644 pkg/transform/cbor/actors/power/state.go delete mode 100644 pkg/transform/cbor/actors/raw/state.go delete mode 100644 pkg/transform/cbor/actors/verifreg/state.go diff --git a/pkg/transform/cbor/actors/init_/state.go b/pkg/transform/cbor/actors/init_/state.go deleted file mode 100644 index 72388ce84..000000000 --- a/pkg/transform/cbor/actors/init_/state.go +++ /dev/null @@ -1,19 +0,0 @@ -package init_ - -import ( - "context" - - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/store" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChanges(ctx context.Context, s store.Store, addresses actors.ActorDiffResult) (cid.Cid, error) { - isc, err := addresses.MarshalStateChange(ctx, s) - if err != nil { - return cid.Undef, err - } - return s.Put(ctx, isc) -} diff --git a/pkg/transform/cbor/actors/market/state.go b/pkg/transform/cbor/actors/market/state.go deleted file mode 100644 index 74a07f87c..000000000 --- a/pkg/transform/cbor/actors/market/state.go +++ /dev/null @@ -1,19 +0,0 @@ -package market - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChange(ctx context.Context, s store.Store, market actors.ActorDiffResult) (cid.Cid, error) { - vsc, err := market.MarshalStateChange(ctx, s) - if err != nil { - return cid.Undef, err - } - return s.Put(ctx, vsc) - -} diff --git a/pkg/transform/cbor/actors/miner/state.go b/pkg/transform/cbor/actors/miner/state.go deleted file mode 100644 index fe134a255..000000000 --- a/pkg/transform/cbor/actors/miner/state.go +++ /dev/null @@ -1,31 +0,0 @@ -package miner - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/filecoin-project/go-state-types/store" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChanges(ctx context.Context, s store.Store, miners map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { - minerHamt, err := adt.MakeEmptyMap(s, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for addr, change := range miners { - msc, err := change.MarshalStateChange(ctx, s) - if err != nil { - return cid.Undef, err - } - - if err := minerHamt.Put(abi.AddrKey(addr), msc); err != nil { - return cid.Undef, err - } - } - return minerHamt.Root() -} diff --git a/pkg/transform/cbor/actors/power/state.go b/pkg/transform/cbor/actors/power/state.go deleted file mode 100644 index 3ac0c2ff2..000000000 --- a/pkg/transform/cbor/actors/power/state.go +++ /dev/null @@ -1,18 +0,0 @@ -package power - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChange(ctx context.Context, s store.Store, power actors.ActorDiffResult) (cid.Cid, error) { - vsc, err := power.MarshalStateChange(ctx, s) - if err != nil { - return cid.Undef, err - } - return s.Put(ctx, vsc) -} diff --git a/pkg/transform/cbor/actors/raw/state.go b/pkg/transform/cbor/actors/raw/state.go deleted file mode 100644 index 8f2f06d0d..000000000 --- a/pkg/transform/cbor/actors/raw/state.go +++ /dev/null @@ -1,33 +0,0 @@ -package raw - -import ( - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - store2 "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/blockstore" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChanges(ctx context.Context, bs blockstore.Blockstore, actors map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { - store := store2.WrapBlockStore(ctx, bs) - actorHamt, err := adt.MakeEmptyMap(store, 5 /*TODO*/) - if err != nil { - return cid.Undef, err - } - for addr, change := range actors { - msc, err := change.MarshalStateChange(ctx, bs) - if err != nil { - return cid.Undef, err - } - - if err := actorHamt.Put(abi.AddrKey(addr), msc); err != nil { - return cid.Undef, err - } - } - return actorHamt.Root() -} diff --git a/pkg/transform/cbor/actors/verifreg/state.go b/pkg/transform/cbor/actors/verifreg/state.go deleted file mode 100644 index 032a022a4..000000000 --- a/pkg/transform/cbor/actors/verifreg/state.go +++ /dev/null @@ -1,18 +0,0 @@ -package verifreg - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/pkg/extract/actors" -) - -func HandleChanges(ctx context.Context, s store.Store, verifreg actors.ActorDiffResult) (cid.Cid, error) { - vsc, err := verifreg.MarshalStateChange(ctx, s) - if err != nil { - return cid.Undef, err - } - return s.Put(ctx, vsc) -} From af687775176d6401e4a0819df1d473ecfff17c0e Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 24 Jan 2023 09:36:26 -0800 Subject: [PATCH 32/48] wip --- .../actors/builtin/datacap/actor.go.template | 5 + chain/actors/builtin/datacap/datacap.go | 8 +- .../actors/builtin/datacap/state.go.template | 40 ++ chain/actors/builtin/datacap/v9.go | 43 +- pkg/extract/actors/datacap/v9/allowances.go | 1 - .../actors/datacapdiff/v9/allowances.go | 125 +++++ .../{datacap => datacapdiff}/v9/balances.go | 14 +- pkg/extract/actors/datacapdiff/v9/cbor_gen.go | 449 ++++++++++++++++++ .../{datacap => datacapdiff}/v9/load.go | 13 + .../{datacap => datacapdiff}/v9/state.go | 0 .../actors/verifregdiff/v9/cbor_gen.go | 8 +- pkg/gen/main.go | 14 +- 12 files changed, 707 insertions(+), 13 deletions(-) delete mode 100644 pkg/extract/actors/datacap/v9/allowances.go create mode 100644 pkg/extract/actors/datacapdiff/v9/allowances.go rename pkg/extract/actors/{datacap => datacapdiff}/v9/balances.go (80%) create mode 100644 pkg/extract/actors/datacapdiff/v9/cbor_gen.go rename pkg/extract/actors/{datacap => datacapdiff}/v9/load.go (64%) rename pkg/extract/actors/{datacap => datacapdiff}/v9/state.go (100%) diff --git a/chain/actors/builtin/datacap/actor.go.template b/chain/actors/builtin/datacap/actor.go.template index 82b00e27b..012fc7f43 100644 --- a/chain/actors/builtin/datacap/actor.go.template +++ b/chain/actors/builtin/datacap/actor.go.template @@ -64,6 +64,11 @@ type State interface { VerifiedClients() (adt.Map, error) VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + + AllowanceMap() (adt.Map, error) + AllowanceMapBitWidth() int + AllowanceMapHashFunction() func(input []byte) []byte + AllowanceMapForOwner(owner address.Address) (adt.Map, error) } func AllCodes() []cid.Cid { diff --git a/chain/actors/builtin/datacap/datacap.go b/chain/actors/builtin/datacap/datacap.go index b71cd602d..fea665cac 100644 --- a/chain/actors/builtin/datacap/datacap.go +++ b/chain/actors/builtin/datacap/datacap.go @@ -10,10 +10,11 @@ import ( "github.com/filecoin-project/go-state-types/cbor" "github.com/ipfs/go-cid" - "github.com/filecoin-project/lily/chain/actors" lotusactors "github.com/filecoin-project/lotus/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/actors" ) var ( @@ -65,6 +66,11 @@ type State interface { VerifiedClients() (adt.Map, error) VerifiedClientsMapBitWidth() int VerifiedClientsMapHashFunction() func(input []byte) []byte + + AllowanceMap() (adt.Map, error) + AllowanceMapBitWidth() int + AllowanceMapHashFunction() func(input []byte) []byte + AllowanceMapForOwner(owner address.Address) (adt.Map, error) } func AllCodes() []cid.Cid { diff --git a/chain/actors/builtin/datacap/state.go.template b/chain/actors/builtin/datacap/state.go.template index e45b65390..94c4226f9 100644 --- a/chain/actors/builtin/datacap/state.go.template +++ b/chain/actors/builtin/datacap/state.go.template @@ -74,6 +74,30 @@ func (s *state{{.v}}) VerifiedClientsMapHashFunction() func(input []byte) []byte } } +func (s *state{{.v}}) AllowanceMap() (adt.Map, error) { + return adt{{.v}}.AsMap(s.store, s.Token.Allowances, int(s.Token.HamtBitWidth)) +} + +func (s *state{{.v}}) AllowanceMapBitWidth() int { + return int(s.Token.HamtBitWidth) +} + +func (s *state{{.v}}) AllowanceMapHashFunction() func(input []byte) []byte { + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +} + +func (s *state{{.v}}) AllowanceMapForOwner(owner address.Address) (adt.Map, error) { + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(owner), s.Token.Allowances, int(s.Token.HamtBitWidth)) + if err != nil { + return nil, err + } + return adt{{.v}}.AsMap(s.store, innerHamtCid, int(s.Token.HamtBitWidth)) +} + + func (s *state{{.v}}) ActorKey() string { return actors.DatacapKey } @@ -90,3 +114,19 @@ func (s *state{{.v}}) Code() cid.Cid { return code } + +func (s *state{{.v}}) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + actorToHamtMap, err := adt{{.v}}.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +} diff --git a/chain/actors/builtin/datacap/v9.go b/chain/actors/builtin/datacap/v9.go index 90ab8ed98..ce6ad13d4 100644 --- a/chain/actors/builtin/datacap/v9.go +++ b/chain/actors/builtin/datacap/v9.go @@ -7,10 +7,12 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/lily/chain/actors" "github.com/filecoin-project/lotus/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors" + datacap9 "github.com/filecoin-project/go-state-types/builtin/v9/datacap" adt9 "github.com/filecoin-project/go-state-types/builtin/v9/util/adt" ) @@ -74,6 +76,29 @@ func (s *state9) VerifiedClientsMapHashFunction() func(input []byte) []byte { } } +func (s *state9) AllowanceMap() (adt.Map, error) { + return adt9.AsMap(s.store, s.Token.Allowances, int(s.Token.HamtBitWidth)) +} + +func (s *state9) AllowanceMapBitWidth() int { + return int(s.Token.HamtBitWidth) +} + +func (s *state9) AllowanceMapHashFunction() func(input []byte) []byte { + return func(input []byte) []byte { + res := sha256.Sum256(input) + return res[:] + } +} + +func (s *state9) AllowanceMapForOwner(owner address.Address) (adt.Map, error) { + innerHamtCid, err := s.getInnerHamtCid(s.store, abi.IdAddrKey(owner), s.Token.Allowances, int(s.Token.HamtBitWidth)) + if err != nil { + return nil, err + } + return adt9.AsMap(s.store, innerHamtCid, int(s.Token.HamtBitWidth)) +} + func (s *state9) ActorKey() string { return actors.DatacapKey } @@ -90,3 +115,19 @@ func (s *state9) Code() cid.Cid { return code } + +func (s *state9) getInnerHamtCid(store adt.Store, key abi.Keyer, mapCid cid.Cid, bitwidth int) (cid.Cid, error) { + actorToHamtMap, err := adt9.AsMap(store, mapCid, bitwidth) + if err != nil { + return cid.Undef, fmt.Errorf("couldn't get outer map: %x", err) + } + + var innerHamtCid cbg.CborCid + if found, err := actorToHamtMap.Get(key, &innerHamtCid); err != nil { + return cid.Undef, fmt.Errorf("looking up key: %s: %w", key, err) + } else if !found { + return cid.Undef, fmt.Errorf("did not find key: %s", key) + } + + return cid.Cid(innerHamtCid), nil +} diff --git a/pkg/extract/actors/datacap/v9/allowances.go b/pkg/extract/actors/datacap/v9/allowances.go deleted file mode 100644 index be1c6b564..000000000 --- a/pkg/extract/actors/datacap/v9/allowances.go +++ /dev/null @@ -1 +0,0 @@ -package v9 diff --git a/pkg/extract/actors/datacapdiff/v9/allowances.go b/pkg/extract/actors/datacapdiff/v9/allowances.go new file mode 100644 index 000000000..20600dff1 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v9/allowances.go @@ -0,0 +1,125 @@ +package v9 + +import ( + "context" + "time" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/abi" + "github.com/ipfs/go-cid" + typegen "github.com/whyrusleeping/cbor-gen" + "go.uber.org/zap" + + adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + + "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/lily/chain/actors/builtin/datacap" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/generic" + "github.com/filecoin-project/lily/tasks" +) + +type AllowanceChange struct { + Owner []byte `cborgen:"owner"` + Operator []byte `cborgen:"operator"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` +} + +type AllowanceChangeMap map[address.Address][]*AllowanceChange + +const KindDataCapAllowance = "datacap_allowance" + +func (a AllowanceChangeMap) Kind() actors.ActorStateKind { + return KindDataCapAllowance +} + +func (a AllowanceChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + topNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for client, changes := range a { + innerNode, err := adt2.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, change := range changes { + if err := innerNode.Put(core.StringKey(change.Owner), change); err != nil { + return cid.Undef, err + } + } + innerRoot, err := innerNode.Root() + if err != nil { + return cid.Undef, err + } + if err := topNode.Put(abi.IdAddrKey(client), typegen.CborCid(innerRoot)); err != nil { + return cid.Undef, err + } + } + return topNode.Root() +} + +type Allowance struct{} + +func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { + start := time.Now() + defer func() { + log.Debugw("Diff", "kind", KindDataCapAllowance, zap.Inline(act), "duration", time.Since(start)) + }() + return DiffAllowances(ctx, api, act) +} + +func DiffAllowances(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (AllowanceChangeMap, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapAllowancesMapLoader) + if err != nil { + return nil, err + } + out := make(AllowanceChangeMap) + for _, change := range mapChange { + ownerId, err := abi.ParseUIntKey(string(change.Key)) + if err != nil { + return nil, err + } + ownerAddress, err := address.NewIDAddress(ownerId) + if err != nil { + return nil, err + } + ownerMapChanges, err := diffOwnerMap(ctx, api, act, ownerAddress, change.Key) + if err != nil { + return nil, err + } + out[ownerAddress] = ownerMapChanges + } + return out, nil +} + +func diffOwnerMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, ownerAddress address.Address, ownerKey []byte) ([]*AllowanceChange, error) { + mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := i.(datacap.State) + clientAllocationMap, err := datacapState.AllowanceMapForOwner(ownerAddress) + if err != nil { + return nil, nil, err + } + return clientAllocationMap, &adt.MapOpts{ + Bitwidth: datacapState.AllowanceMapBitWidth(), + HashFunc: datacapState.AllowanceMapHashFunction(), + }, nil + }) + if err != nil { + return nil, err + } + out := make([]*AllowanceChange, 0, len(mapChange)) + for _, change := range mapChange { + out = append(out, &AllowanceChange{ + Owner: ownerKey, + Operator: change.Key, + Current: change.Current, + Previous: change.Previous, + Change: change.Type, + }) + } + return out, nil +} diff --git a/pkg/extract/actors/datacap/v9/balances.go b/pkg/extract/actors/datacapdiff/v9/balances.go similarity index 80% rename from pkg/extract/actors/datacap/v9/balances.go rename to pkg/extract/actors/datacapdiff/v9/balances.go index eb975372a..a354015d7 100644 --- a/pkg/extract/actors/datacap/v9/balances.go +++ b/pkg/extract/actors/datacapdiff/v9/balances.go @@ -4,10 +4,12 @@ import ( "context" "time" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" + "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -17,10 +19,10 @@ import ( var log = logging.Logger("lily/extract/actors/balance/v9") type BalanceChange struct { - Client []byte - Current *typegen.Deferred - Previous *typegen.Deferred - Change core.ChangeType + Client []byte `cborgen:"client"` + Current *typegen.Deferred `cborgen:"current"` + Previous *typegen.Deferred `cborgen:"previous"` + Change core.ChangeType `cborgen:"change"` } type BalanceChangeList []*BalanceChange @@ -31,6 +33,10 @@ func (b BalanceChangeList) Kind() actors.ActorStateKind { return KindDataCapBalance } +func (b BalanceChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { + +} + type Balance struct{} func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { diff --git a/pkg/extract/actors/datacapdiff/v9/cbor_gen.go b/pkg/extract/actors/datacapdiff/v9/cbor_gen.go new file mode 100644 index 000000000..0bd521411 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/v9/cbor_gen.go @@ -0,0 +1,449 @@ +// Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. + +package v9 + +import ( + "fmt" + "io" + "math" + "sort" + + core "github.com/filecoin-project/lily/pkg/core" + cid "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + xerrors "golang.org/x/xerrors" +) + +var _ = xerrors.Errorf +var _ = cid.Undef +var _ = math.E +var _ = sort.Sort + +func (t *AllowanceChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{165}); err != nil { + return err + } + + // t.Owner ([]uint8) (slice) + if len("owner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"owner\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("owner"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("owner")); err != nil { + return err + } + + if len(t.Owner) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Owner was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Owner))); err != nil { + return err + } + + if _, err := cw.Write(t.Owner[:]); err != nil { + return err + } + + // t.Operator ([]uint8) (slice) + if len("operator") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"operator\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("operator"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("operator")); err != nil { + return err + } + + if len(t.Operator) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Operator was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Operator))); err != nil { + return err + } + + if _, err := cw.Write(t.Operator[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *AllowanceChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = AllowanceChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("AllowanceChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Owner ([]uint8) (slice) + case "owner": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Owner: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Owner = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Owner[:]); err != nil { + return err + } + // t.Operator ([]uint8) (slice) + case "operator": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Operator: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Operator = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Operator[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} +func (t *BalanceChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{164}); err != nil { + return err + } + + // t.Client ([]uint8) (slice) + if len("client") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"client\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("client"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("client")); err != nil { + return err + } + + if len(t.Client) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Client was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Client))); err != nil { + return err + } + + if _, err := cw.Write(t.Client[:]); err != nil { + return err + } + + // t.Current (typegen.Deferred) (struct) + if len("current") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"current\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("current"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("current")); err != nil { + return err + } + + if err := t.Current.MarshalCBOR(cw); err != nil { + return err + } + + // t.Previous (typegen.Deferred) (struct) + if len("previous") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"previous\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("previous"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("previous")); err != nil { + return err + } + + if err := t.Previous.MarshalCBOR(cw); err != nil { + return err + } + + // t.Change (core.ChangeType) (uint8) + if len("change") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"change\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("change"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("change")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajUnsignedInt, uint64(t.Change)); err != nil { + return err + } + return nil +} + +func (t *BalanceChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = BalanceChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("BalanceChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Client ([]uint8) (slice) + case "client": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + + if extra > cbg.ByteArrayMaxLen { + return fmt.Errorf("t.Client: byte array too large (%d)", extra) + } + if maj != cbg.MajByteString { + return fmt.Errorf("expected byte array") + } + + if extra > 0 { + t.Client = make([]uint8, extra) + } + + if _, err := io.ReadFull(cr, t.Client[:]); err != nil { + return err + } + // t.Current (typegen.Deferred) (struct) + case "current": + + { + + t.Current = new(cbg.Deferred) + + if err := t.Current.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Previous (typegen.Deferred) (struct) + case "previous": + + { + + t.Previous = new(cbg.Deferred) + + if err := t.Previous.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("failed to read deferred field: %w", err) + } + } + // t.Change (core.ChangeType) (uint8) + case "change": + + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } + if maj != cbg.MajUnsignedInt { + return fmt.Errorf("wrong type for uint8 field") + } + if extra > math.MaxUint8 { + return fmt.Errorf("integer in input was too large for uint8 field") + } + t.Change = core.ChangeType(extra) + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/datacap/v9/load.go b/pkg/extract/actors/datacapdiff/v9/load.go similarity index 64% rename from pkg/extract/actors/datacap/v9/load.go rename to pkg/extract/actors/datacapdiff/v9/load.go index d44beb5af..67cc77862 100644 --- a/pkg/extract/actors/datacap/v9/load.go +++ b/pkg/extract/actors/datacapdiff/v9/load.go @@ -22,3 +22,16 @@ var DatacapBalancesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error HashFunc: datacapState.VerifiedClientsMapHashFunction(), }, nil } + +var DatacapAllowancesMapLoader = func(m interface{}) (adt.Map, *adt.MapOpts, error) { + datacapState := m.(datacap.State) + allowanceMap, err := datacapState.AllowanceMap() + if err != nil { + return nil, nil, err + } + return allowanceMap, &adt.MapOpts{ + Bitwidth: datacapState.AllowanceMapBitWidth(), + HashFunc: datacapState.VerifiedClientsMapHashFunction(), + }, nil + +} diff --git a/pkg/extract/actors/datacap/v9/state.go b/pkg/extract/actors/datacapdiff/v9/state.go similarity index 100% rename from pkg/extract/actors/datacap/v9/state.go rename to pkg/extract/actors/datacapdiff/v9/state.go diff --git a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go index 6caf28fa1..b13a960bd 100644 --- a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v9/cbor_gen.go @@ -228,7 +228,7 @@ func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { return err } - // t.Client ([]uint8) (slice) + // t.Provider ([]uint8) (slice) if len("provider") > cbg.MaxLength { return xerrors.Errorf("Value in field \"provider\" was too long") } @@ -241,7 +241,7 @@ func (t *ClaimsChange) MarshalCBOR(w io.Writer) error { } if len(t.Provider) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Client was too long") + return xerrors.Errorf("Byte array in field t.Provider was too long") } if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Provider))); err != nil { @@ -364,7 +364,7 @@ func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { } switch name { - // t.Client ([]uint8) (slice) + // t.Provider ([]uint8) (slice) case "provider": maj, extra, err = cr.ReadHeader() @@ -373,7 +373,7 @@ func (t *ClaimsChange) UnmarshalCBOR(r io.Reader) (err error) { } if extra > cbg.ByteArrayMaxLen { - return fmt.Errorf("t.Client: byte array too large (%d)", extra) + return fmt.Errorf("t.Provider: byte array too large (%d)", extra) } if maj != cbg.MajByteString { return fmt.Errorf("expected byte array") diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 5f8205aab..58247cfd6 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -3,6 +3,7 @@ package main import ( cbg "github.com/whyrusleeping/cbor-gen" + datacapV9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v9" initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" @@ -16,8 +17,11 @@ import ( "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) -const actorDiffPath = "pkg/extract/actors/actordiff/cbor_gen.go" -const actorDiffPkg = "actordiff" +const actorDiffPath = "pkg/extract/actors/rawdiff/cbor_gen.go" +const actorDiffPkg = "rawdiff" + +const datacapDiffPath = "pkg/extract/actors/datacapdiff/v9/cbor_gen.go" +const datacapDiffPkg = "v9" const minerDiffPath = "pkg/extract/actors/minerdiff/v0/cbor_gen.go" const minerDiffPkg = "v0" @@ -50,6 +54,12 @@ const RootStatePath = "pkg/transform/cbor/cbor_gen.go" const RootStatePkg = "cbor" func main() { + if err := cbg.WriteMapEncodersToFile(datacapDiffPath, datacapDiffPkg, + datacapV9.AllowanceChange{}, + datacapV9.BalanceChange{}, + ); err != nil { + panic(err) + } if err := cbg.WriteMapEncodersToFile(actorDiffPath, actorDiffPkg, rawdiff.ActorChange{}, rawdiff.StateChange{}, From ad9ac03ce3c55694082cd4dcae6cb19db67c9f21 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 24 Jan 2023 16:53:24 -0800 Subject: [PATCH 33/48] checkpoint, all works --- pkg/extract/actors.go | 201 +++++++++++++----- .../actors/datacapdiff/v9/allowances.go | 6 +- pkg/extract/actors/datacapdiff/v9/balances.go | 14 +- pkg/extract/actors/datacapdiff/v9/cbor_gen.go | 152 +++++++++++++ pkg/extract/actors/datacapdiff/v9/state.go | 81 +++++++ pkg/extract/actors/datacapdiff/version.go | 25 +++ pkg/extract/chain/cbor_gen.go | 20 +- pkg/gen/main.go | 5 +- pkg/transform/cbor/actors/cbor_gen.go | 70 +++--- pkg/transform/cbor/actors/serialize.go | 22 +- pkg/transform/cbor/messages/cbor_gen.go | 21 +- pkg/transform/timescale/processor.go | 14 +- 12 files changed, 497 insertions(+), 134 deletions(-) create mode 100644 pkg/extract/actors/datacapdiff/version.go diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 59c637141..a042dc76f 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/lily/chain/actors/builtin" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" @@ -34,10 +35,11 @@ type ActorStateChanges struct { Executed *types.TipSet RawActors map[address.Address]actors.ActorDiffResult MinerActors map[address.Address]actors.ActorDiffResult - VerifregActor actors.ActorDiffResult + DatacapActor actors.ActorDiffResult InitActor actors.ActorDiffResult - PowerActor actors.ActorDiffResult MarketActor actors.ActorDiffResult + PowerActor actors.ActorDiffResult + VerifregActor actors.ActorDiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { @@ -70,17 +72,76 @@ func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Ver } start := time.Now() // diff the actors state and collect changes + log.Infow("diff actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String()) diffRes, err := diff.State(ctx, api, act) if err != nil { return nil, err } - log.Infow("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "duration", time.Since(start)) + log.Infow("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String(), "duration", time.Since(start)) return &StateDiffResult{ ActorDiff: diffRes, Address: act.Address, }, nil } +func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) (*ActorStateChanges, error) { + actorDiff := &rawdiff.StateDiff{ + DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, + } + actorStateChanges, err := actorDiff.State(ctx, api, act) + if err != nil { + return nil, err + } + results <- &StateDiffResult{ + ActorDiff: actorStateChanges, + Address: act.Address, + } + if core.DataCapCodes.Has(act.Current.Code) { + res, err := DiffActor(ctx, api, version, act, datacapdiff.StateDiffFor) + if err != nil { + return nil, err + } + results <- res + } + if core.MinerCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.VerifregCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.InitCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.PowerCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.MarketCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + return nil + +} + func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { @@ -94,67 +155,78 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. } grp, grpCtx := errgroup.WithContext(ctx) + grp.SetLimit(8) results := make(chan *StateDiffResult, len(actorChanges)) - for addr, change := range actorChanges { - addr := addr - change := change - act := &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - } - - grp.Go(func() error { - actorDiff := &rawdiff.StateDiff{ - DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, + grp.Go(func() error { + for addr, change := range actorChanges { + addr := addr + change := change + act := &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, } - actorStateChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - results <- &StateDiffResult{ - ActorDiff: actorStateChanges, - Address: act.Address, - } - if core.MinerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) - if err != nil { - return err + + grp.Go(func() error { + actorDiff := &rawdiff.StateDiff{ + DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, } - results <- res - } - if core.VerifregCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) + actorStateChanges, err := actorDiff.State(grpCtx, api, act) if err != nil { return err } - results <- res - } - if core.InitCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) - if err != nil { - return err + results <- &StateDiffResult{ + ActorDiff: actorStateChanges, + Address: act.Address, } - results <- res - } - if core.PowerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) - if err != nil { - return err + if core.DataCapCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, datacapdiff.StateDiffFor) + if err != nil { + return err + } + results <- res } - results <- res - } - if core.MarketCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) - if err != nil { - return err + if core.MinerCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) + if err != nil { + return err + } + results <- res } - results <- res - } - return nil - }) - } + if core.VerifregCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.InitCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.PowerCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + if core.MarketCodes.Has(change.Current.Code) { + res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) + if err != nil { + return err + } + results <- res + } + return nil + }) + } + return nil + }) go func() { if err := grp.Wait(); err != nil { log.Error(err) @@ -175,10 +247,29 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. asc.PowerActor = stateDiff.ActorDiff case "market": asc.MarketActor = stateDiff.ActorDiff + case "datacap": + asc.DatacapActor = stateDiff.ActorDiff default: panic(stateDiff.ActorDiff.Kind()) } } return asc, nil +} + +type DiffWorker struct { + workers int +} + +func (w *DiffWorker) Start(ctx context.Context) error { + for i := 0; i < w.workers; i++ { + go func() { + w.work(ctx, nil, nil) + }() + } +} +func (w *DiffWorker) work(ctx context.Context, in, out chan interface{}) { + todo := <-in + time.Sleep(10) + out <- struct{}{} } diff --git a/pkg/extract/actors/datacapdiff/v9/allowances.go b/pkg/extract/actors/datacapdiff/v9/allowances.go index 20600dff1..8509e5cb5 100644 --- a/pkg/extract/actors/datacapdiff/v9/allowances.go +++ b/pkg/extract/actors/datacapdiff/v9/allowances.go @@ -41,13 +41,13 @@ func (a AllowanceChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { if err != nil { return cid.Undef, err } - for client, changes := range a { + for owner, changes := range a { innerNode, err := adt2.MakeEmptyMap(store, bw) if err != nil { return cid.Undef, err } for _, change := range changes { - if err := innerNode.Put(core.StringKey(change.Owner), change); err != nil { + if err := innerNode.Put(core.StringKey(change.Operator), change); err != nil { return cid.Undef, err } } @@ -55,7 +55,7 @@ func (a AllowanceChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { if err != nil { return cid.Undef, err } - if err := topNode.Put(abi.IdAddrKey(client), typegen.CborCid(innerRoot)); err != nil { + if err := topNode.Put(abi.IdAddrKey(owner), typegen.CborCid(innerRoot)); err != nil { return cid.Undef, err } } diff --git a/pkg/extract/actors/datacapdiff/v9/balances.go b/pkg/extract/actors/datacapdiff/v9/balances.go index a354015d7..3ba32bbf0 100644 --- a/pkg/extract/actors/datacapdiff/v9/balances.go +++ b/pkg/extract/actors/datacapdiff/v9/balances.go @@ -9,7 +9,8 @@ import ( typegen "github.com/whyrusleeping/cbor-gen" "go.uber.org/zap" - "github.com/filecoin-project/lily/chain/actors/adt" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" @@ -34,7 +35,16 @@ func (b BalanceChangeList) Kind() actors.ActorStateKind { } func (b BalanceChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { - + node, err := adt.MakeEmptyMap(store, bw) + if err != nil { + return cid.Undef, err + } + for _, l := range b { + if err := node.Put(core.StringKey(l.Client), l); err != nil { + return cid.Undef, err + } + } + return node.Root() } type Balance struct{} diff --git a/pkg/extract/actors/datacapdiff/v9/cbor_gen.go b/pkg/extract/actors/datacapdiff/v9/cbor_gen.go index 0bd521411..eed251d33 100644 --- a/pkg/extract/actors/datacapdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/datacapdiff/v9/cbor_gen.go @@ -447,3 +447,155 @@ func (t *BalanceChange) UnmarshalCBOR(r io.Reader) (err error) { return nil } +func (t *StateChange) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Balances (cid.Cid) (struct) + if len("balances") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"balances\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("balances"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("balances")); err != nil { + return err + } + + if t.Balances == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Balances); err != nil { + return xerrors.Errorf("failed to write cid field t.Balances: %w", err) + } + } + + // t.Allowances (cid.Cid) (struct) + if len("allowances") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"allowances\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("allowances"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("allowances")); err != nil { + return err + } + + if t.Allowances == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if err := cbg.WriteCid(cw, *t.Allowances); err != nil { + return xerrors.Errorf("failed to write cid field t.Allowances: %w", err) + } + } + + return nil +} + +func (t *StateChange) UnmarshalCBOR(r io.Reader) (err error) { + *t = StateChange{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("StateChange: map struct too large (%d)", extra) + } + + var name string + n := extra + + for i := uint64(0); i < n; i++ { + + { + sval, err := cbg.ReadString(cr) + if err != nil { + return err + } + + name = string(sval) + } + + switch name { + // t.Balances (cid.Cid) (struct) + case "balances": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Balances: %w", err) + } + + t.Balances = &c + } + + } + // t.Allowances (cid.Cid) (struct) + case "allowances": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + + c, err := cbg.ReadCid(cr) + if err != nil { + return xerrors.Errorf("failed to read cid field t.Allowances: %w", err) + } + + t.Allowances = &c + } + + } + + default: + // Field doesn't exist on this type, so ignore it + cbg.ScanForLinks(r, func(cid.Cid) {}) + } + } + + return nil +} diff --git a/pkg/extract/actors/datacapdiff/v9/state.go b/pkg/extract/actors/datacapdiff/v9/state.go index be1c6b564..eff42b34e 100644 --- a/pkg/extract/actors/datacapdiff/v9/state.go +++ b/pkg/extract/actors/datacapdiff/v9/state.go @@ -1 +1,82 @@ package v9 + +import ( + "context" + + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/pkg/extract/actors" + "github.com/filecoin-project/lily/tasks" +) + +type StateDiffResult struct { + BalanceChanges BalanceChangeList + AllowanceChanges AllowanceChangeMap +} + +func (sd *StateDiffResult) Kind() string { + return "datacap" +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if balances := sd.BalanceChanges; balances != nil { + root, err := balances.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Balances = &root + } + if allowances := sd.AllowanceChanges; allowances != nil { + root, err := allowances.ToAdtMap(s, 5) + if err != nil { + return nil, err + } + out.Allowances = &root + } + return out, nil +} + +type StateChange struct { + Balances *cid.Cid `cborgen:"balances"` + Allowances *cid.Cid `cborgen:"allowances"` +} + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + return ChangeHandler( + ctx, + api, + act, + func(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch stateChange.Kind() { + case KindDataCapAllowance: + stateDiff.AllowanceChanges = stateChange.(AllowanceChangeMap) + case KindDataCapBalance: + stateDiff.BalanceChanges = stateChange.(BalanceChangeList) + } + } + return stateDiff, nil + }, + s.DiffMethods...) +} + +type StateDiffHandlerFn = func(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) + +func ChangeHandler(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, handler StateDiffHandlerFn, differs ...actors.ActorStateDiff) (actors.ActorDiffResult, error) { + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, differs...) + if err != nil { + return nil, err + } + return handler(results) +} diff --git a/pkg/extract/actors/datacapdiff/version.go b/pkg/extract/actors/datacapdiff/version.go new file mode 100644 index 000000000..056816862 --- /dev/null +++ b/pkg/extract/actors/datacapdiff/version.go @@ -0,0 +1,25 @@ +package datacapdiff + +import ( + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v9" +) + +func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { + switch av { + case actortypes.Version9: + return &v9.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v9.Allowance{}, + v9.Balance{}, + }, + }, nil + case actortypes.Version10: + panic("Not yet implemented") + } + return nil, fmt.Errorf("unsupported actor version %d", av) +} diff --git a/pkg/extract/chain/cbor_gen.go b/pkg/extract/chain/cbor_gen.go index e6f7c5997..46590c24e 100644 --- a/pkg/extract/chain/cbor_gen.go +++ b/pkg/extract/chain/cbor_gen.go @@ -48,7 +48,7 @@ func (t *ChainMessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.GasOutputs (processor.MessageGasOutputs) (struct) + // t.GasOutputs (chain.MessageGasOutputs) (struct) if len("gas") > cbg.MaxLength { return xerrors.Errorf("Value in field \"gas\" was too long") } @@ -64,7 +64,7 @@ func (t *ChainMessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.ActorError (processor.ActorError) (struct) + // t.ActorError (chain.ActorError) (struct) if len("errors") > cbg.MaxLength { return xerrors.Errorf("Value in field \"errors\" was too long") } @@ -152,7 +152,7 @@ func (t *ChainMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.GasOutputs (processor.MessageGasOutputs) (struct) + // t.GasOutputs (chain.MessageGasOutputs) (struct) case "gas": { @@ -172,7 +172,7 @@ func (t *ChainMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.ActorError (processor.ActorError) (struct) + // t.ActorError (chain.ActorError) (struct) case "errors": { @@ -255,7 +255,7 @@ func (t *ImplicitMessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.GasOutputs (processor.MessageGasOutputs) (struct) + // t.GasOutputs (chain.MessageGasOutputs) (struct) if len("gas") > cbg.MaxLength { return xerrors.Errorf("Value in field \"gas\" was too long") } @@ -271,7 +271,7 @@ func (t *ImplicitMessageReceipt) MarshalCBOR(w io.Writer) error { return err } - // t.ActorError (processor.ActorError) (struct) + // t.ActorError (chain.ActorError) (struct) if len("errors") > cbg.MaxLength { return xerrors.Errorf("Value in field \"errors\" was too long") } @@ -337,7 +337,7 @@ func (t *ImplicitMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.GasOutputs (processor.MessageGasOutputs) (struct) + // t.GasOutputs (chain.MessageGasOutputs) (struct) case "gas": { @@ -357,7 +357,7 @@ func (t *ImplicitMessageReceipt) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.ActorError (processor.ActorError) (struct) + // t.ActorError (chain.ActorError) (struct) case "errors": { @@ -1119,7 +1119,7 @@ func (t *VmMessageGasTrace) MarshalCBOR(w io.Writer) error { return err } - // t.Location ([]processor.Loc) (slice) + // t.Location ([]chain.Loc) (slice) if len("location") > cbg.MaxLength { return xerrors.Errorf("Value in field \"location\" was too long") } @@ -1327,7 +1327,7 @@ func (t *VmMessageGasTrace) UnmarshalCBOR(r io.Reader) (err error) { t.Name = string(sval) } - // t.Location ([]processor.Loc) (slice) + // t.Location ([]chain.Loc) (slice) case "location": maj, extra, err = cr.ReadHeader() diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 58247cfd6..e8dbf3fcc 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -44,8 +44,8 @@ const powerDiffPkg = "v0" const IPLDActorContainerPath = "pkg/transform/cbor/actors/cbor_gen.go" const IPLDActorContainerPkg = "actors" -const MessageStatePath = "pkg/extract/processor/cbor_gen.go" -const MessageStatePkg = "processor" +const MessageStatePath = "pkg/extract/chain/cbor_gen.go" +const MessageStatePkg = "chain" const MessageContainerPath = "pkg/transform/cbor/messages/cbor_gen.go" const MessageContainerPkg = "messages" @@ -57,6 +57,7 @@ func main() { if err := cbg.WriteMapEncodersToFile(datacapDiffPath, datacapDiffPkg, datacapV9.AllowanceChange{}, datacapV9.BalanceChange{}, + datacapV9.StateChange{}, ); err != nil { panic(err) } diff --git a/pkg/transform/cbor/actors/cbor_gen.go b/pkg/transform/cbor/actors/cbor_gen.go index a237d6ea5..f686fbb89 100644 --- a/pkg/transform/cbor/actors/cbor_gen.go +++ b/pkg/transform/cbor/actors/cbor_gen.go @@ -31,14 +31,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.DataCapActor (cid.Cid) (struct) - if len("DataCapActor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"DataCapActor\" was too long") + if len("datacap") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"datacap\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("DataCapActor"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("datacap"))); err != nil { return err } - if _, err := io.WriteString(w, string("DataCapActor")); err != nil { + if _, err := io.WriteString(w, string("datacap")); err != nil { return err } @@ -53,14 +53,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.InitActor (cid.Cid) (struct) - if len("InitActor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"InitActor\" was too long") + if len("init") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"init\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("InitActor"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("init"))); err != nil { return err } - if _, err := io.WriteString(w, string("InitActor")); err != nil { + if _, err := io.WriteString(w, string("init")); err != nil { return err } @@ -75,14 +75,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.MarketActor (cid.Cid) (struct) - if len("MarketActor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"MarketActor\" was too long") + if len("market") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"market\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MarketActor"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("market"))); err != nil { return err } - if _, err := io.WriteString(w, string("MarketActor")); err != nil { + if _, err := io.WriteString(w, string("market")); err != nil { return err } @@ -97,14 +97,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.MinerActors (cid.Cid) (struct) - if len("MinerActors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"MinerActors\" was too long") + if len("miner") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"miner\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("MinerActors"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("miner"))); err != nil { return err } - if _, err := io.WriteString(w, string("MinerActors")); err != nil { + if _, err := io.WriteString(w, string("miner")); err != nil { return err } @@ -119,14 +119,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.PowerActor (cid.Cid) (struct) - if len("PowerActor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"PowerActor\" was too long") + if len("power") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"power\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("PowerActor"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("power"))); err != nil { return err } - if _, err := io.WriteString(w, string("PowerActor")); err != nil { + if _, err := io.WriteString(w, string("power")); err != nil { return err } @@ -141,14 +141,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.RawActors (cid.Cid) (struct) - if len("RawActors") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"RawActors\" was too long") + if len("raw") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"raw\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("RawActors"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("raw"))); err != nil { return err } - if _, err := io.WriteString(w, string("RawActors")); err != nil { + if _, err := io.WriteString(w, string("raw")); err != nil { return err } @@ -163,14 +163,14 @@ func (t *ActorStateChangesIPLD) MarshalCBOR(w io.Writer) error { } // t.VerifregActor (cid.Cid) (struct) - if len("VerifregActor") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"VerifregActor\" was too long") + if len("verifreg") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"verifreg\" was too long") } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("VerifregActor"))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("verifreg"))); err != nil { return err } - if _, err := io.WriteString(w, string("VerifregActor")); err != nil { + if _, err := io.WriteString(w, string("verifreg")); err != nil { return err } @@ -226,7 +226,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { switch name { // t.DataCapActor (cid.Cid) (struct) - case "DataCapActor": + case "datacap": { @@ -249,7 +249,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.InitActor (cid.Cid) (struct) - case "InitActor": + case "init": { @@ -272,7 +272,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.MarketActor (cid.Cid) (struct) - case "MarketActor": + case "market": { @@ -295,7 +295,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.MinerActors (cid.Cid) (struct) - case "MinerActors": + case "miner": { @@ -318,7 +318,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.PowerActor (cid.Cid) (struct) - case "PowerActor": + case "power": { @@ -341,7 +341,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.RawActors (cid.Cid) (struct) - case "RawActors": + case "raw": { @@ -364,7 +364,7 @@ func (t *ActorStateChangesIPLD) UnmarshalCBOR(r io.Reader) (err error) { } // t.VerifregActor (cid.Cid) (struct) - case "VerifregActor": + case "verifreg": { diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 416cf7241..fa7b5fc7a 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -16,13 +16,13 @@ import ( ) type ActorStateChangesIPLD struct { - DataCapActor *cid.Cid // DataCap - InitActor *cid.Cid // Init - MarketActor *cid.Cid // Market - MinerActors *cid.Cid // HAMT[address]Miner - PowerActor *cid.Cid // Power - RawActors *cid.Cid // HAMT[address]Raw - VerifregActor *cid.Cid // Veriferg + DataCapActor *cid.Cid `cborgen:"datacap"` // DataCap + InitActor *cid.Cid `cborgen:"init"` // Init + MarketActor *cid.Cid `cborgen:"market"` // Market + MinerActors *cid.Cid `cborgen:"miner"` // HAMT[address]Miner + PowerActor *cid.Cid `cborgen:"power"` // Power + RawActors *cid.Cid `cborgen:"raw"` // HAMT[address]Raw + VerifregActor *cid.Cid `cborgen:"verifreg"` // Veriferg } func (a *ActorStateChangesIPLD) Attributes() []attribute.KeyValue { @@ -61,7 +61,13 @@ func (a *ActorStateChangesIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) erro func ProcessActorsStates(ctx context.Context, s store.Store, changes *extract.ActorStateChanges) (*ActorStateChangesIPLD, error) { out := &ActorStateChangesIPLD{} - // TODO DataCap + if changes.DatacapActor != nil { + dcapRoot, err := PutActorDiffResult(ctx, s, changes.DatacapActor) + if err != nil { + return nil, err + } + out.DataCapActor = &dcapRoot + } if changes.InitActor != nil { initRoot, err := PutActorDiffResult(ctx, s, changes.InitActor) diff --git a/pkg/transform/cbor/messages/cbor_gen.go b/pkg/transform/cbor/messages/cbor_gen.go index 5ee42ca6a..c04814292 100644 --- a/pkg/transform/cbor/messages/cbor_gen.go +++ b/pkg/transform/cbor/messages/cbor_gen.go @@ -8,12 +8,11 @@ import ( "math" "sort" + chain "github.com/filecoin-project/lily/pkg/extract/chain" types "github.com/filecoin-project/lotus/chain/types" cid "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" xerrors "golang.org/x/xerrors" - - processor "github.com/filecoin-project/lily/pkg/extract/chain" ) var _ = xerrors.Errorf @@ -205,7 +204,7 @@ func (t *ChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { return err } - // t.Receipt (processor.ChainMessageReceipt) (struct) + // t.Receipt (chain.ChainMessageReceipt) (struct) if len("receipt") > cbg.MaxLength { return xerrors.Errorf("Value in field \"receipt\" was too long") } @@ -298,7 +297,7 @@ func (t *ChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Receipt (processor.ChainMessageReceipt) (struct) + // t.Receipt (chain.ChainMessageReceipt) (struct) case "receipt": { @@ -311,7 +310,7 @@ func (t *ChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Receipt = new(processor.ChainMessageReceipt) + t.Receipt = new(chain.ChainMessageReceipt) if err := t.Receipt.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) } @@ -368,7 +367,7 @@ func (t *SignedChainMessageIPLDContainer) MarshalCBOR(w io.Writer) error { return err } - // t.Receipt (processor.ChainMessageReceipt) (struct) + // t.Receipt (chain.ChainMessageReceipt) (struct) if len("receipt") > cbg.MaxLength { return xerrors.Errorf("Value in field \"receipt\" was too long") } @@ -461,7 +460,7 @@ func (t *SignedChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) } } - // t.Receipt (processor.ChainMessageReceipt) (struct) + // t.Receipt (chain.ChainMessageReceipt) (struct) case "receipt": { @@ -474,7 +473,7 @@ func (t *SignedChainMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) if err := cr.UnreadByte(); err != nil { return err } - t.Receipt = new(processor.ChainMessageReceipt) + t.Receipt = new(chain.ChainMessageReceipt) if err := t.Receipt.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) } @@ -531,7 +530,7 @@ func (t *ImplicitMessageIPLDContainer) MarshalCBOR(w io.Writer) error { return err } - // t.Receipt (processor.ImplicitMessageReceipt) (struct) + // t.Receipt (chain.ImplicitMessageReceipt) (struct) if len("receipt") > cbg.MaxLength { return xerrors.Errorf("Value in field \"receipt\" was too long") } @@ -624,7 +623,7 @@ func (t *ImplicitMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.Receipt (processor.ImplicitMessageReceipt) (struct) + // t.Receipt (chain.ImplicitMessageReceipt) (struct) case "receipt": { @@ -637,7 +636,7 @@ func (t *ImplicitMessageIPLDContainer) UnmarshalCBOR(r io.Reader) (err error) { if err := cr.UnreadByte(); err != nil { return err } - t.Receipt = new(processor.ImplicitMessageReceipt) + t.Receipt = new(chain.ImplicitMessageReceipt) if err := t.Receipt.UnmarshalCBOR(cr); err != nil { return xerrors.Errorf("unmarshaling t.Receipt pointer: %w", err) } diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go index 0e1e922ab..a675fa289 100644 --- a/pkg/transform/timescale/processor.go +++ b/pkg/transform/timescale/processor.go @@ -154,16 +154,14 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent } out = append(out, rawModels) } - /* - if actorIPLDContainer.VerifregActor != nil { - verifregModels, err := ProcessVerifregActor(ctx, s, current, parent, av, *actorIPLDContainer.VerifregActor) - if err != nil { - return nil, err - } - out = append(out, verifregModels) + if actorIPLDContainer.VerifregActor != nil { + verifregModels, err := ProcessVerifregActor(ctx, s, current, parent, av, *actorIPLDContainer.VerifregActor) + if err != nil { + return nil, err } + out = append(out, verifregModels) + } - */ return out, nil } From 0e7235eed5285d90538b4fc8ced3bf76d1087e2f Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 25 Jan 2023 09:44:04 -0800 Subject: [PATCH 34/48] clean up actor extraction --- pkg/core/array.go | 4 + pkg/core/map.go | 1 + pkg/extract/actors.go | 228 +++++++------------- pkg/extract/actors/verifregdiff/v9/state.go | 1 - pkg/extract/extract.go | 5 + pkg/transform/cbor/actors/serialize.go | 1 - 6 files changed, 92 insertions(+), 148 deletions(-) diff --git a/pkg/core/array.go b/pkg/core/array.go index 89ca1ed41..25bdf9f72 100644 --- a/pkg/core/array.go +++ b/pkg/core/array.go @@ -4,12 +4,15 @@ import ( "context" "github.com/filecoin-project/go-amt-ipld/v4" + logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" "github.com/filecoin-project/lily/chain/actors/adt" "github.com/filecoin-project/lily/chain/actors/adt/diff" ) +var log = logging.Logger("lily/core") + type ArrayModification struct { Key uint64 Type ChangeType @@ -26,6 +29,7 @@ func DiffArray(ctx context.Context, store adt.Store, child, parent adt.Array, ch Modified: []*ArrayModification{}, Removed: []*ArrayModification{}, } + log.Warn("diffing array using slow comparison") if err := diff.CompareArray(child, parent, diffContainer); err != nil { return nil, err } diff --git a/pkg/core/map.go b/pkg/core/map.go index fb9d1e52d..0f94dcf56 100644 --- a/pkg/core/map.go +++ b/pkg/core/map.go @@ -27,6 +27,7 @@ func DiffMap(ctx context.Context, store adt.Store, child, parent adt.Map, childO Modified: []*MapModification{}, Removed: []*MapModification{}, } + log.Warn("diffing array using slow comparison") if err := diff.CompareMap(child, parent, diffContainer); err != nil { return nil, err } diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index a042dc76f..5383d7f2a 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -2,6 +2,8 @@ package extract import ( "context" + "sort" + "sync" "time" "github.com/filecoin-project/go-address" @@ -9,10 +11,10 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" + "github.com/gammazero/workerpool" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/chain/actors/builtin" "github.com/filecoin-project/lily/pkg/core" @@ -60,11 +62,12 @@ func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { type StateDiffResult struct { ActorDiff actors.ActorDiffResult Address address.Address + Error error } type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version -func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Version, act *actors.ActorChange, diffLoader func(av actortypes.Version) (actors.ActorDiff, error)) (*StateDiffResult, error) { +func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Version, act *actors.ActorChange, diffLoader func(av actortypes.Version) (actors.ActorDiff, error)) (actors.ActorDiffResult, error) { // construct the state differ required by this actor version diff, err := diffLoader(version) if err != nil { @@ -72,74 +75,34 @@ func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Ver } start := time.Now() // diff the actors state and collect changes - log.Infow("diff actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String()) + log.Debugw("diff actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String()) diffRes, err := diff.State(ctx, api, act) if err != nil { return nil, err } - log.Infow("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String(), "duration", time.Since(start)) - return &StateDiffResult{ - ActorDiff: diffRes, - Address: act.Address, - }, nil + log.Debugw("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String(), "duration", time.Since(start)) + return diffRes, nil } -func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) (*ActorStateChanges, error) { - actorDiff := &rawdiff.StateDiff{ - DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, - } - actorStateChanges, err := actorDiff.State(ctx, api, act) - if err != nil { - return nil, err - } - results <- &StateDiffResult{ - ActorDiff: actorStateChanges, - Address: act.Address, - } - if core.DataCapCodes.Has(act.Current.Code) { - res, err := DiffActor(ctx, api, version, act, datacapdiff.StateDiffFor) - if err != nil { - return nil, err - } - results <- res - } - if core.MinerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.VerifregCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) - if err != nil { - return err - } - results <- res +func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []address.Address { + keys := make([]address.Address, 0, len(actors)) + + for k := range actors { + keys = append(keys, k) } - if core.InitCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) + sort.Slice(keys, func(i, j int) bool { + iKey, err := address.IDFromAddress(keys[i]) if err != nil { - return err + panic(err) } - results <- res - } - if core.PowerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) + jKey, err := address.IDFromAddress(keys[j]) if err != nil { - return err + panic(err) } - results <- res - } - if core.MarketCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - return nil + return iKey < jKey + }) + return keys } func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version) (*ActorStateChanges, error) { @@ -154,86 +117,36 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } - grp, grpCtx := errgroup.WithContext(ctx) - grp.SetLimit(8) + pool := workerpool.New(8) results := make(chan *StateDiffResult, len(actorChanges)) - grp.Go(func() error { - for addr, change := range actorChanges { - addr := addr - change := change - act := &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, - } - - grp.Go(func() error { - actorDiff := &rawdiff.StateDiff{ - DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, - } - actorStateChanges, err := actorDiff.State(grpCtx, api, act) - if err != nil { - return err - } - results <- &StateDiffResult{ - ActorDiff: actorStateChanges, - Address: act.Address, - } - if core.DataCapCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, datacapdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.MinerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, minerdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.VerifregCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, verifregdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.InitCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, initdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.PowerCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, powerdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - if core.MarketCodes.Has(change.Current.Code) { - res, err := DiffActor(ctx, api, actorVersion, act, marketdiff.StateDiffFor) - if err != nil { - return err - } - results <- res - } - return nil - }) + wg := sync.WaitGroup{} + // sort actors on actor id in ascending order, causes market actor to be differed early, which is the slowest actor to diff. + sortedKeys := sortedActorChangeKeys(actorChanges) + for _, addr := range sortedKeys { + addr := addr + change := actorChanges[addr] + act := &actors.ActorChange{ + Address: addr, + Executed: change.Executed, + Current: change.Current, + Type: change.ChangeType, } - return nil - }) + wg.Add(1) + pool.Submit(func() { + doActorDiff(ctx, api, act, actorVersion, results) + wg.Done() + }) + + } go func() { - if err := grp.Wait(); err != nil { - log.Error(err) - } + wg.Wait() close(results) }() for stateDiff := range results { + if stateDiff.Error != nil { + pool.Stop() + return nil, err + } switch stateDiff.ActorDiff.Kind() { case "actor": asc.RawActors[stateDiff.Address] = stateDiff.ActorDiff @@ -256,20 +169,43 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. return asc, nil } -type DiffWorker struct { - workers int -} - -func (w *DiffWorker) Start(ctx context.Context) error { - for i := 0; i < w.workers; i++ { - go func() { - w.work(ctx, nil, nil) - }() +func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { + actorDiff := &rawdiff.StateDiff{ + DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, + } + actorStateChanges, err := actorDiff.State(ctx, api, act) + results <- &StateDiffResult{ + ActorDiff: actorStateChanges, + Address: act.Address, + Error: err, + } + var actorDiffer func(av actortypes.Version) (actors.ActorDiff, error) + if core.DataCapCodes.Has(act.Current.Code) { + actorDiffer = datacapdiff.StateDiffFor + } + if core.MinerCodes.Has(act.Current.Code) { + actorDiffer = minerdiff.StateDiffFor + } + if core.VerifregCodes.Has(act.Current.Code) { + actorDiffer = verifregdiff.StateDiffFor + } + if core.InitCodes.Has(act.Current.Code) { + actorDiffer = initdiff.StateDiffFor + } + if core.PowerCodes.Has(act.Current.Code) { + actorDiffer = powerdiff.StateDiffFor + } + if core.MarketCodes.Has(act.Current.Code) { + actorDiffer = marketdiff.StateDiffFor + } + if actorDiffer == nil { + return } -} -func (w *DiffWorker) work(ctx context.Context, in, out chan interface{}) { - todo := <-in - time.Sleep(10) - out <- struct{}{} + res, err := DiffActor(ctx, api, version, act, actorDiffer) + results <- &StateDiffResult{ + ActorDiff: res, + Address: act.Address, + Error: err, + } } diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go index 744987a8f..65872f67f 100644 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ b/pkg/extract/actors/verifregdiff/v9/state.go @@ -113,6 +113,5 @@ func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*S } } - panic("TODO") return out, nil } diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go index 563f98d4f..0e83a6010 100644 --- a/pkg/extract/extract.go +++ b/pkg/extract/extract.go @@ -2,6 +2,7 @@ package extract import ( "context" + "time" actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/network" @@ -43,13 +44,17 @@ func State(ctx context.Context, api tasks.DataSource, current, executed *types.T grp, grpCtx := errgroup.WithContext(ctx) grp.Go(func() error { + start := time.Now() // all blocks, messages, implicit messages, from executed and receipts from current blockmessages, err = FullBlockMessages(grpCtx, api, current, executed) + log.Infow("extracted fullblock", "duration", time.Since(start)) return err }) grp.Go(func() error { + start := time.Now() // all actor changes between current and parent, actor state exists in current. actorChanges, err = Actors(grpCtx, api, current, executed, actorVersion) + log.Infow("extracted actors", "duration", time.Since(start)) return err }) diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index fa7b5fc7a..235bba3dd 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -114,7 +114,6 @@ func ProcessActorsStates(ctx context.Context, s store.Store, changes *extract.Ac if err != nil { return nil, err } - out.VerifregActor = &verifregRoot } From b10d395c144a27d76566a378cecd2ad776a208f1 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 25 Jan 2023 18:46:11 -0800 Subject: [PATCH 35/48] checkin --- .../timescale/actors/market/router.go | 82 ++++++++++- .../timescale/actors/market/v0/deals.go | 2 +- .../timescale/actors/market/v0/processor.go | 53 ------- .../timescale/actors/market/v0/proposals.go | 2 +- .../timescale/actors/market/v2/deals.go | 55 +++++++ .../timescale/actors/market/v2/proposals.go | 68 +++++++++ .../timescale/actors/market/v3/deals.go | 55 +++++++ .../timescale/actors/market/v3/proposals.go | 68 +++++++++ .../timescale/actors/market/v4/deals.go | 55 +++++++ .../timescale/actors/market/v4/proposals.go | 68 +++++++++ .../timescale/actors/market/v5/deals.go | 55 +++++++ .../timescale/actors/market/v5/proposals.go | 68 +++++++++ .../timescale/actors/market/v6/deals.go | 55 +++++++ .../timescale/actors/market/v6/proposals.go | 68 +++++++++ .../timescale/actors/market/v7/deals.go | 55 +++++++ .../timescale/actors/market/v7/proposals.go | 68 +++++++++ .../timescale/actors/market/v8/deals.go | 55 +++++++ .../timescale/actors/market/v8/proposals.go | 92 ++++++++++++ .../timescale/actors/market/v9/deals.go | 2 +- .../timescale/actors/market/v9/processor.go | 53 ------- .../timescale/actors/market/v9/proposals.go | 2 +- .../timescale/actors/miner/router.go | 137 +++++++++++++++--- .../timescale/actors/miner/v0/info.go | 2 +- .../timescale/actors/miner/v0/precommits.go | 2 +- .../timescale/actors/miner/v0/processor.go | 86 ----------- .../timescale/actors/miner/v0/sector_deals.go | 2 +- .../actors/miner/v0/sector_events.go | 2 +- .../timescale/actors/miner/v0/sectors.go | 2 +- .../timescale/actors/miner/v2/info.go | 2 +- .../timescale/actors/miner/v2/precommits.go | 2 +- .../timescale/actors/miner/v2/processor.go | 86 ----------- .../timescale/actors/miner/v2/sector_deals.go | 2 +- .../actors/miner/v2/sector_events.go | 2 +- .../timescale/actors/miner/v2/sectors.go | 2 +- .../timescale/actors/miner/v3/info.go | 2 +- .../timescale/actors/miner/v3/precommits.go | 2 +- .../timescale/actors/miner/v3/processor.go | 86 ----------- .../timescale/actors/miner/v3/sector_deals.go | 2 +- .../actors/miner/v3/sector_events.go | 2 +- .../timescale/actors/miner/v3/sectors.go | 2 +- .../timescale/actors/miner/v4/info.go | 2 +- .../timescale/actors/miner/v4/precommits.go | 2 +- .../timescale/actors/miner/v4/processor.go | 86 ----------- .../timescale/actors/miner/v4/sector_deals.go | 2 +- .../actors/miner/v4/sector_events.go | 2 +- .../timescale/actors/miner/v4/sectors.go | 2 +- .../timescale/actors/miner/v5/info.go | 2 +- .../timescale/actors/miner/v5/precommits.go | 2 +- .../timescale/actors/miner/v5/processor.go | 86 ----------- .../timescale/actors/miner/v5/sector_deals.go | 2 +- .../actors/miner/v5/sector_events.go | 2 +- .../timescale/actors/miner/v5/sectors.go | 2 +- .../timescale/actors/miner/v6/info.go | 2 +- .../timescale/actors/miner/v6/precommits.go | 2 +- .../timescale/actors/miner/v6/processor.go | 86 ----------- .../timescale/actors/miner/v6/sector_deals.go | 2 +- .../actors/miner/v6/sector_events.go | 2 +- .../timescale/actors/miner/v6/sectors.go | 2 +- .../timescale/actors/miner/v7/info.go | 2 +- .../timescale/actors/miner/v7/precommits.go | 2 +- .../timescale/actors/miner/v7/processor.go | 86 ----------- .../timescale/actors/miner/v7/sector_deals.go | 2 +- .../actors/miner/v7/sector_events.go | 2 +- .../timescale/actors/miner/v7/sectors.go | 2 +- .../timescale/actors/miner/v8/info.go | 2 +- .../timescale/actors/miner/v8/precommits.go | 2 +- .../timescale/actors/miner/v8/processor.go | 86 ----------- .../timescale/actors/miner/v8/sector_deals.go | 2 +- .../actors/miner/v8/sector_events.go | 2 +- .../timescale/actors/miner/v8/sectors.go | 2 +- .../timescale/actors/miner/v9/info.go | 2 +- .../timescale/actors/miner/v9/precommits.go | 2 +- .../timescale/actors/miner/v9/processor.go | 86 ----------- .../timescale/actors/miner/v9/sector_deals.go | 2 +- .../actors/miner/v9/sector_events.go | 2 +- .../timescale/actors/miner/v9/sectors.go | 2 +- .../timescale/actors/verifreg/router.go | 56 ++++++- .../timescale/actors/verifreg/v0/clients.go | 2 +- .../timescale/actors/verifreg/v0/processor.go | 53 ------- .../timescale/actors/verifreg/v0/verifiers.go | 2 +- .../timescale/actors/verifreg/v9/processor.go | 54 ------- .../timescale/actors/verifreg/v9/verifiers.go | 2 +- pkg/transform/timescale/processor.go | 32 ++-- tasks/actorstate/account/account.go | 2 +- tasks/actorstate/datacap/balance.go | 2 +- tasks/actorstate/init_/init.go | 2 +- tasks/actorstate/market/deal_proposal.go | 2 +- tasks/actorstate/market/deal_state.go | 2 +- tasks/actorstate/miner/beneficaries.go | 2 +- tasks/actorstate/miner/deadline_info.go | 2 +- tasks/actorstate/miner/fee_debt.go | 2 +- tasks/actorstate/miner/info.go | 2 +- tasks/actorstate/miner/locked_funds.go | 2 +- tasks/actorstate/miner/post.go | 2 +- tasks/actorstate/miner/precommit.go | 2 +- tasks/actorstate/miner/precommitv9.go | 2 +- tasks/actorstate/miner/sector.go | 2 +- tasks/actorstate/miner/sector_deals.go | 2 +- tasks/actorstate/miner/sector_events.go | 2 +- tasks/actorstate/miner/sectorv7.go | 2 +- tasks/actorstate/multisig/multisig.go | 2 +- tasks/actorstate/power/chain_power.go | 2 +- tasks/actorstate/power/claimed_power.go | 2 +- tasks/actorstate/power/power.go | 2 +- tasks/actorstate/raw/actor.go | 4 +- tasks/actorstate/raw/actor_state.go | 4 +- tasks/actorstate/reward/reward.go | 2 +- tasks/actorstate/verifreg/client.go | 2 +- tasks/actorstate/verifreg/verifier.go | 2 +- tasks/actorstate/verifreg/verifreg.go | 2 +- 110 files changed, 1221 insertions(+), 1120 deletions(-) delete mode 100644 pkg/transform/timescale/actors/market/v0/processor.go create mode 100644 pkg/transform/timescale/actors/market/v2/deals.go create mode 100644 pkg/transform/timescale/actors/market/v2/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v3/deals.go create mode 100644 pkg/transform/timescale/actors/market/v3/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v4/deals.go create mode 100644 pkg/transform/timescale/actors/market/v4/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v5/deals.go create mode 100644 pkg/transform/timescale/actors/market/v5/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v6/deals.go create mode 100644 pkg/transform/timescale/actors/market/v6/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v7/deals.go create mode 100644 pkg/transform/timescale/actors/market/v7/proposals.go create mode 100644 pkg/transform/timescale/actors/market/v8/deals.go create mode 100644 pkg/transform/timescale/actors/market/v8/proposals.go delete mode 100644 pkg/transform/timescale/actors/market/v9/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v0/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v2/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v3/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v4/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v5/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v6/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v7/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v8/processor.go delete mode 100644 pkg/transform/timescale/actors/miner/v9/processor.go delete mode 100644 pkg/transform/timescale/actors/verifreg/v0/processor.go delete mode 100644 pkg/transform/timescale/actors/verifreg/v9/processor.go diff --git a/pkg/transform/timescale/actors/market/router.go b/pkg/transform/timescale/actors/market/router.go index 0e7dd3b6b..dec69b380 100644 --- a/pkg/transform/timescale/actors/market/router.go +++ b/pkg/transform/timescale/actors/market/router.go @@ -10,19 +10,89 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v0" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v2" + v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v3" + v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v4" + v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v5" + v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v6" + v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v7" + v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v8" v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v9" ) -type MarketHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformMarketState(ctx context.Context, s store.Store, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { + marketState := new(marketdiff.StateChange) + if err := s.Get(ctx, root, marketState); err != nil { + return nil, err + } + marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, marketStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} -func MakeMarketProcessor(av actortypes.Version) (MarketHandler, error) { +func LookupMarketStateTransformer(av actortypes.Version) ([]Transformer, error) { switch av { case actortypes.Version0: - return v0.MarketHandler, nil + return []Transformer{ + v0.Deals{}, + v0.Proposals{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v2.Deals{}, + v2.Proposals{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v3.Deals{}, + v3.Proposals{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v4.Deals{}, + v4.Proposals{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v5.Deals{}, + v5.Proposals{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v6.Deals{}, + v6.Proposals{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v7.Deals{}, + v7.Proposals{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v8.Deals{}, + v8.Proposals{}, + }, nil case actortypes.Version9: - return v9.MarketHandler, nil - default: - return nil, fmt.Errorf("unsupported market actor version: %d", av) + return []Transformer{ + v9.Deals{}, + v9.Proposals{}, + }, nil } + return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) } diff --git a/pkg/transform/timescale/actors/market/v0/deals.go b/pkg/transform/timescale/actors/market/v0/deals.go index 24a4247ab..a9d436696 100644 --- a/pkg/transform/timescale/actors/market/v0/deals.go +++ b/pkg/transform/timescale/actors/market/v0/deals.go @@ -15,7 +15,7 @@ import ( type Deals struct{} -func (Deals) Extract(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states diff --git a/pkg/transform/timescale/actors/market/v0/processor.go b/pkg/transform/timescale/actors/market/v0/processor.go deleted file mode 100644 index 35b7dff0d..000000000 --- a/pkg/transform/timescale/actors/market/v0/processor.go +++ /dev/null @@ -1,53 +0,0 @@ -package v0 - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/model" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" -) - -type Extractor interface { - Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - Extractors []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.Extractors)) - for _, e := range se.Extractors { - m, err := e.Extract(ctx, current, executed, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MarketHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) { - marketState := new(marketdiff.StateChange) - if err := s.Get(ctx, marketRoot, marketState); err != nil { - return nil, err - } - marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err - } - - stateExtractor := &StateExtract{ - Extractors: []Extractor{ - Deals{}, - Proposals{}, - }, - } - return stateExtractor.Process(ctx, current, executed, marketStateDiff) -} diff --git a/pkg/transform/timescale/actors/market/v0/proposals.go b/pkg/transform/timescale/actors/market/v0/proposals.go index 6eaf67bbd..e193a6415 100644 --- a/pkg/transform/timescale/actors/market/v0/proposals.go +++ b/pkg/transform/timescale/actors/market/v0/proposals.go @@ -17,7 +17,7 @@ import ( type Proposals struct{} -func (Proposals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals diff --git a/pkg/transform/timescale/actors/market/v2/deals.go b/pkg/transform/timescale/actors/market/v2/deals.go new file mode 100644 index 000000000..f1de6b3ca --- /dev/null +++ b/pkg/transform/timescale/actors/market/v2/deals.go @@ -0,0 +1,55 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v2/proposals.go b/pkg/transform/timescale/actors/market/v2/proposals.go new file mode 100644 index 000000000..ad2c265a5 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v2/proposals.go @@ -0,0 +1,68 @@ +package v2 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v3/deals.go b/pkg/transform/timescale/actors/market/v3/deals.go new file mode 100644 index 000000000..c9994ed7b --- /dev/null +++ b/pkg/transform/timescale/actors/market/v3/deals.go @@ -0,0 +1,55 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v3/proposals.go b/pkg/transform/timescale/actors/market/v3/proposals.go new file mode 100644 index 000000000..670082bc6 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v3/proposals.go @@ -0,0 +1,68 @@ +package v3 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v4/deals.go b/pkg/transform/timescale/actors/market/v4/deals.go new file mode 100644 index 000000000..ee54e073b --- /dev/null +++ b/pkg/transform/timescale/actors/market/v4/deals.go @@ -0,0 +1,55 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v4/proposals.go b/pkg/transform/timescale/actors/market/v4/proposals.go new file mode 100644 index 000000000..7d54d5d8c --- /dev/null +++ b/pkg/transform/timescale/actors/market/v4/proposals.go @@ -0,0 +1,68 @@ +package v4 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v5/deals.go b/pkg/transform/timescale/actors/market/v5/deals.go new file mode 100644 index 000000000..dc6ba07c4 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v5/deals.go @@ -0,0 +1,55 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v5/proposals.go b/pkg/transform/timescale/actors/market/v5/proposals.go new file mode 100644 index 000000000..aecc5fa5f --- /dev/null +++ b/pkg/transform/timescale/actors/market/v5/proposals.go @@ -0,0 +1,68 @@ +package v5 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v6/deals.go b/pkg/transform/timescale/actors/market/v6/deals.go new file mode 100644 index 000000000..51e7f1ca2 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v6/deals.go @@ -0,0 +1,55 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v6/proposals.go b/pkg/transform/timescale/actors/market/v6/proposals.go new file mode 100644 index 000000000..ace360b34 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v6/proposals.go @@ -0,0 +1,68 @@ +package v6 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v7/deals.go b/pkg/transform/timescale/actors/market/v7/deals.go new file mode 100644 index 000000000..cb6868f33 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v7/deals.go @@ -0,0 +1,55 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v7/proposals.go b/pkg/transform/timescale/actors/market/v7/proposals.go new file mode 100644 index 000000000..f89c57292 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v7/proposals.go @@ -0,0 +1,68 @@ +package v7 + +import ( + "bytes" + "context" + "encoding/base64" + + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: label, + IsString: false, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v8/deals.go b/pkg/transform/timescale/actors/market/v8/deals.go new file mode 100644 index 000000000..dbfa52324 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v8/deals.go @@ -0,0 +1,55 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" +) + +type Deals struct{} + +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketDeals []*deals + for _, change := range changes.DealStateChanges { + // only care about new and modified deal states + if change.Change == core.ChangeTypeRemove { + continue + } + dealState := new(market.DealState) + if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketDeals = append(marketDeals, &deals{ + DealID: change.DealID, + State: dealState, + }) + } + return MarketDealStateChangesAsModel(ctx, current, marketDeals) +} + +type deals struct { + DealID uint64 + State *market.DealState +} + +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { + dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) + for i, deal := range dealStates { + dealStateModel[i] = &marketmodel.MarketDealState{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + DealID: deal.DealID, + SectorStartEpoch: int64(deal.State.SectorStartEpoch), + LastUpdateEpoch: int64(deal.State.LastUpdatedEpoch), + SlashEpoch: int64(deal.State.SlashEpoch), + } + } + return dealStateModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v8/proposals.go b/pkg/transform/timescale/actors/market/v8/proposals.go new file mode 100644 index 000000000..182ff09cd --- /dev/null +++ b/pkg/transform/timescale/actors/market/v8/proposals.go @@ -0,0 +1,92 @@ +package v8 + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + + "github.com/filecoin-project/go-state-types/builtin/v8/market" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + marketmodel "github.com/filecoin-project/lily/model/actors/market" + "github.com/filecoin-project/lily/pkg/core" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" +) + +type Proposals struct{} + +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { + var marketProposals []*proposal + for _, change := range change.DealProposalChanges { + // we only car about new and modified deal proposals + if change.Change == core.ChangeTypeRemove { + continue + } + dealProp := new(market.DealProposal) + if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + marketProposals = append(marketProposals, &proposal{ + DealID: change.DealID, + State: dealProp, + }) + } + return MarketDealProposalChangesAsModel(ctx, current, marketProposals) +} + +type proposal struct { + DealID uint64 + State *market.DealProposal +} + +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { + dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) + for i, prop := range dealProps { + var isString bool + var base64Label string + if prop.State.Label.IsString() { + labelString, err := prop.State.Label.ToString() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not a string despite claiming it is (developer error?)", prop.DealID) + } + + isString = true + base64Label = base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(labelString))) + + } else if prop.State.Label.IsBytes() { + labelBytes, err := prop.State.Label.ToBytes() + if err != nil { + return nil, fmt.Errorf("deal proposal (ID: %d) label is not bytes despit claiming it is (developer error?)", prop.DealID) + } + + isString = false + base64Label = base64.StdEncoding.EncodeToString(labelBytes) + + } else { + // TODO this should never happen, but if it does it indicates a logic. + return nil, fmt.Errorf("deal proposal (ID: %d) label is neither bytes nor string (DEVELOPER ERROR)", prop.DealID) + } + dealPropsModel[i] = &marketmodel.MarketDealProposal{ + Height: int64(current.Height()), + DealID: prop.DealID, + StateRoot: current.ParentState().String(), + PaddedPieceSize: uint64(prop.State.PieceSize), + UnpaddedPieceSize: uint64(prop.State.PieceSize.Unpadded()), + StartEpoch: int64(prop.State.StartEpoch), + EndEpoch: int64(prop.State.EndEpoch), + ClientID: prop.State.Client.String(), + ProviderID: prop.State.Provider.String(), + ClientCollateral: prop.State.ClientCollateral.String(), + ProviderCollateral: prop.State.ProviderCollateral.String(), + StoragePricePerEpoch: prop.State.StoragePricePerEpoch.String(), + PieceCID: prop.State.PieceCID.String(), + IsVerified: prop.State.VerifiedDeal, + Label: base64Label, + IsString: isString, + } + } + return dealPropsModel, nil +} diff --git a/pkg/transform/timescale/actors/market/v9/deals.go b/pkg/transform/timescale/actors/market/v9/deals.go index aab7ecdb9..e84be56a4 100644 --- a/pkg/transform/timescale/actors/market/v9/deals.go +++ b/pkg/transform/timescale/actors/market/v9/deals.go @@ -15,7 +15,7 @@ import ( type Deals struct{} -func (Deals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { var marketDeals []*deals for _, change := range change.DealStateChanges { // only care about new and modified deal states diff --git a/pkg/transform/timescale/actors/market/v9/processor.go b/pkg/transform/timescale/actors/market/v9/processor.go deleted file mode 100644 index 7c367bf6e..000000000 --- a/pkg/transform/timescale/actors/market/v9/processor.go +++ /dev/null @@ -1,53 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/model" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" -) - -type Extractor interface { - Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - Extractors []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.Extractors)) - for _, e := range se.Extractors { - m, err := e.Extract(ctx, current, executed, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MarketHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, marketRoot cid.Cid) (model.PersistableList, error) { - marketState := new(marketdiff.StateChange) - if err := s.Get(ctx, marketRoot, marketState); err != nil { - return nil, err - } - marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err - } - - stateExtractor := &StateExtract{ - Extractors: []Extractor{ - Deals{}, - Proposals{}, - }, - } - return stateExtractor.Process(ctx, current, executed, marketStateDiff) -} diff --git a/pkg/transform/timescale/actors/market/v9/proposals.go b/pkg/transform/timescale/actors/market/v9/proposals.go index 66406124c..0b81541b2 100644 --- a/pkg/transform/timescale/actors/market/v9/proposals.go +++ b/pkg/transform/timescale/actors/market/v9/proposals.go @@ -18,7 +18,7 @@ import ( type Proposals struct{} -func (Proposals) Extract(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index 5ecb4ca82..986e46407 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -6,11 +6,13 @@ import ( "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" @@ -26,23 +28,23 @@ import ( func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { switch version { case actortypes.Version0: - return v0.MinerStateHandler(ctx, current, executed, addr, change) + return v0.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version2: - return v2.MinerStateHandler(ctx, current, executed, addr, change) + return v2.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version3: - return v3.MinerStateHandler(ctx, current, executed, addr, change) + return v3.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version4: - return v4.MinerStateHandler(ctx, current, executed, addr, change) + return v4.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version5: - return v5.MinerStateHandler(ctx, current, executed, addr, change) + return v5.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version6: - return v6.MinerStateHandler(ctx, current, executed, addr, change) + return v6.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version7: - return v7.MinerStateHandler(ctx, current, executed, addr, change) + return v7.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version8: - return v8.MinerStateHandler(ctx, current, executed, addr, change) + return v8.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version9: - return v9.MinerStateHandler(ctx, current, executed, addr, change) + return v9.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version10: panic("not yet implemented") default: @@ -50,29 +52,120 @@ func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr addr } } -type MinerHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) +func TransformMinerStates(ctx context.Context, s store.Store, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { + out := model.PersistableList{} -func MakeMinerProcessor(av actortypes.Version) (MinerHandler, error) { + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(s, root, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) + if err != nil { + return err + } + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, m) + } + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +func LookupMinerStateTransformer(av actortypes.Version) ([]Transformer, error) { switch av { case actortypes.Version0: - return v0.MinerHandler, nil + return []Transformer{ + v0.Info{}, + v0.PreCommit{}, + v0.SectorDeal{}, + v0.SectorEvent{}, + v0.Sector{}, + }, nil case actortypes.Version2: - return v2.MinerHandler, nil + return []Transformer{ + v2.Info{}, + v2.PreCommit{}, + v2.SectorDeal{}, + v2.SectorEvent{}, + v2.Sector{}, + }, nil case actortypes.Version3: - return v3.MinerHandler, nil + return []Transformer{ + v3.Info{}, + v3.PreCommit{}, + v3.SectorDeal{}, + v3.SectorEvent{}, + v3.Sector{}, + }, nil case actortypes.Version4: - return v4.MinerHandler, nil + return []Transformer{ + v4.Info{}, + v4.PreCommit{}, + v4.SectorDeal{}, + v4.SectorEvent{}, + v4.Sector{}, + }, nil case actortypes.Version5: - return v5.MinerHandler, nil + return []Transformer{ + v5.Info{}, + v5.PreCommit{}, + v5.SectorDeal{}, + v5.SectorEvent{}, + v5.Sector{}, + }, nil case actortypes.Version6: - return v6.MinerHandler, nil + return []Transformer{ + v6.Info{}, + v6.PreCommit{}, + v6.SectorDeal{}, + v6.SectorEvent{}, + v6.Sector{}, + }, nil case actortypes.Version7: - return v7.MinerHandler, nil + return []Transformer{ + v7.Info{}, + v7.PreCommit{}, + v7.SectorDeal{}, + v7.SectorEvent{}, + v7.Sector{}, + }, nil case actortypes.Version8: - return v8.MinerHandler, nil + return []Transformer{ + v8.Info{}, + v8.PreCommit{}, + v8.SectorDeal{}, + v8.SectorEvent{}, + v8.Sector{}, + }, nil case actortypes.Version9: - return v9.MinerHandler, nil - default: - return nil, fmt.Errorf("unsupported miner actor version: %d", av) + return []Transformer{ + v9.Info{}, + v9.PreCommit{}, + v9.SectorDeal{}, + v9.SectorEvent{}, + v9.Sector{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for miner transform: %d", av) } diff --git a/pkg/transform/timescale/actors/miner/v0/info.go b/pkg/transform/timescale/actors/miner/v0/info.go index c7e0cba8f..bb51c8370 100644 --- a/pkg/transform/timescale/actors/miner/v0/info.go +++ b/pkg/transform/timescale/actors/miner/v0/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v0/precommits.go b/pkg/transform/timescale/actors/miner/v0/precommits.go index 50ace3984..f640aa60e 100644 --- a/pkg/transform/timescale/actors/miner/v0/precommits.go +++ b/pkg/transform/timescale/actors/miner/v0/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v0/processor.go b/pkg/transform/timescale/actors/miner/v0/processor.go deleted file mode 100644 index 1c9e68642..000000000 --- a/pkg/transform/timescale/actors/miner/v0/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v0 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v0/sector_deals.go index dda6fcdc3..e78a7b616 100644 --- a/pkg/transform/timescale/actors/miner/v0/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v0/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v0/sector_events.go index 8f117cfb3..dde7d8fec 100644 --- a/pkg/transform/timescale/actors/miner/v0/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v0/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v0/sectors.go b/pkg/transform/timescale/actors/miner/v0/sectors.go index 492a42c86..b27602359 100644 --- a/pkg/transform/timescale/actors/miner/v0/sectors.go +++ b/pkg/transform/timescale/actors/miner/v0/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v2/info.go b/pkg/transform/timescale/actors/miner/v2/info.go index 8d5373459..31460cfbb 100644 --- a/pkg/transform/timescale/actors/miner/v2/info.go +++ b/pkg/transform/timescale/actors/miner/v2/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v2/precommits.go b/pkg/transform/timescale/actors/miner/v2/precommits.go index 773af4829..f9fb12cf7 100644 --- a/pkg/transform/timescale/actors/miner/v2/precommits.go +++ b/pkg/transform/timescale/actors/miner/v2/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v2/processor.go b/pkg/transform/timescale/actors/miner/v2/processor.go deleted file mode 100644 index fe8216265..000000000 --- a/pkg/transform/timescale/actors/miner/v2/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v2 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v2/sector_deals.go index e50c97211..387fd0fe5 100644 --- a/pkg/transform/timescale/actors/miner/v2/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v2/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v2/sector_events.go index 8ca79914e..49d858dbd 100644 --- a/pkg/transform/timescale/actors/miner/v2/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v2/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v2/sectors.go b/pkg/transform/timescale/actors/miner/v2/sectors.go index 99ad0eb44..f3d93d306 100644 --- a/pkg/transform/timescale/actors/miner/v2/sectors.go +++ b/pkg/transform/timescale/actors/miner/v2/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v3/info.go b/pkg/transform/timescale/actors/miner/v3/info.go index b42474c1e..85d694c81 100644 --- a/pkg/transform/timescale/actors/miner/v3/info.go +++ b/pkg/transform/timescale/actors/miner/v3/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v3/precommits.go b/pkg/transform/timescale/actors/miner/v3/precommits.go index 494c06740..f65293b57 100644 --- a/pkg/transform/timescale/actors/miner/v3/precommits.go +++ b/pkg/transform/timescale/actors/miner/v3/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v3/processor.go b/pkg/transform/timescale/actors/miner/v3/processor.go deleted file mode 100644 index c3ef08d0c..000000000 --- a/pkg/transform/timescale/actors/miner/v3/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v3 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v3/sector_deals.go index 466beaeae..ec3354c4b 100644 --- a/pkg/transform/timescale/actors/miner/v3/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v3/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v3/sector_events.go index 2a4e056b4..5ca1c8009 100644 --- a/pkg/transform/timescale/actors/miner/v3/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v3/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v3/sectors.go b/pkg/transform/timescale/actors/miner/v3/sectors.go index 447b4ca58..751333bad 100644 --- a/pkg/transform/timescale/actors/miner/v3/sectors.go +++ b/pkg/transform/timescale/actors/miner/v3/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v4/info.go b/pkg/transform/timescale/actors/miner/v4/info.go index b000f78b2..139533888 100644 --- a/pkg/transform/timescale/actors/miner/v4/info.go +++ b/pkg/transform/timescale/actors/miner/v4/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v4/precommits.go b/pkg/transform/timescale/actors/miner/v4/precommits.go index 352b64fda..a22d45420 100644 --- a/pkg/transform/timescale/actors/miner/v4/precommits.go +++ b/pkg/transform/timescale/actors/miner/v4/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v4/processor.go b/pkg/transform/timescale/actors/miner/v4/processor.go deleted file mode 100644 index 785aa5ddc..000000000 --- a/pkg/transform/timescale/actors/miner/v4/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v4 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v4/sector_deals.go index 8b0ef4dd8..2047206d4 100644 --- a/pkg/transform/timescale/actors/miner/v4/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v4/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v4/sector_events.go index 22d52fb34..064ef33b8 100644 --- a/pkg/transform/timescale/actors/miner/v4/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v4/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v4/sectors.go b/pkg/transform/timescale/actors/miner/v4/sectors.go index 7400a8058..0e3208fec 100644 --- a/pkg/transform/timescale/actors/miner/v4/sectors.go +++ b/pkg/transform/timescale/actors/miner/v4/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v5/info.go b/pkg/transform/timescale/actors/miner/v5/info.go index 1261f4815..fa635d9bb 100644 --- a/pkg/transform/timescale/actors/miner/v5/info.go +++ b/pkg/transform/timescale/actors/miner/v5/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v5/precommits.go b/pkg/transform/timescale/actors/miner/v5/precommits.go index 1925086ef..93dca424d 100644 --- a/pkg/transform/timescale/actors/miner/v5/precommits.go +++ b/pkg/transform/timescale/actors/miner/v5/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v5/processor.go b/pkg/transform/timescale/actors/miner/v5/processor.go deleted file mode 100644 index 08d167e12..000000000 --- a/pkg/transform/timescale/actors/miner/v5/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v5 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v5/sector_deals.go index 03baec818..1a90727f7 100644 --- a/pkg/transform/timescale/actors/miner/v5/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v5/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v5/sector_events.go index 8b854e658..2d4445eb7 100644 --- a/pkg/transform/timescale/actors/miner/v5/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v5/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v5/sectors.go b/pkg/transform/timescale/actors/miner/v5/sectors.go index 970ab2267..bc4e7aff4 100644 --- a/pkg/transform/timescale/actors/miner/v5/sectors.go +++ b/pkg/transform/timescale/actors/miner/v5/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v6/info.go b/pkg/transform/timescale/actors/miner/v6/info.go index c67e58029..83fa91815 100644 --- a/pkg/transform/timescale/actors/miner/v6/info.go +++ b/pkg/transform/timescale/actors/miner/v6/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v6/precommits.go b/pkg/transform/timescale/actors/miner/v6/precommits.go index 8dea5427f..aa8d4e86e 100644 --- a/pkg/transform/timescale/actors/miner/v6/precommits.go +++ b/pkg/transform/timescale/actors/miner/v6/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v6/processor.go b/pkg/transform/timescale/actors/miner/v6/processor.go deleted file mode 100644 index c2816bc17..000000000 --- a/pkg/transform/timescale/actors/miner/v6/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v6 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v6/sector_deals.go index 8f9363c4b..72cfceb2d 100644 --- a/pkg/transform/timescale/actors/miner/v6/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v6/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v6/sector_events.go index 7f20b46f0..1d1e8c998 100644 --- a/pkg/transform/timescale/actors/miner/v6/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v6/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v6/sectors.go b/pkg/transform/timescale/actors/miner/v6/sectors.go index 859061a57..233b2cca1 100644 --- a/pkg/transform/timescale/actors/miner/v6/sectors.go +++ b/pkg/transform/timescale/actors/miner/v6/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v7/info.go b/pkg/transform/timescale/actors/miner/v7/info.go index d5c325bfe..2bab6e0ad 100644 --- a/pkg/transform/timescale/actors/miner/v7/info.go +++ b/pkg/transform/timescale/actors/miner/v7/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v7/precommits.go b/pkg/transform/timescale/actors/miner/v7/precommits.go index 52659edd3..d5c629c2c 100644 --- a/pkg/transform/timescale/actors/miner/v7/precommits.go +++ b/pkg/transform/timescale/actors/miner/v7/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v7/processor.go b/pkg/transform/timescale/actors/miner/v7/processor.go deleted file mode 100644 index 233cf3f1d..000000000 --- a/pkg/transform/timescale/actors/miner/v7/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v7 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v7/sector_deals.go index 8e1639b48..821f5706a 100644 --- a/pkg/transform/timescale/actors/miner/v7/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v7/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v7/sector_events.go index 7e7afea53..a243a0e1f 100644 --- a/pkg/transform/timescale/actors/miner/v7/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v7/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v7/sectors.go b/pkg/transform/timescale/actors/miner/v7/sectors.go index afd482b80..a359f8822 100644 --- a/pkg/transform/timescale/actors/miner/v7/sectors.go +++ b/pkg/transform/timescale/actors/miner/v7/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v8/info.go b/pkg/transform/timescale/actors/miner/v8/info.go index 4f220aadc..fae1ab727 100644 --- a/pkg/transform/timescale/actors/miner/v8/info.go +++ b/pkg/transform/timescale/actors/miner/v8/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v8/precommits.go b/pkg/transform/timescale/actors/miner/v8/precommits.go index 5e2c1bc50..2c85ffd0e 100644 --- a/pkg/transform/timescale/actors/miner/v8/precommits.go +++ b/pkg/transform/timescale/actors/miner/v8/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v8/processor.go b/pkg/transform/timescale/actors/miner/v8/processor.go deleted file mode 100644 index fe26bfc0b..000000000 --- a/pkg/transform/timescale/actors/miner/v8/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v8 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v8/sector_deals.go index d4e361a5c..e3ef0bcaf 100644 --- a/pkg/transform/timescale/actors/miner/v8/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v8/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v8/sector_events.go index 833b04fba..63245ad5f 100644 --- a/pkg/transform/timescale/actors/miner/v8/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v8/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v8/sectors.go b/pkg/transform/timescale/actors/miner/v8/sectors.go index d419c6afe..24ba7f2ba 100644 --- a/pkg/transform/timescale/actors/miner/v8/sectors.go +++ b/pkg/transform/timescale/actors/miner/v8/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/miner/v9/info.go b/pkg/transform/timescale/actors/miner/v9/info.go index bcb1a7ac4..e2bae0c99 100644 --- a/pkg/transform/timescale/actors/miner/v9/info.go +++ b/pkg/transform/timescale/actors/miner/v9/info.go @@ -19,7 +19,7 @@ import ( type Info struct{} -func (i Info) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { // if there is no info nothing changed. if change.InfoChange == nil { return nil, nil diff --git a/pkg/transform/timescale/actors/miner/v9/precommits.go b/pkg/transform/timescale/actors/miner/v9/precommits.go index 2bb4e4237..764c9ecfd 100644 --- a/pkg/transform/timescale/actors/miner/v9/precommits.go +++ b/pkg/transform/timescale/actors/miner/v9/precommits.go @@ -18,7 +18,7 @@ import ( type PreCommit struct{} -func (PreCommit) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var precommits []*miner.SectorPreCommitOnChainInfo for _, change := range change.PreCommitChanges { // only care about precommits added diff --git a/pkg/transform/timescale/actors/miner/v9/processor.go b/pkg/transform/timescale/actors/miner/v9/processor.go deleted file mode 100644 index 6d1ba61d4..000000000 --- a/pkg/transform/timescale/actors/miner/v9/processor.go +++ /dev/null @@ -1,86 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-address" - store "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" -) - -type Extractor interface { - Extract(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - ExtractMethods []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, addr address.Address, changes *minerdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.ExtractMethods)) - for _, e := range se.ExtractMethods { - m, err := e.Extract(ctx, current, executed, addr, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func MinerStateHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - return ExtractMinerStateChanges(ctx, current, executed, addr, change) -} - -func MinerHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, minerMapRoot cid.Cid) (model.PersistableList, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, minerMapRoot, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - // register extractors to run over the miners state. - stateExtractor := &StateExtract{ - ExtractMethods: []Extractor{ - Info{}, - PreCommit{}, - SectorDeal{}, - SectorEvent{}, - Sector{}, - }, - } - models, err := stateExtractor.Process(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, models...) - return nil - }); err != nil { - return nil, err - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v9/sector_deals.go index 235976d1a..95cd68998 100644 --- a/pkg/transform/timescale/actors/miner/v9/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_deals.go @@ -19,7 +19,7 @@ import ( type SectorDeal struct{} -func (s SectorDeal) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { sectors := change.SectorChanges out := minermodel.MinerSectorDealList{} height := int64(current.Height()) diff --git a/pkg/transform/timescale/actors/miner/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v9/sector_events.go index d74ce0d82..6885913b3 100644 --- a/pkg/transform/timescale/actors/miner/v9/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v9/sector_events.go @@ -19,7 +19,7 @@ import ( type SectorEvent struct{} -func (s SectorEvent) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var ( precommits = change.PreCommitChanges sectors = change.SectorChanges diff --git a/pkg/transform/timescale/actors/miner/v9/sectors.go b/pkg/transform/timescale/actors/miner/v9/sectors.go index 4e08e93e7..332525ba9 100644 --- a/pkg/transform/timescale/actors/miner/v9/sectors.go +++ b/pkg/transform/timescale/actors/miner/v9/sectors.go @@ -19,7 +19,7 @@ import ( type Sector struct { } -func (s Sector) Extract(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { var sectors []*miner.SectorOnChainInfo changes := change.SectorChanges for _, sector := range changes { diff --git a/pkg/transform/timescale/actors/verifreg/router.go b/pkg/transform/timescale/actors/verifreg/router.go index 95e0f54cc..33a0d5ad3 100644 --- a/pkg/transform/timescale/actors/verifreg/router.go +++ b/pkg/transform/timescale/actors/verifreg/router.go @@ -10,19 +10,63 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v0" v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v9" ) -type VerifregHandler = func(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformVerifregState(ctx context.Context, s store.Store, av actortypes.Version, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { + if av < actortypes.Version9 { + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, root, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, verifrefStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil + + } +} -func MakeVerifregProcessor(av actortypes.Version) (VerifregHandler, error) { +func LookupMarketStateTransformerV8(av actortypes.Version) ([]Transformer, error) { switch av { case actortypes.Version0: - return v0.VerifregHandler, nil + return []Transformer{ + v0.Clients{}, + v0.Verifiers{}, + }, nil + case actortypes.Version2: + return []Transformer{}, nil + case actortypes.Version3: + return []Transformer{}, nil + case actortypes.Version4: + return []Transformer{}, nil + case actortypes.Version5: + return []Transformer{}, nil + case actortypes.Version6: + return []Transformer{}, nil + case actortypes.Version7: + return []Transformer{}, nil + case actortypes.Version8: + return []Transformer{}, nil case actortypes.Version9: - return v9.VerifregHandler, nil - default: - return nil, fmt.Errorf("unsupported verifreg actor version: %d", av) + return []Transformer{ + v9.Verifiers{}, + }, nil } + return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) } diff --git a/pkg/transform/timescale/actors/verifreg/v0/clients.go b/pkg/transform/timescale/actors/verifreg/v0/clients.go index 655594c0d..e96ad4fa4 100644 --- a/pkg/transform/timescale/actors/verifreg/v0/clients.go +++ b/pkg/transform/timescale/actors/verifreg/v0/clients.go @@ -16,7 +16,7 @@ import ( type Clients struct{} -func (Clients) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) for i, change := range changes.ClientChanges { addr, err := address.NewFromBytes(change.Client) diff --git a/pkg/transform/timescale/actors/verifreg/v0/processor.go b/pkg/transform/timescale/actors/verifreg/v0/processor.go deleted file mode 100644 index dae0d545f..000000000 --- a/pkg/transform/timescale/actors/verifreg/v0/processor.go +++ /dev/null @@ -1,53 +0,0 @@ -package v0 - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/model" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" -) - -type Extractor interface { - Extract(ctx context.Context, current, executed *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - Extractors []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.Extractors)) - for _, e := range se.Extractors { - m, err := e.Extract(ctx, current, executed, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func VerifregHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) { - verifregState := new(verifregdiff.StateChange) - if err := s.Get(ctx, verifregRoot, verifregState); err != nil { - return nil, err - } - verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err - } - - stateExtractor := &StateExtract{ - Extractors: []Extractor{ - Clients{}, - Verifiers{}, - }, - } - return stateExtractor.Process(ctx, current, executed, verifrefStateDiff) -} diff --git a/pkg/transform/timescale/actors/verifreg/v0/verifiers.go b/pkg/transform/timescale/actors/verifreg/v0/verifiers.go index 9520f3595..9f9c64701 100644 --- a/pkg/transform/timescale/actors/verifreg/v0/verifiers.go +++ b/pkg/transform/timescale/actors/verifreg/v0/verifiers.go @@ -16,7 +16,7 @@ import ( type Verifiers struct{} -func (Verifiers) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) for i, change := range changes.VerifierChanges { addr, err := address.NewFromBytes(change.Verifier) diff --git a/pkg/transform/timescale/actors/verifreg/v9/processor.go b/pkg/transform/timescale/actors/verifreg/v9/processor.go deleted file mode 100644 index 64d2ddb16..000000000 --- a/pkg/transform/timescale/actors/verifreg/v9/processor.go +++ /dev/null @@ -1,54 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-state-types/store" - "github.com/filecoin-project/lotus/chain/types" - "github.com/ipfs/go-cid" - - "github.com/filecoin-project/lily/model" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" -) - -type Extractor interface { - Extract(ctx context.Context, current, executed *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) -} - -type StateExtract struct { - Extractors []Extractor -} - -func (se *StateExtract) Process(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.PersistableList, error) { - out := make(model.PersistableList, 0, len(se.Extractors)) - for _, e := range se.Extractors { - m, err := e.Extract(ctx, current, executed, changes) - if err != nil { - return nil, err - } - if m != nil { - out = append(out, m) - } - } - return out, nil -} - -func VerifregHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, verifregRoot cid.Cid) (model.PersistableList, error) { - verifregState := new(verifregdiff.StateChange) - if err := s.Get(ctx, verifregRoot, verifregState); err != nil { - return nil, err - } - verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err - } - - stateExtractor := &StateExtract{ - Extractors: []Extractor{ - Verifiers{}, - Claims{}, - Allocations{}, - }, - } - return stateExtractor.Process(ctx, current, executed, verifrefStateDiff) -} diff --git a/pkg/transform/timescale/actors/verifreg/v9/verifiers.go b/pkg/transform/timescale/actors/verifreg/v9/verifiers.go index a3c3b3d13..89a1b4461 100644 --- a/pkg/transform/timescale/actors/verifreg/v9/verifiers.go +++ b/pkg/transform/timescale/actors/verifreg/v9/verifiers.go @@ -16,7 +16,7 @@ import ( type Verifiers struct{} -func (Verifiers) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) for i, change := range changes.VerifierChanges { addr, err := address.NewFromBytes(change.Verifier) diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go index a675fa289..af5b1b7cd 100644 --- a/pkg/transform/timescale/processor.go +++ b/pkg/transform/timescale/processor.go @@ -91,26 +91,10 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe return strg.PersistBatch(ctx, toStorage) } -func ProcessMiners(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { - minerHandler, err := miner.MakeMinerProcessor(av) - if err != nil { - return nil, err - } - return minerHandler(ctx, s, current, executed, root) -} - func ProcessInitAddresses(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { return init_.InitHandler(ctx, s, current, executed, root) } -func ProcessMarketActor(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { - marketHandler, err := market.MakeMarketProcessor(av) - if err != nil { - return nil, err - } - return marketHandler(ctx, s, current, executed, root) -} - func ProcessVerifregActor(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { verifregHandler, err := verifreg.MakeVerifregProcessor(av) if err != nil { @@ -127,19 +111,29 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) out := model.PersistableList{} if actorIPLDContainer.MarketActor != nil { - marketModels, err := ProcessMarketActor(ctx, s, current, parent, av, *actorIPLDContainer.MarketActor) + transformers, err := market.LookupMarketStateTransformer(av) + if err != nil { + return nil, err + } + marketModels, err := market.TransformMarketState(ctx, s, current, parent, *actorIPLDContainer.MarketActor, transformers...) if err != nil { return nil, err } out = append(out, marketModels) } + if actorIPLDContainer.MinerActors != nil { - minerModels, err := ProcessMiners(ctx, s, current, parent, av, *actorIPLDContainer.MinerActors) + transformers, err := miner.LookupMinerStateTransformer(av) + if err != nil { + return nil, err + } + minerModels, err := miner.TransformMinerStates(ctx, s, current, parent, *actorIPLDContainer.MinerActors, transformers...) if err != nil { return nil, err } out = append(out, minerModels) } + if actorIPLDContainer.InitActor != nil { initModels, err := ProcessInitAddresses(ctx, s, current, parent, av, *actorIPLDContainer.InitActor) if err != nil { @@ -147,6 +141,7 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent } out = append(out, initModels) } + if actorIPLDContainer.RawActors != nil { rawModels, err := ProcessActorStates(ctx, s, current, parent, av, *actorIPLDContainer.RawActors) if err != nil { @@ -154,6 +149,7 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent } out = append(out, rawModels) } + if actorIPLDContainer.VerifregActor != nil { verifregModels, err := ProcessVerifregActor(ctx, s, current, parent, av, *actorIPLDContainer.VerifregActor) if err != nil { diff --git a/tasks/actorstate/account/account.go b/tasks/actorstate/account/account.go index 6ac21a2ab..76564c176 100644 --- a/tasks/actorstate/account/account.go +++ b/tasks/actorstate/account/account.go @@ -16,7 +16,7 @@ type AccountExtractor struct{} // Extract will create persistable data for a given actor's state. func (AccountExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - _, span := otel.Tracer("").Start(ctx, "AccountExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "AccountExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/datacap/balance.go b/tasks/actorstate/datacap/balance.go index a4b91a545..390620581 100644 --- a/tasks/actorstate/datacap/balance.go +++ b/tasks/actorstate/datacap/balance.go @@ -23,7 +23,7 @@ type BalanceExtractor struct{} func (BalanceExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "BalanceExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "BalancesExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "BalancesExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/init_/init.go b/tasks/actorstate/init_/init.go index 5c8dadd74..29cde1fbe 100644 --- a/tasks/actorstate/init_/init.go +++ b/tasks/actorstate/init_/init.go @@ -24,7 +24,7 @@ type InitExtractor struct{} func (InitExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "InitExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "InitExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "InitExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/market/deal_proposal.go b/tasks/actorstate/market/deal_proposal.go index a12ee870f..2b44bb89a 100644 --- a/tasks/actorstate/market/deal_proposal.go +++ b/tasks/actorstate/market/deal_proposal.go @@ -24,7 +24,7 @@ type DealProposalExtractor struct{} func (DealProposalExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DealProposalExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DealProposalExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DealProposalExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/market/deal_state.go b/tasks/actorstate/market/deal_state.go index 225c4769a..0939c8eb7 100644 --- a/tasks/actorstate/market/deal_state.go +++ b/tasks/actorstate/market/deal_state.go @@ -20,7 +20,7 @@ type DealStateExtractor struct{} func (DealStateExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DealStateExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DealStateExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DealStateExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/beneficaries.go b/tasks/actorstate/miner/beneficaries.go index b0316b03d..bbaee38a4 100644 --- a/tasks/actorstate/miner/beneficaries.go +++ b/tasks/actorstate/miner/beneficaries.go @@ -17,7 +17,7 @@ type BeneficiaryExtractor struct{} func (BeneficiaryExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "BeneficiaryExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "BeneficiaryExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "BeneficiaryExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/deadline_info.go b/tasks/actorstate/miner/deadline_info.go index 76f884d0f..e87e2cda9 100644 --- a/tasks/actorstate/miner/deadline_info.go +++ b/tasks/actorstate/miner/deadline_info.go @@ -16,7 +16,7 @@ type DeadlineInfoExtractor struct{} func (DeadlineInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "DeadlineInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "DeadlineInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "DeadlineInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/fee_debt.go b/tasks/actorstate/miner/fee_debt.go index 0a7c9a25f..254e1bb33 100644 --- a/tasks/actorstate/miner/fee_debt.go +++ b/tasks/actorstate/miner/fee_debt.go @@ -16,7 +16,7 @@ type FeeDebtExtractor struct{} func (FeeDebtExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "FeeDebtExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "FeeDebtExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "FeeDebtExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/info.go b/tasks/actorstate/miner/info.go index f545b82ef..f58fcb37c 100644 --- a/tasks/actorstate/miner/info.go +++ b/tasks/actorstate/miner/info.go @@ -22,7 +22,7 @@ type InfoExtractor struct{} func (InfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "InfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "InfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "InfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/locked_funds.go b/tasks/actorstate/miner/locked_funds.go index 0e3c4d0f8..7acde81cd 100644 --- a/tasks/actorstate/miner/locked_funds.go +++ b/tasks/actorstate/miner/locked_funds.go @@ -16,7 +16,7 @@ type LockedFundsExtractor struct{} func (LockedFundsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "LockedFundsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "LockedFundsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "LockedFundsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/post.go b/tasks/actorstate/miner/post.go index a891751fd..45f307be2 100644 --- a/tasks/actorstate/miner/post.go +++ b/tasks/actorstate/miner/post.go @@ -22,7 +22,7 @@ type PoStExtractor struct{} func (PoStExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PoStExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PoStExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PoStExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/precommit.go b/tasks/actorstate/miner/precommit.go index 8c7d70dcc..85c381f73 100644 --- a/tasks/actorstate/miner/precommit.go +++ b/tasks/actorstate/miner/precommit.go @@ -18,7 +18,7 @@ type PreCommitInfoExtractorV8 struct{} func (PreCommitInfoExtractorV8) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PreCommitInfoV8Extractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfo.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfo.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/precommitv9.go b/tasks/actorstate/miner/precommitv9.go index b558c950a..59c213afd 100644 --- a/tasks/actorstate/miner/precommitv9.go +++ b/tasks/actorstate/miner/precommitv9.go @@ -17,7 +17,7 @@ type PreCommitInfoExtractorV9 struct{} func (PreCommitInfoExtractorV9) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "PreCommitInfoV9Extractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfoV9.Extract") + ctx, span := otel.Tracer("").Start(ctx, "PreCommitInfoV9.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector.go b/tasks/actorstate/miner/sector.go index 267288611..567e4c693 100644 --- a/tasks/actorstate/miner/sector.go +++ b/tasks/actorstate/miner/sector.go @@ -17,7 +17,7 @@ type SectorInfoExtractor struct{} func (SectorInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector_deals.go b/tasks/actorstate/miner/sector_deals.go index 32bd078fa..ea5773fe2 100644 --- a/tasks/actorstate/miner/sector_deals.go +++ b/tasks/actorstate/miner/sector_deals.go @@ -18,7 +18,7 @@ type SectorDealsExtractor struct{} func (SectorDealsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorDealsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorDealsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorDealsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sector_events.go b/tasks/actorstate/miner/sector_events.go index 28ab6fdef..ef4913a5c 100644 --- a/tasks/actorstate/miner/sector_events.go +++ b/tasks/actorstate/miner/sector_events.go @@ -23,7 +23,7 @@ type SectorEventsExtractor struct{} func (SectorEventsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "SectorEventsExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "SectorEventsExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "SectorEventsExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/miner/sectorv7.go b/tasks/actorstate/miner/sectorv7.go index c02eba91e..f2ba52250 100644 --- a/tasks/actorstate/miner/sectorv7.go +++ b/tasks/actorstate/miner/sectorv7.go @@ -17,7 +17,7 @@ type V7SectorInfoExtractor struct{} func (V7SectorInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "V7SectorInfoExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "V7SectorInfoExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "V7SectorInfoExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/multisig/multisig.go b/tasks/actorstate/multisig/multisig.go index a739bcb65..87fca0e8e 100644 --- a/tasks/actorstate/multisig/multisig.go +++ b/tasks/actorstate/multisig/multisig.go @@ -23,7 +23,7 @@ type MultiSigActorExtractor struct{} func (MultiSigActorExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "MultiSigActorExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "MultiSigExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "MultiSigExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/chain_power.go b/tasks/actorstate/power/chain_power.go index c068f23be..cc992491f 100644 --- a/tasks/actorstate/power/chain_power.go +++ b/tasks/actorstate/power/chain_power.go @@ -20,7 +20,7 @@ type ChainPowerExtractor struct{} func (ChainPowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ChainPowerExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "ChainPowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "ChainPowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/claimed_power.go b/tasks/actorstate/power/claimed_power.go index 5dc8810bf..6e2842768 100644 --- a/tasks/actorstate/power/claimed_power.go +++ b/tasks/actorstate/power/claimed_power.go @@ -19,7 +19,7 @@ type ClaimedPowerExtractor struct{} func (c ClaimedPowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ClaimedPowerExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "ClaimedPowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "ClaimedPowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/power/power.go b/tasks/actorstate/power/power.go index 40ead37de..727ab33a0 100644 --- a/tasks/actorstate/power/power.go +++ b/tasks/actorstate/power/power.go @@ -68,7 +68,7 @@ func (p *PowerStateExtractionContext) HasPreviousState() bool { } func (StoragePowerExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - ctx, span := otel.Tracer("").Start(ctx, "StoragePowerExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "StoragePowerExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/raw/actor.go b/tasks/actorstate/raw/actor.go index 1eec97005..db0d72f6d 100644 --- a/tasks/actorstate/raw/actor.go +++ b/tasks/actorstate/raw/actor.go @@ -19,9 +19,9 @@ var log = logging.Logger("lily/tasks/rawactor") type RawActorExtractor struct{} func (RawActorExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - log.Debugw("Extract", zap.String("extractor", "RawActorExtractor"), zap.Inline(a)) + log.Debugw("Transform", zap.String("extractor", "RawActorExtractor"), zap.Inline(a)) - _, span := otel.Tracer("").Start(ctx, "RawActorExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "RawActorExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/raw/actor_state.go b/tasks/actorstate/raw/actor_state.go index c599f8759..7319de31b 100644 --- a/tasks/actorstate/raw/actor_state.go +++ b/tasks/actorstate/raw/actor_state.go @@ -16,8 +16,8 @@ import ( type RawActorStateExtractor struct{} func (RawActorStateExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - log.Debugw("Extract", zap.String("extractor", "RawStateActorExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "RawActorStateExtractor.Extract") + log.Debugw("Transform", zap.String("extractor", "RawStateActorExtractor"), zap.Inline(a)) + ctx, span := otel.Tracer("").Start(ctx, "RawActorStateExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/reward/reward.go b/tasks/actorstate/reward/reward.go index f32519f44..2ed53af01 100644 --- a/tasks/actorstate/reward/reward.go +++ b/tasks/actorstate/reward/reward.go @@ -21,7 +21,7 @@ type RewardExtractor struct{} func (RewardExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "RewardExtractor"), zap.Inline(a)) - _, span := otel.Tracer("").Start(ctx, "RewardExtractor.Extract") + _, span := otel.Tracer("").Start(ctx, "RewardExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/client.go b/tasks/actorstate/verifreg/client.go index c958f7fbd..9c8075f08 100644 --- a/tasks/actorstate/verifreg/client.go +++ b/tasks/actorstate/verifreg/client.go @@ -19,7 +19,7 @@ type ClientExtractor struct{} func (ClientExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "ClientExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryClientExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryClientExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/verifier.go b/tasks/actorstate/verifreg/verifier.go index a7ef3b477..93e6f2413 100644 --- a/tasks/actorstate/verifreg/verifier.go +++ b/tasks/actorstate/verifreg/verifier.go @@ -22,7 +22,7 @@ type VerifierExtractor struct{} func (VerifierExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { log.Debugw("extract", zap.String("extractor", "VerifierExtractor"), zap.Inline(a)) - ctx, span := otel.Tracer("").Start(ctx, "VerifierExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifierExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) diff --git a/tasks/actorstate/verifreg/verifreg.go b/tasks/actorstate/verifreg/verifreg.go index e16dd7f8c..022cb5eda 100644 --- a/tasks/actorstate/verifreg/verifreg.go +++ b/tasks/actorstate/verifreg/verifreg.go @@ -66,7 +66,7 @@ func NewVerifiedRegistryExtractorContext(ctx context.Context, a actorstate.Actor } func (VerifiedRegistryExtractor) Extract(ctx context.Context, a actorstate.ActorInfo, node actorstate.ActorStateAPI) (model.Persistable, error) { - ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryExtractor.Extract") + ctx, span := otel.Tracer("").Start(ctx, "VerifiedRegistryExtractor.Transform") defer span.End() if span.IsRecording() { span.SetAttributes(a.Attributes()...) From 5c7ca7b52b9f73d05dcaf0d87ed133d7796be7d2 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 25 Jan 2023 20:19:59 -0800 Subject: [PATCH 36/48] refactor timescale transformer - wire up power actor --- .../verifregdiff/{v0 => v1}/cbor_gen.go | 2 +- .../actors/verifregdiff/{v0 => v1}/clients.go | 2 +- .../actors/verifregdiff/{v0 => v1}/load.go | 2 +- .../actors/verifregdiff/{v0 => v1}/state.go | 2 +- .../verifregdiff/{v0 => v1}/verifiers.go | 2 +- .../verifregdiff/{v9 => v2}/allocations.go | 4 +- .../verifregdiff/{v9 => v2}/cbor_gen.go | 2 +- .../actors/verifregdiff/{v9 => v2}/claims.go | 4 +- pkg/extract/actors/verifregdiff/v2/clients.go | 7 - .../actors/verifregdiff/{v9 => v2}/load.go | 2 +- pkg/extract/actors/verifregdiff/v2/state.go | 115 +++++++++++- .../actors/verifregdiff/v2/verifiers.go | 4 +- pkg/extract/actors/verifregdiff/v3/clients.go | 7 - pkg/extract/actors/verifregdiff/v3/state.go | 5 - .../actors/verifregdiff/v3/verifiers.go | 7 - pkg/extract/actors/verifregdiff/v4/clients.go | 7 - pkg/extract/actors/verifregdiff/v4/state.go | 5 - .../actors/verifregdiff/v4/verifiers.go | 7 - pkg/extract/actors/verifregdiff/v5/clients.go | 7 - pkg/extract/actors/verifregdiff/v5/state.go | 5 - .../actors/verifregdiff/v5/verifiers.go | 7 - pkg/extract/actors/verifregdiff/v6/clients.go | 7 - pkg/extract/actors/verifregdiff/v6/state.go | 5 - .../actors/verifregdiff/v6/verifiers.go | 7 - pkg/extract/actors/verifregdiff/v7/clients.go | 7 - pkg/extract/actors/verifregdiff/v7/state.go | 5 - .../actors/verifregdiff/v7/verifiers.go | 7 - pkg/extract/actors/verifregdiff/v8/clients.go | 7 - pkg/extract/actors/verifregdiff/v8/state.go | 5 - .../actors/verifregdiff/v8/verifiers.go | 9 - pkg/extract/actors/verifregdiff/v9/state.go | 117 ------------ .../actors/verifregdiff/v9/verifiers.go | 9 - pkg/extract/actors/verifregdiff/version.go | 68 +------ pkg/gen/main.go | 4 +- pkg/transform/cbor/actors/serialize.go | 1 + .../timescale/actors/market/router.go | 95 ++-------- .../timescale/actors/market/v1/router.go | 102 +++++++++++ .../actors/market/{ => v1}/v0/deals.go | 0 .../actors/market/{ => v1}/v0/proposals.go | 0 .../actors/market/{ => v1}/v2/deals.go | 0 .../actors/market/{ => v1}/v2/proposals.go | 0 .../actors/market/{ => v1}/v3/deals.go | 0 .../actors/market/{ => v1}/v3/proposals.go | 0 .../actors/market/{ => v1}/v4/deals.go | 0 .../actors/market/{ => v1}/v4/proposals.go | 0 .../actors/market/{ => v1}/v5/deals.go | 0 .../actors/market/{ => v1}/v5/proposals.go | 0 .../actors/market/{ => v1}/v6/deals.go | 0 .../actors/market/{ => v1}/v6/proposals.go | 0 .../actors/market/{ => v1}/v7/deals.go | 0 .../actors/market/{ => v1}/v7/proposals.go | 0 .../actors/market/{ => v1}/v8/deals.go | 0 .../actors/market/{ => v1}/v8/proposals.go | 0 .../actors/market/{ => v1}/v9/deals.go | 0 .../actors/market/{ => v1}/v9/proposals.go | 0 .../timescale/actors/miner/router.go | 166 ++++-------------- .../timescale/actors/miner/v1/router.go | 151 ++++++++++++++++ .../actors/miner/{ => v1}/v0/info.go | 0 .../actors/miner/{ => v1}/v0/precommits.go | 0 .../actors/miner/{ => v1}/v0/sector_deals.go | 0 .../actors/miner/{ => v1}/v0/sector_events.go | 0 .../actors/miner/{ => v1}/v0/sectors.go | 0 .../actors/miner/{ => v1}/v0/state.go | 0 .../actors/miner/{ => v1}/v2/info.go | 0 .../actors/miner/{ => v1}/v2/precommits.go | 0 .../actors/miner/{ => v1}/v2/sector_deals.go | 0 .../actors/miner/{ => v1}/v2/sector_events.go | 0 .../actors/miner/{ => v1}/v2/sectors.go | 0 .../actors/miner/{ => v1}/v2/state.go | 0 .../actors/miner/{ => v1}/v3/info.go | 0 .../actors/miner/{ => v1}/v3/precommits.go | 0 .../actors/miner/{ => v1}/v3/sector_deals.go | 0 .../actors/miner/{ => v1}/v3/sector_events.go | 0 .../actors/miner/{ => v1}/v3/sectors.go | 0 .../actors/miner/{ => v1}/v3/state.go | 0 .../actors/miner/{ => v1}/v4/info.go | 0 .../actors/miner/{ => v1}/v4/precommits.go | 0 .../actors/miner/{ => v1}/v4/sector_deals.go | 0 .../actors/miner/{ => v1}/v4/sector_events.go | 0 .../actors/miner/{ => v1}/v4/sectors.go | 0 .../actors/miner/{ => v1}/v4/state.go | 0 .../actors/miner/{ => v1}/v5/info.go | 0 .../actors/miner/{ => v1}/v5/precommits.go | 0 .../actors/miner/{ => v1}/v5/sector_deals.go | 0 .../actors/miner/{ => v1}/v5/sector_events.go | 0 .../actors/miner/{ => v1}/v5/sectors.go | 0 .../actors/miner/{ => v1}/v5/state.go | 0 .../actors/miner/{ => v1}/v6/info.go | 0 .../actors/miner/{ => v1}/v6/precommits.go | 0 .../actors/miner/{ => v1}/v6/sector_deals.go | 0 .../actors/miner/{ => v1}/v6/sector_events.go | 0 .../actors/miner/{ => v1}/v6/sectors.go | 0 .../actors/miner/{ => v1}/v6/state.go | 0 .../actors/miner/{ => v1}/v7/info.go | 0 .../actors/miner/{ => v1}/v7/precommits.go | 0 .../actors/miner/{ => v1}/v7/sector_deals.go | 0 .../actors/miner/{ => v1}/v7/sector_events.go | 0 .../actors/miner/{ => v1}/v7/sectors.go | 0 .../actors/miner/{ => v1}/v7/state.go | 0 .../actors/miner/{ => v1}/v8/info.go | 0 .../actors/miner/{ => v1}/v8/precommits.go | 0 .../actors/miner/{ => v1}/v8/sector_deals.go | 0 .../actors/miner/{ => v1}/v8/sector_events.go | 0 .../actors/miner/{ => v1}/v8/sectors.go | 0 .../actors/miner/{ => v1}/v8/state.go | 0 .../actors/miner/{ => v1}/v9/info.go | 0 .../actors/miner/{ => v1}/v9/precommits.go | 0 .../actors/miner/{ => v1}/v9/sector_deals.go | 0 .../actors/miner/{ => v1}/v9/sector_events.go | 0 .../actors/miner/{ => v1}/v9/sectors.go | 0 .../actors/miner/{ => v1}/v9/state.go | 0 .../timescale/actors/power/router.go | 68 +++++++ .../timescale/actors/power/v1/router.go | 93 ++++++++++ .../timescale/actors/power/v1/v0/claims.go | 43 +++++ .../timescale/actors/power/v1/v0/state.go | 38 ++++ .../timescale/actors/power/v1/v2/claims.go | 43 +++++ .../timescale/actors/power/v1/v2/state.go | 38 ++++ .../timescale/actors/power/v1/v3/claims.go | 43 +++++ .../timescale/actors/power/v1/v3/state.go | 38 ++++ .../timescale/actors/power/v1/v4/claims.go | 43 +++++ .../timescale/actors/power/v1/v4/state.go | 38 ++++ .../timescale/actors/power/v1/v5/claims.go | 43 +++++ .../timescale/actors/power/v1/v5/state.go | 38 ++++ .../timescale/actors/power/v1/v6/claims.go | 43 +++++ .../timescale/actors/power/v1/v6/state.go | 38 ++++ .../timescale/actors/power/v1/v7/claims.go | 43 +++++ .../timescale/actors/power/v1/v7/state.go | 38 ++++ .../timescale/actors/power/v1/v8/claims.go | 43 +++++ .../timescale/actors/power/v1/v8/state.go | 38 ++++ .../timescale/actors/power/v1/v9/claims.go | 43 +++++ .../timescale/actors/power/v1/v9/state.go | 38 ++++ .../timescale/actors/verifreg/router.go | 70 ++------ .../timescale/actors/verifreg/v1/router.go | 93 ++++++++++ .../actors/verifreg/{ => v1}/v0/clients.go | 2 +- .../actors/verifreg/{ => v1}/v0/verifiers.go | 2 +- .../actors/verifreg/v1/v2/clients.go | 62 +++++++ .../actors/verifreg/v1/v2/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v3/clients.go | 62 +++++++ .../actors/verifreg/v1/v3/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v4/clients.go | 62 +++++++ .../actors/verifreg/v1/v4/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v5/clients.go | 62 +++++++ .../actors/verifreg/v1/v5/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v6/clients.go | 62 +++++++ .../actors/verifreg/v1/v6/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v7/clients.go | 62 +++++++ .../actors/verifreg/v1/v7/verifiers.go | 62 +++++++ .../actors/verifreg/v1/v8/clients.go | 62 +++++++ .../actors/verifreg/v1/v8/verifiers.go | 62 +++++++ .../timescale/actors/verifreg/v2/router.go | 52 ++++++ .../verifreg/{ => v2}/v9/allocations.go | 4 +- .../actors/verifreg/{ => v2}/v9/claims.go | 4 +- .../actors/verifreg/{ => v2}/v9/verifiers.go | 2 +- pkg/transform/timescale/processor.go | 39 ++-- 154 files changed, 2383 insertions(+), 620 deletions(-) rename pkg/extract/actors/verifregdiff/{v0 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/verifregdiff/{v0 => v1}/clients.go (99%) rename pkg/extract/actors/verifregdiff/{v0 => v1}/load.go (99%) rename pkg/extract/actors/verifregdiff/{v0 => v1}/state.go (99%) rename pkg/extract/actors/verifregdiff/{v0 => v1}/verifiers.go (99%) rename pkg/extract/actors/verifregdiff/{v9 => v2}/allocations.go (99%) rename pkg/extract/actors/verifregdiff/{v9 => v2}/cbor_gen.go (99%) rename pkg/extract/actors/verifregdiff/{v9 => v2}/claims.go (99%) delete mode 100644 pkg/extract/actors/verifregdiff/v2/clients.go rename pkg/extract/actors/verifregdiff/{v9 => v2}/load.go (98%) delete mode 100644 pkg/extract/actors/verifregdiff/v3/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v3/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v3/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v4/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v4/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v4/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v5/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v5/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v5/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v6/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v6/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v6/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v7/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v7/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v7/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v8/clients.go delete mode 100644 pkg/extract/actors/verifregdiff/v8/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v8/verifiers.go delete mode 100644 pkg/extract/actors/verifregdiff/v9/state.go delete mode 100644 pkg/extract/actors/verifregdiff/v9/verifiers.go create mode 100644 pkg/transform/timescale/actors/market/v1/router.go rename pkg/transform/timescale/actors/market/{ => v1}/v0/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v0/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v2/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v2/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v3/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v3/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v4/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v4/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v5/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v5/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v6/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v6/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v7/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v7/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v8/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v8/proposals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v9/deals.go (100%) rename pkg/transform/timescale/actors/market/{ => v1}/v9/proposals.go (100%) create mode 100644 pkg/transform/timescale/actors/miner/v1/router.go rename pkg/transform/timescale/actors/miner/{ => v1}/v0/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v0/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v0/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v0/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v0/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v0/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v2/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v3/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v4/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v5/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v6/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v7/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v8/state.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/info.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/precommits.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/sector_deals.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/sector_events.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/sectors.go (100%) rename pkg/transform/timescale/actors/miner/{ => v1}/v9/state.go (100%) create mode 100644 pkg/transform/timescale/actors/power/router.go create mode 100644 pkg/transform/timescale/actors/power/v1/router.go create mode 100644 pkg/transform/timescale/actors/power/v1/v0/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v0/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v2/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v2/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v3/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v3/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v4/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v4/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v5/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v5/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v6/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v6/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v7/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v7/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v8/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v8/state.go create mode 100644 pkg/transform/timescale/actors/power/v1/v9/claims.go create mode 100644 pkg/transform/timescale/actors/power/v1/v9/state.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/router.go rename pkg/transform/timescale/actors/verifreg/{ => v1}/v0/clients.go (99%) rename pkg/transform/timescale/actors/verifreg/{ => v1}/v0/verifiers.go (99%) create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v2/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v3/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v4/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v5/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v6/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v7/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v8/clients.go create mode 100644 pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go create mode 100644 pkg/transform/timescale/actors/verifreg/v2/router.go rename pkg/transform/timescale/actors/verifreg/{ => v2}/v9/allocations.go (57%) rename pkg/transform/timescale/actors/verifreg/{ => v2}/v9/claims.go (58%) rename pkg/transform/timescale/actors/verifreg/{ => v2}/v9/verifiers.go (99%) diff --git a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go b/pkg/extract/actors/verifregdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v0/cbor_gen.go rename to pkg/extract/actors/verifregdiff/v1/cbor_gen.go index 714d557be..adfef2e46 100644 --- a/pkg/extract/actors/verifregdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v0 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/verifregdiff/v0/clients.go b/pkg/extract/actors/verifregdiff/v1/clients.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v0/clients.go rename to pkg/extract/actors/verifregdiff/v1/clients.go index fc0c5e75e..52a9bc2ae 100644 --- a/pkg/extract/actors/verifregdiff/v0/clients.go +++ b/pkg/extract/actors/verifregdiff/v1/clients.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/verifregdiff/v0/load.go b/pkg/extract/actors/verifregdiff/v1/load.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v0/load.go rename to pkg/extract/actors/verifregdiff/v1/load.go index 2722c6ae6..3e88d9e8e 100644 --- a/pkg/extract/actors/verifregdiff/v0/load.go +++ b/pkg/extract/actors/verifregdiff/v1/load.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/verifregdiff/v0/state.go b/pkg/extract/actors/verifregdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v0/state.go rename to pkg/extract/actors/verifregdiff/v1/state.go index 7a572728f..095cb513b 100644 --- a/pkg/extract/actors/verifregdiff/v0/state.go +++ b/pkg/extract/actors/verifregdiff/v1/state.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/verifregdiff/v0/verifiers.go b/pkg/extract/actors/verifregdiff/v1/verifiers.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v0/verifiers.go rename to pkg/extract/actors/verifregdiff/v1/verifiers.go index 423c5b147..39d95c2ed 100644 --- a/pkg/extract/actors/verifregdiff/v0/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v1/verifiers.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/verifregdiff/v9/allocations.go b/pkg/extract/actors/verifregdiff/v2/allocations.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v9/allocations.go rename to pkg/extract/actors/verifregdiff/v2/allocations.go index 2a57da891..16f0fce7f 100644 --- a/pkg/extract/actors/verifregdiff/v9/allocations.go +++ b/pkg/extract/actors/verifregdiff/v2/allocations.go @@ -1,4 +1,4 @@ -package v9 +package v2 import ( "context" @@ -17,7 +17,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/generic" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" "github.com/filecoin-project/lily/tasks" ) diff --git a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go b/pkg/extract/actors/verifregdiff/v2/cbor_gen.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v9/cbor_gen.go rename to pkg/extract/actors/verifregdiff/v2/cbor_gen.go index b13a960bd..d836c2eb6 100644 --- a/pkg/extract/actors/verifregdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/verifregdiff/v2/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v9 +package v2 import ( "fmt" diff --git a/pkg/extract/actors/verifregdiff/v9/claims.go b/pkg/extract/actors/verifregdiff/v2/claims.go similarity index 99% rename from pkg/extract/actors/verifregdiff/v9/claims.go rename to pkg/extract/actors/verifregdiff/v2/claims.go index ae6ca0594..594b8e5a8 100644 --- a/pkg/extract/actors/verifregdiff/v9/claims.go +++ b/pkg/extract/actors/verifregdiff/v2/claims.go @@ -1,4 +1,4 @@ -package v9 +package v2 import ( "context" @@ -11,7 +11,7 @@ import ( "go.uber.org/zap" "github.com/filecoin-project/lily/chain/actors/builtin/verifreg" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" adt2 "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" diff --git a/pkg/extract/actors/verifregdiff/v2/clients.go b/pkg/extract/actors/verifregdiff/v2/clients.go deleted file mode 100644 index 501b0bbb5..000000000 --- a/pkg/extract/actors/verifregdiff/v2/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v9/load.go b/pkg/extract/actors/verifregdiff/v2/load.go similarity index 98% rename from pkg/extract/actors/verifregdiff/v9/load.go rename to pkg/extract/actors/verifregdiff/v2/load.go index 46cb8e10c..40c397138 100644 --- a/pkg/extract/actors/verifregdiff/v9/load.go +++ b/pkg/extract/actors/verifregdiff/v2/load.go @@ -1,4 +1,4 @@ -package v9 +package v2 import ( "github.com/filecoin-project/lily/chain/actors/adt" diff --git a/pkg/extract/actors/verifregdiff/v2/state.go b/pkg/extract/actors/verifregdiff/v2/state.go index 7aebd90b3..b05272bc0 100644 --- a/pkg/extract/actors/verifregdiff/v2/state.go +++ b/pkg/extract/actors/verifregdiff/v2/state.go @@ -1,5 +1,116 @@ package v2 -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +import ( + "context" -type StateDiff = v0.StateDiff + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/ipfs/go-cid" + logging "github.com/ipfs/go-log/v2" + cbg "github.com/whyrusleeping/cbor-gen" + "golang.org/x/sync/errgroup" + + "github.com/filecoin-project/lily/pkg/extract/actors" + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + "github.com/filecoin-project/lily/tasks" +) + +var log = logging.Logger("extract/actors/verifreg") + +type StateDiff struct { + DiffMethods []actors.ActorStateDiff +} + +func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { + grp, grpctx := errgroup.WithContext(ctx) + results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) + if err != nil { + return nil, err + } + + var stateDiff = new(StateDiffResult) + for _, stateChange := range results { + if stateChange == nil { + continue + } + switch stateChange.Kind() { + case v0.KindVerifregVerifiers: + stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + case KindVerifregClaims: + stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) + case KindVerifregAllocations: + stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) + } + } + return stateDiff, nil +} + +type StateDiffResult struct { + VerifierChanges VerifiersChangeList + ClaimChanges ClaimsChangeMap + AllocationsChanges AllocationsChangeMap +} + +func (sd *StateDiffResult) Kind() string { + return "verifreg" +} + +func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store store.Store) (cbg.CBORMarshaler, error) { + out := &StateChange{} + + if verifiers := sd.VerifierChanges; verifiers != nil { + root, err := verifiers.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Verifiers = &root + } + + if claims := sd.ClaimChanges; claims != nil { + root, err := claims.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Claims = &root + } + + if allocations := sd.AllocationsChanges; allocations != nil { + root, err := allocations.ToAdtMap(store, 5) + if err != nil { + return nil, err + } + out.Allocations = &root + } + return out, nil +} + +type StateChange struct { + Verifiers *cid.Cid `cborgen:"verifiers"` + Claims *cid.Cid `cborgen:"claims"` + Allocations *cid.Cid `cborgen:"allocations"` +} + +func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { + out := &StateDiffResult{ + VerifierChanges: VerifiersChangeList{}, + } + + if sc.Verifiers != nil { + verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) + if err != nil { + return nil, err + } + + verifierChange := new(VerifiersChange) + if err := verifierMap.ForEach(verifierChange, func(key string) error { + val := new(VerifiersChange) + *val = *verifierChange + out.VerifierChanges = append(out.VerifierChanges, val) + return nil + }); err != nil { + return nil, err + } + } + + return out, nil +} diff --git a/pkg/extract/actors/verifregdiff/v2/verifiers.go b/pkg/extract/actors/verifregdiff/v2/verifiers.go index 71df2621a..0cf2e372c 100644 --- a/pkg/extract/actors/verifregdiff/v2/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v2/verifiers.go @@ -1,6 +1,8 @@ package v2 -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" +import ( + v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) type VerifiersChange = v0.VerifiersChange type VerifiersChangeList = v0.VerifiersChangeList diff --git a/pkg/extract/actors/verifregdiff/v3/clients.go b/pkg/extract/actors/verifregdiff/v3/clients.go deleted file mode 100644 index c17bcd44d..000000000 --- a/pkg/extract/actors/verifregdiff/v3/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v3/state.go b/pkg/extract/actors/verifregdiff/v3/state.go deleted file mode 100644 index d4587fd5d..000000000 --- a/pkg/extract/actors/verifregdiff/v3/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v3/verifiers.go b/pkg/extract/actors/verifregdiff/v3/verifiers.go deleted file mode 100644 index abae8239a..000000000 --- a/pkg/extract/actors/verifregdiff/v3/verifiers.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v4/clients.go b/pkg/extract/actors/verifregdiff/v4/clients.go deleted file mode 100644 index c20b0c368..000000000 --- a/pkg/extract/actors/verifregdiff/v4/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v4/state.go b/pkg/extract/actors/verifregdiff/v4/state.go deleted file mode 100644 index cee9a4dfa..000000000 --- a/pkg/extract/actors/verifregdiff/v4/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v4/verifiers.go b/pkg/extract/actors/verifregdiff/v4/verifiers.go deleted file mode 100644 index 9b45da959..000000000 --- a/pkg/extract/actors/verifregdiff/v4/verifiers.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v5/clients.go b/pkg/extract/actors/verifregdiff/v5/clients.go deleted file mode 100644 index f767999b3..000000000 --- a/pkg/extract/actors/verifregdiff/v5/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v5/state.go b/pkg/extract/actors/verifregdiff/v5/state.go deleted file mode 100644 index 3632eab46..000000000 --- a/pkg/extract/actors/verifregdiff/v5/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v5/verifiers.go b/pkg/extract/actors/verifregdiff/v5/verifiers.go deleted file mode 100644 index 1ed52d298..000000000 --- a/pkg/extract/actors/verifregdiff/v5/verifiers.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v6/clients.go b/pkg/extract/actors/verifregdiff/v6/clients.go deleted file mode 100644 index 637882000..000000000 --- a/pkg/extract/actors/verifregdiff/v6/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v6/state.go b/pkg/extract/actors/verifregdiff/v6/state.go deleted file mode 100644 index 41901625c..000000000 --- a/pkg/extract/actors/verifregdiff/v6/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v6/verifiers.go b/pkg/extract/actors/verifregdiff/v6/verifiers.go deleted file mode 100644 index 0c3e02d8a..000000000 --- a/pkg/extract/actors/verifregdiff/v6/verifiers.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v7/clients.go b/pkg/extract/actors/verifregdiff/v7/clients.go deleted file mode 100644 index ef01f2763..000000000 --- a/pkg/extract/actors/verifregdiff/v7/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v7/state.go b/pkg/extract/actors/verifregdiff/v7/state.go deleted file mode 100644 index 35af0c1a3..000000000 --- a/pkg/extract/actors/verifregdiff/v7/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v7/verifiers.go b/pkg/extract/actors/verifregdiff/v7/verifiers.go deleted file mode 100644 index 3573cb58b..000000000 --- a/pkg/extract/actors/verifregdiff/v7/verifiers.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v8/clients.go b/pkg/extract/actors/verifregdiff/v8/clients.go deleted file mode 100644 index 1090cabf5..000000000 --- a/pkg/extract/actors/verifregdiff/v8/clients.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type ClientsChange = v0.ClientsChange -type ClientsChangeList = v0.ClientsChangeList -type Clients = v0.Clients diff --git a/pkg/extract/actors/verifregdiff/v8/state.go b/pkg/extract/actors/verifregdiff/v8/state.go deleted file mode 100644 index 9f295f3e3..000000000 --- a/pkg/extract/actors/verifregdiff/v8/state.go +++ /dev/null @@ -1,5 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/verifregdiff/v8/verifiers.go b/pkg/extract/actors/verifregdiff/v8/verifiers.go deleted file mode 100644 index 9a8a8e9ad..000000000 --- a/pkg/extract/actors/verifregdiff/v8/verifiers.go +++ /dev/null @@ -1,9 +0,0 @@ -package v8 - -import ( - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" -) - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/v9/state.go b/pkg/extract/actors/verifregdiff/v9/state.go deleted file mode 100644 index 65872f67f..000000000 --- a/pkg/extract/actors/verifregdiff/v9/state.go +++ /dev/null @@ -1,117 +0,0 @@ -package v9 - -import ( - "context" - - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" - "github.com/filecoin-project/go-state-types/store" - "github.com/ipfs/go-cid" - logging "github.com/ipfs/go-log/v2" - cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" - - "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v8" - "github.com/filecoin-project/lily/tasks" -) - -var log = logging.Logger("extract/actors/verifreg") - -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } - - var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case v0.KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) - case KindVerifregClaims: - stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) - case KindVerifregAllocations: - stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) - } - } - return stateDiff, nil -} - -type StateDiffResult struct { - VerifierChanges v8.VerifiersChangeList - ClaimChanges ClaimsChangeMap - AllocationsChanges AllocationsChangeMap -} - -func (sd *StateDiffResult) Kind() string { - return "verifreg" -} - -func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store store.Store) (cbg.CBORMarshaler, error) { - out := &StateChange{} - - if verifiers := sd.VerifierChanges; verifiers != nil { - root, err := verifiers.ToAdtMap(store, 5) - if err != nil { - return nil, err - } - out.Verifiers = &root - } - - if claims := sd.ClaimChanges; claims != nil { - root, err := claims.ToAdtMap(store, 5) - if err != nil { - return nil, err - } - out.Claims = &root - } - - if allocations := sd.AllocationsChanges; allocations != nil { - root, err := allocations.ToAdtMap(store, 5) - if err != nil { - return nil, err - } - out.Allocations = &root - } - return out, nil -} - -type StateChange struct { - Verifiers *cid.Cid `cborgen:"verifiers"` - Claims *cid.Cid `cborgen:"claims"` - Allocations *cid.Cid `cborgen:"allocations"` -} - -func (sc *StateChange) ToStateDiffResult(ctx context.Context, s store.Store) (*StateDiffResult, error) { - out := &StateDiffResult{ - VerifierChanges: VerifiersChangeList{}, - } - - if sc.Verifiers != nil { - verifierMap, err := adt.AsMap(s, *sc.Verifiers, 5) - if err != nil { - return nil, err - } - - verifierChange := new(VerifiersChange) - if err := verifierMap.ForEach(verifierChange, func(key string) error { - val := new(VerifiersChange) - *val = *verifierChange - out.VerifierChanges = append(out.VerifierChanges, val) - return nil - }); err != nil { - return nil, err - } - } - - return out, nil -} diff --git a/pkg/extract/actors/verifregdiff/v9/verifiers.go b/pkg/extract/actors/verifregdiff/v9/verifiers.go deleted file mode 100644 index b65a39e8a..000000000 --- a/pkg/extract/actors/verifregdiff/v9/verifiers.go +++ /dev/null @@ -1,9 +0,0 @@ -package v9 - -import ( - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" -) - -type VerifiersChange = v0.VerifiersChange -type VerifiersChangeList = v0.VerifiersChangeList -type Verifiers = v0.Verifiers diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go index 1f9c19e8b..3a9a5fbee 100644 --- a/pkg/extract/actors/verifregdiff/version.go +++ b/pkg/extract/actors/verifregdiff/version.go @@ -6,76 +6,24 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" v2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" - v3 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v3" - v4 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v4" - v5 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v5" - v6 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v6" - v7 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v7" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v8" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { - switch av { - case actortypes.Version0: - return &v0.StateDiff{ + if av < actortypes.Version9 { + return &v1.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v0.Clients{}, - v0.Verifiers{}, + v1.Clients{}, + v1.Verifiers{}, }}, nil - case actortypes.Version2: + } else if av == actortypes.Version9 { return &v2.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v2.Clients{}, v2.Verifiers{}, + v2.Claims{}, + v2.Allocations{}, }}, nil - case actortypes.Version3: - return &v3.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v3.Clients{}, - v3.Verifiers{}, - }}, nil - case actortypes.Version4: - return &v4.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v4.Clients{}, - v4.Verifiers{}, - }}, nil - case actortypes.Version5: - return &v5.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v5.Clients{}, - v5.Verifiers{}, - }}, nil - case actortypes.Version6: - return &v6.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v6.Clients{}, - v6.Verifiers{}, - }}, nil - case actortypes.Version7: - return &v7.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v7.Clients{}, - v7.Verifiers{}, - }}, nil - case actortypes.Version8: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v8.Clients{}, - v8.Verifiers{}, - }}, nil - case actortypes.Version9: - return &v9.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v9.Verifiers{}, - v9.Claims{}, - v9.Allocations{}, - }}, nil - case actortypes.Version10: - panic("Not yet implemented") } return nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/gen/main.go b/pkg/gen/main.go index e8dbf3fcc..0ce82ebfe 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -9,8 +9,8 @@ import ( minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" + verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" "github.com/filecoin-project/lily/pkg/extract/chain" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/actors" diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 235bba3dd..8c4a30792 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -15,6 +15,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) +// TODO version the content at these CIDs type ActorStateChangesIPLD struct { DataCapActor *cid.Cid `cborgen:"datacap"` // DataCap InitActor *cid.Cid `cborgen:"init"` // Init diff --git a/pkg/transform/timescale/actors/market/router.go b/pkg/transform/timescale/actors/market/router.go index dec69b380..8ca8c6d80 100644 --- a/pkg/transform/timescale/actors/market/router.go +++ b/pkg/transform/timescale/actors/market/router.go @@ -10,89 +10,22 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v0" - v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v2" - v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v3" - v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v4" - v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v5" - v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v6" - v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v7" - v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v8" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v9" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1" ) -type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) -} - -func TransformMarketState(ctx context.Context, s store.Store, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { - marketState := new(marketdiff.StateChange) - if err := s.Get(ctx, root, marketState); err != nil { - return nil, err - } - marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err +func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformMarketState(ctx, s, version, current, executed, root) } - out := model.PersistableList{} - for _, t := range transformers { - m, err := t.Transform(ctx, current, executed, marketStateDiff) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, nil -} + return nil, fmt.Errorf("unsupported version : %d", version) -func LookupMarketStateTransformer(av actortypes.Version) ([]Transformer, error) { - switch av { - case actortypes.Version0: - return []Transformer{ - v0.Deals{}, - v0.Proposals{}, - }, nil - case actortypes.Version2: - return []Transformer{ - v2.Deals{}, - v2.Proposals{}, - }, nil - case actortypes.Version3: - return []Transformer{ - v3.Deals{}, - v3.Proposals{}, - }, nil - case actortypes.Version4: - return []Transformer{ - v4.Deals{}, - v4.Proposals{}, - }, nil - case actortypes.Version5: - return []Transformer{ - v5.Deals{}, - v5.Proposals{}, - }, nil - case actortypes.Version6: - return []Transformer{ - v6.Deals{}, - v6.Proposals{}, - }, nil - case actortypes.Version7: - return []Transformer{ - v7.Deals{}, - v7.Proposals{}, - }, nil - case actortypes.Version8: - return []Transformer{ - v8.Deals{}, - v8.Proposals{}, - }, nil - case actortypes.Version9: - return []Transformer{ - v9.Deals{}, - v9.Proposals{}, - }, nil - } - return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) } diff --git a/pkg/transform/timescale/actors/market/v1/router.go b/pkg/transform/timescale/actors/market/v1/router.go new file mode 100644 index 000000000..51c0b5e86 --- /dev/null +++ b/pkg/transform/timescale/actors/market/v1/router.go @@ -0,0 +1,102 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + marketState := new(marketdiff.StateChange) + if err := s.Get(ctx, root, marketState); err != nil { + return nil, err + } + marketStateDiff, err := marketState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + transformers, err := LookupMarketStateTransformer(version) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, marketStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupMarketStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Deals{}, + v1_0.Proposals{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Deals{}, + v1_2.Proposals{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Deals{}, + v1_3.Proposals{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Deals{}, + v1_4.Proposals{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Deals{}, + v1_5.Proposals{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Deals{}, + v1_6.Proposals{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Deals{}, + v1_7.Proposals{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Deals{}, + v1_8.Proposals{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Deals{}, + v1_9.Proposals{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/market/v0/deals.go b/pkg/transform/timescale/actors/market/v1/v0/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v0/deals.go rename to pkg/transform/timescale/actors/market/v1/v0/deals.go diff --git a/pkg/transform/timescale/actors/market/v0/proposals.go b/pkg/transform/timescale/actors/market/v1/v0/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v0/proposals.go rename to pkg/transform/timescale/actors/market/v1/v0/proposals.go diff --git a/pkg/transform/timescale/actors/market/v2/deals.go b/pkg/transform/timescale/actors/market/v1/v2/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v2/deals.go rename to pkg/transform/timescale/actors/market/v1/v2/deals.go diff --git a/pkg/transform/timescale/actors/market/v2/proposals.go b/pkg/transform/timescale/actors/market/v1/v2/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v2/proposals.go rename to pkg/transform/timescale/actors/market/v1/v2/proposals.go diff --git a/pkg/transform/timescale/actors/market/v3/deals.go b/pkg/transform/timescale/actors/market/v1/v3/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v3/deals.go rename to pkg/transform/timescale/actors/market/v1/v3/deals.go diff --git a/pkg/transform/timescale/actors/market/v3/proposals.go b/pkg/transform/timescale/actors/market/v1/v3/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v3/proposals.go rename to pkg/transform/timescale/actors/market/v1/v3/proposals.go diff --git a/pkg/transform/timescale/actors/market/v4/deals.go b/pkg/transform/timescale/actors/market/v1/v4/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v4/deals.go rename to pkg/transform/timescale/actors/market/v1/v4/deals.go diff --git a/pkg/transform/timescale/actors/market/v4/proposals.go b/pkg/transform/timescale/actors/market/v1/v4/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v4/proposals.go rename to pkg/transform/timescale/actors/market/v1/v4/proposals.go diff --git a/pkg/transform/timescale/actors/market/v5/deals.go b/pkg/transform/timescale/actors/market/v1/v5/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v5/deals.go rename to pkg/transform/timescale/actors/market/v1/v5/deals.go diff --git a/pkg/transform/timescale/actors/market/v5/proposals.go b/pkg/transform/timescale/actors/market/v1/v5/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v5/proposals.go rename to pkg/transform/timescale/actors/market/v1/v5/proposals.go diff --git a/pkg/transform/timescale/actors/market/v6/deals.go b/pkg/transform/timescale/actors/market/v1/v6/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v6/deals.go rename to pkg/transform/timescale/actors/market/v1/v6/deals.go diff --git a/pkg/transform/timescale/actors/market/v6/proposals.go b/pkg/transform/timescale/actors/market/v1/v6/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v6/proposals.go rename to pkg/transform/timescale/actors/market/v1/v6/proposals.go diff --git a/pkg/transform/timescale/actors/market/v7/deals.go b/pkg/transform/timescale/actors/market/v1/v7/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v7/deals.go rename to pkg/transform/timescale/actors/market/v1/v7/deals.go diff --git a/pkg/transform/timescale/actors/market/v7/proposals.go b/pkg/transform/timescale/actors/market/v1/v7/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v7/proposals.go rename to pkg/transform/timescale/actors/market/v1/v7/proposals.go diff --git a/pkg/transform/timescale/actors/market/v8/deals.go b/pkg/transform/timescale/actors/market/v1/v8/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v8/deals.go rename to pkg/transform/timescale/actors/market/v1/v8/deals.go diff --git a/pkg/transform/timescale/actors/market/v8/proposals.go b/pkg/transform/timescale/actors/market/v1/v8/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v8/proposals.go rename to pkg/transform/timescale/actors/market/v1/v8/proposals.go diff --git a/pkg/transform/timescale/actors/market/v9/deals.go b/pkg/transform/timescale/actors/market/v1/v9/deals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v9/deals.go rename to pkg/transform/timescale/actors/market/v1/v9/deals.go diff --git a/pkg/transform/timescale/actors/market/v9/proposals.go b/pkg/transform/timescale/actors/market/v1/v9/proposals.go similarity index 100% rename from pkg/transform/timescale/actors/market/v9/proposals.go rename to pkg/transform/timescale/actors/market/v1/v9/proposals.go diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index 986e46407..e31fb2aa2 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -6,45 +6,44 @@ import ( "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v0" - v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v2" - v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v3" - v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v4" - v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v5" - v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v6" - v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v7" - v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v8" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v9" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v9" ) func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { switch version { case actortypes.Version0: - return v0.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_0.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version2: - return v2.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_2.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version3: - return v3.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_3.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version4: - return v4.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_4.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version5: - return v5.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_5.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version6: - return v6.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_6.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version7: - return v7.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_7.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version8: - return v8.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_8.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version9: - return v9.ExtractMinerStateChanges(ctx, current, executed, addr, change) + return v1_9.ExtractMinerStateChanges(ctx, current, executed, addr, change) case actortypes.Version10: panic("not yet implemented") default: @@ -52,120 +51,19 @@ func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr addr } } -func TransformMinerStates(ctx context.Context, s store.Store, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { - out := model.PersistableList{} - - // a map of all miners whose state has changes - minerMap, err := adt.AsMap(s, root, 5) - if err != nil { - return nil, err - } - // iterate over each miner with a state change - minerState := new(minerdiff.StateChange) - if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - // decode the miner state change from the IPLD structure to a type we can inspect. - minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) - if err != nil { - return err - } - for _, t := range transformers { - m, err := t.Transform(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, m) - } - return nil - }); err != nil { - return nil, err +func TransformMinerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformMinerStates(ctx, s, version, current, executed, root) } - return out, nil -} + return nil, fmt.Errorf("unsupported version : %d", version) -type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) -} - -func LookupMinerStateTransformer(av actortypes.Version) ([]Transformer, error) { - switch av { - case actortypes.Version0: - return []Transformer{ - v0.Info{}, - v0.PreCommit{}, - v0.SectorDeal{}, - v0.SectorEvent{}, - v0.Sector{}, - }, nil - case actortypes.Version2: - return []Transformer{ - v2.Info{}, - v2.PreCommit{}, - v2.SectorDeal{}, - v2.SectorEvent{}, - v2.Sector{}, - }, nil - case actortypes.Version3: - return []Transformer{ - v3.Info{}, - v3.PreCommit{}, - v3.SectorDeal{}, - v3.SectorEvent{}, - v3.Sector{}, - }, nil - case actortypes.Version4: - return []Transformer{ - v4.Info{}, - v4.PreCommit{}, - v4.SectorDeal{}, - v4.SectorEvent{}, - v4.Sector{}, - }, nil - case actortypes.Version5: - return []Transformer{ - v5.Info{}, - v5.PreCommit{}, - v5.SectorDeal{}, - v5.SectorEvent{}, - v5.Sector{}, - }, nil - case actortypes.Version6: - return []Transformer{ - v6.Info{}, - v6.PreCommit{}, - v6.SectorDeal{}, - v6.SectorEvent{}, - v6.Sector{}, - }, nil - case actortypes.Version7: - return []Transformer{ - v7.Info{}, - v7.PreCommit{}, - v7.SectorDeal{}, - v7.SectorEvent{}, - v7.Sector{}, - }, nil - case actortypes.Version8: - return []Transformer{ - v8.Info{}, - v8.PreCommit{}, - v8.SectorDeal{}, - v8.SectorEvent{}, - v8.Sector{}, - }, nil - case actortypes.Version9: - return []Transformer{ - v9.Info{}, - v9.PreCommit{}, - v9.SectorDeal{}, - v9.SectorEvent{}, - v9.Sector{}, - }, nil - - } - return nil, fmt.Errorf("unsupported actor version for miner transform: %d", av) } diff --git a/pkg/transform/timescale/actors/miner/v1/router.go b/pkg/transform/timescale/actors/miner/v1/router.go new file mode 100644 index 000000000..1dcdefd52 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/router.go @@ -0,0 +1,151 @@ +package v1 + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v9" +) + +func TransformMinerStates(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + out := model.PersistableList{} + + // a map of all miners whose state has changes + minerMap, err := adt.AsMap(s, root, 5) + if err != nil { + return nil, err + } + // iterate over each miner with a state change + transformers, err := LookupMinerStateTransformer(version) + if err != nil { + return nil, err + } + minerState := new(minerdiff.StateChange) + if err := minerMap.ForEach(minerState, func(key string) error { + + // the map is keyed by the miner address, the value of the map contains the miners state change + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + + // decode the miner state change from the IPLD structure to a type we can inspect. + minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) + if err != nil { + return err + } + + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, addr, minerStateDiff) + if err != nil { + return err + } + out = append(out, m) + } + + return nil + }); err != nil { + return nil, err + } + return out, nil +} + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) +} + +func LookupMinerStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Info{}, + v1_0.PreCommit{}, + v1_0.SectorDeal{}, + v1_0.SectorEvent{}, + v1_0.Sector{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Info{}, + v1_2.PreCommit{}, + v1_2.SectorDeal{}, + v1_2.SectorEvent{}, + v1_2.Sector{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Info{}, + v1_3.PreCommit{}, + v1_3.SectorDeal{}, + v1_3.SectorEvent{}, + v1_3.Sector{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Info{}, + v1_4.PreCommit{}, + v1_4.SectorDeal{}, + v1_4.SectorEvent{}, + v1_4.Sector{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Info{}, + v1_5.PreCommit{}, + v1_5.SectorDeal{}, + v1_5.SectorEvent{}, + v1_5.Sector{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Info{}, + v1_6.PreCommit{}, + v1_6.SectorDeal{}, + v1_6.SectorEvent{}, + v1_6.Sector{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Info{}, + v1_7.PreCommit{}, + v1_7.SectorDeal{}, + v1_7.SectorEvent{}, + v1_7.Sector{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Info{}, + v1_8.PreCommit{}, + v1_8.SectorDeal{}, + v1_8.SectorEvent{}, + v1_8.Sector{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Info{}, + v1_9.PreCommit{}, + v1_9.SectorDeal{}, + v1_9.SectorEvent{}, + v1_9.Sector{}, + }, nil + + } + return nil, fmt.Errorf("unsupported actor version for miner transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/miner/v0/info.go b/pkg/transform/timescale/actors/miner/v1/v0/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/info.go rename to pkg/transform/timescale/actors/miner/v1/v0/info.go diff --git a/pkg/transform/timescale/actors/miner/v0/precommits.go b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v0/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v0/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v0/sectors.go b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v0/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v0/state.go b/pkg/transform/timescale/actors/miner/v1/v0/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v0/state.go rename to pkg/transform/timescale/actors/miner/v1/v0/state.go diff --git a/pkg/transform/timescale/actors/miner/v2/info.go b/pkg/transform/timescale/actors/miner/v1/v2/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/info.go rename to pkg/transform/timescale/actors/miner/v1/v2/info.go diff --git a/pkg/transform/timescale/actors/miner/v2/precommits.go b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v2/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v2/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v2/sectors.go b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v2/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v2/state.go b/pkg/transform/timescale/actors/miner/v1/v2/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v2/state.go rename to pkg/transform/timescale/actors/miner/v1/v2/state.go diff --git a/pkg/transform/timescale/actors/miner/v3/info.go b/pkg/transform/timescale/actors/miner/v1/v3/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/info.go rename to pkg/transform/timescale/actors/miner/v1/v3/info.go diff --git a/pkg/transform/timescale/actors/miner/v3/precommits.go b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v3/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v3/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v3/sectors.go b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v3/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v3/state.go b/pkg/transform/timescale/actors/miner/v1/v3/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v3/state.go rename to pkg/transform/timescale/actors/miner/v1/v3/state.go diff --git a/pkg/transform/timescale/actors/miner/v4/info.go b/pkg/transform/timescale/actors/miner/v1/v4/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/info.go rename to pkg/transform/timescale/actors/miner/v1/v4/info.go diff --git a/pkg/transform/timescale/actors/miner/v4/precommits.go b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v4/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v4/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v4/sectors.go b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v4/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v4/state.go b/pkg/transform/timescale/actors/miner/v1/v4/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v4/state.go rename to pkg/transform/timescale/actors/miner/v1/v4/state.go diff --git a/pkg/transform/timescale/actors/miner/v5/info.go b/pkg/transform/timescale/actors/miner/v1/v5/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/info.go rename to pkg/transform/timescale/actors/miner/v1/v5/info.go diff --git a/pkg/transform/timescale/actors/miner/v5/precommits.go b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v5/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v5/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v5/sectors.go b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v5/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v5/state.go b/pkg/transform/timescale/actors/miner/v1/v5/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v5/state.go rename to pkg/transform/timescale/actors/miner/v1/v5/state.go diff --git a/pkg/transform/timescale/actors/miner/v6/info.go b/pkg/transform/timescale/actors/miner/v1/v6/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/info.go rename to pkg/transform/timescale/actors/miner/v1/v6/info.go diff --git a/pkg/transform/timescale/actors/miner/v6/precommits.go b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v6/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v6/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v6/sectors.go b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v6/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v6/state.go b/pkg/transform/timescale/actors/miner/v1/v6/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v6/state.go rename to pkg/transform/timescale/actors/miner/v1/v6/state.go diff --git a/pkg/transform/timescale/actors/miner/v7/info.go b/pkg/transform/timescale/actors/miner/v1/v7/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/info.go rename to pkg/transform/timescale/actors/miner/v1/v7/info.go diff --git a/pkg/transform/timescale/actors/miner/v7/precommits.go b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v7/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v7/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v7/sectors.go b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v7/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v7/state.go b/pkg/transform/timescale/actors/miner/v1/v7/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v7/state.go rename to pkg/transform/timescale/actors/miner/v1/v7/state.go diff --git a/pkg/transform/timescale/actors/miner/v8/info.go b/pkg/transform/timescale/actors/miner/v1/v8/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/info.go rename to pkg/transform/timescale/actors/miner/v1/v8/info.go diff --git a/pkg/transform/timescale/actors/miner/v8/precommits.go b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v8/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v8/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v8/sectors.go b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v8/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v8/state.go b/pkg/transform/timescale/actors/miner/v1/v8/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v8/state.go rename to pkg/transform/timescale/actors/miner/v1/v8/state.go diff --git a/pkg/transform/timescale/actors/miner/v9/info.go b/pkg/transform/timescale/actors/miner/v1/v9/info.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/info.go rename to pkg/transform/timescale/actors/miner/v1/v9/info.go diff --git a/pkg/transform/timescale/actors/miner/v9/precommits.go b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/precommits.go rename to pkg/transform/timescale/actors/miner/v1/v9/precommits.go diff --git a/pkg/transform/timescale/actors/miner/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/sector_deals.go rename to pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go diff --git a/pkg/transform/timescale/actors/miner/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/sector_events.go rename to pkg/transform/timescale/actors/miner/v1/v9/sector_events.go diff --git a/pkg/transform/timescale/actors/miner/v9/sectors.go b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/sectors.go rename to pkg/transform/timescale/actors/miner/v1/v9/sectors.go diff --git a/pkg/transform/timescale/actors/miner/v9/state.go b/pkg/transform/timescale/actors/miner/v1/v9/state.go similarity index 100% rename from pkg/transform/timescale/actors/miner/v9/state.go rename to pkg/transform/timescale/actors/miner/v1/v9/state.go diff --git a/pkg/transform/timescale/actors/power/router.go b/pkg/transform/timescale/actors/power/router.go new file mode 100644 index 000000000..c22343a4d --- /dev/null +++ b/pkg/transform/timescale/actors/power/router.go @@ -0,0 +1,68 @@ +package power + +import ( + "context" + "fmt" + + "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v9" +) + +func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8, + actortypes.Version9: + return v1.TransformPowerState(ctx, s, version, current, executed, root) + } + return nil, fmt.Errorf("unsupported version : %d", version) +} + +func HandlePower(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { + switch version { + case actortypes.Version0: + return v1_0.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version2: + return v1_2.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version3: + return v1_3.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version4: + return v1_4.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version5: + return v1_5.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version6: + return v1_6.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version7: + return v1_7.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version8: + return v1_8.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version9: + return v1_9.ExtractPowerStateChanges(ctx, current, executed, addr, change) + case actortypes.Version10: + panic("not yet implemented") + default: + return nil, fmt.Errorf("unsupported miner actor version: %d", version) + } +} diff --git a/pkg/transform/timescale/actors/power/v1/router.go b/pkg/transform/timescale/actors/power/v1/router.go new file mode 100644 index 000000000..1b9716fac --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/router.go @@ -0,0 +1,93 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v8" + v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *powerdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + tramsformers, err := LookupPowerStateTransformers(version) + if err != nil { + return nil, err + } + powerState := new(powerdiff.StateChange) + if err := s.Get(ctx, root, powerState); err != nil { + return nil, err + } + powerStateDiff, err := powerState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range tramsformers { + m, err := t.Transform(ctx, current, executed, powerStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupPowerStateTransformers(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Claims{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Claims{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Claims{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Claims{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Claims{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Claims{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Claims{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Claims{}, + }, nil + case actortypes.Version9: + return []Transformer{ + v1_9.Claims{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for power transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/power/v1/v0/claims.go b/pkg/transform/timescale/actors/power/v1/v0/claims.go new file mode 100644 index 000000000..abb264d89 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v0/claims.go @@ -0,0 +1,43 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v0/state.go b/pkg/transform/timescale/actors/power/v1/v0/state.go new file mode 100644 index 000000000..b3056feb7 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v0/state.go @@ -0,0 +1,38 @@ +package v0 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v2/claims.go b/pkg/transform/timescale/actors/power/v1/v2/claims.go new file mode 100644 index 000000000..215f42df9 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v2/claims.go @@ -0,0 +1,43 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v2/state.go b/pkg/transform/timescale/actors/power/v1/v2/state.go new file mode 100644 index 000000000..fb61c5f65 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v2/state.go @@ -0,0 +1,38 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v3/claims.go b/pkg/transform/timescale/actors/power/v1/v3/claims.go new file mode 100644 index 000000000..1018fdf8f --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v3/claims.go @@ -0,0 +1,43 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v3/state.go b/pkg/transform/timescale/actors/power/v1/v3/state.go new file mode 100644 index 000000000..d69662fb2 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v3/state.go @@ -0,0 +1,38 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v4/claims.go b/pkg/transform/timescale/actors/power/v1/v4/claims.go new file mode 100644 index 000000000..514314669 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v4/claims.go @@ -0,0 +1,43 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v4/state.go b/pkg/transform/timescale/actors/power/v1/v4/state.go new file mode 100644 index 000000000..9fdd86b2f --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v4/state.go @@ -0,0 +1,38 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v5/claims.go b/pkg/transform/timescale/actors/power/v1/v5/claims.go new file mode 100644 index 000000000..054b80d4c --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v5/claims.go @@ -0,0 +1,43 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v5/state.go b/pkg/transform/timescale/actors/power/v1/v5/state.go new file mode 100644 index 000000000..7a74a9ae4 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v5/state.go @@ -0,0 +1,38 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v6/claims.go b/pkg/transform/timescale/actors/power/v1/v6/claims.go new file mode 100644 index 000000000..f33b85f88 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v6/claims.go @@ -0,0 +1,43 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v6/state.go b/pkg/transform/timescale/actors/power/v1/v6/state.go new file mode 100644 index 000000000..763113e9b --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v6/state.go @@ -0,0 +1,38 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v7/claims.go b/pkg/transform/timescale/actors/power/v1/v7/claims.go new file mode 100644 index 000000000..ff445d1aa --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v7/claims.go @@ -0,0 +1,43 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v7/state.go b/pkg/transform/timescale/actors/power/v1/v7/state.go new file mode 100644 index 000000000..1084caf62 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v7/state.go @@ -0,0 +1,38 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/power" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v8/claims.go b/pkg/transform/timescale/actors/power/v1/v8/claims.go new file mode 100644 index 000000000..7f1382778 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v8/claims.go @@ -0,0 +1,43 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v8/state.go b/pkg/transform/timescale/actors/power/v1/v8/state.go new file mode 100644 index 000000000..737b8bc01 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v8/state.go @@ -0,0 +1,38 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v9/claims.go b/pkg/transform/timescale/actors/power/v1/v9/claims.go new file mode 100644 index 000000000..ef6ad4923 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v9/claims.go @@ -0,0 +1,43 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v9/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" +) + +type Claims struct{} + +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { + out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) + for _, change := range changes.ClaimsChanges { + // only care about new and modified power entries + if change.Change == core.ChangeTypeRemove { + continue + } + miner, err := address.NewFromBytes(change.Miner) + if err != nil { + return nil, err + } + claim := new(power.Claim) + if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out = append(out, &powermodel.PowerActorClaim{ + Height: int64(current.Height()), + MinerID: miner.String(), + StateRoot: current.ParentState().String(), + RawBytePower: claim.RawBytePower.String(), + QualityAdjPower: claim.QualityAdjPower.String(), + }) + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/power/v1/v9/state.go b/pkg/transform/timescale/actors/power/v1/v9/state.go new file mode 100644 index 000000000..32f82d255 --- /dev/null +++ b/pkg/transform/timescale/actors/power/v1/v9/state.go @@ -0,0 +1,38 @@ +package v9 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v9/power" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + powermodel "github.com/filecoin-project/lily/model/actors/power" + "github.com/filecoin-project/lily/pkg/core" + "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" +) + +func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { + if change.Change == core.ChangeTypeRemove { + panic("power actor should never be removed from the state tree") + } + powerState := new(power.State) + if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { + return nil, err + } + return &powermodel.ChainPower{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + TotalRawBytesPower: powerState.TotalRawBytePower.String(), + TotalQABytesPower: powerState.TotalQualityAdjPower.String(), + TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), + TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), + TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), + QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), + QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), + MinerCount: uint64(powerState.MinerCount), + ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), + }, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/router.go b/pkg/transform/timescale/actors/verifreg/router.go index 33a0d5ad3..fc4c4e158 100644 --- a/pkg/transform/timescale/actors/verifreg/router.go +++ b/pkg/transform/timescale/actors/verifreg/router.go @@ -10,63 +10,25 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" - v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v0" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v9" + v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1" + v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v2" ) -type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) -} - -func TransformVerifregState(ctx context.Context, s store.Store, av actortypes.Version, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { - if av < actortypes.Version9 { - verifregState := new(verifregdiff.StateChange) - if err := s.Get(ctx, root, verifregState); err != nil { - return nil, err - } - verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) - if err != nil { - return nil, err - } - out := model.PersistableList{} - for _, t := range transformers { - m, err := t.Transform(ctx, current, executed, verifrefStateDiff) - if err != nil { - return nil, err - } - out = append(out, m) - } - return out, nil - - } -} +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + switch version { + case actortypes.Version0, + actortypes.Version2, + actortypes.Version3, + actortypes.Version4, + actortypes.Version5, + actortypes.Version6, + actortypes.Version7, + actortypes.Version8: + return v1.TransformVerifregState(ctx, s, version, current, executed, root) -func LookupMarketStateTransformerV8(av actortypes.Version) ([]Transformer, error) { - switch av { - case actortypes.Version0: - return []Transformer{ - v0.Clients{}, - v0.Verifiers{}, - }, nil - case actortypes.Version2: - return []Transformer{}, nil - case actortypes.Version3: - return []Transformer{}, nil - case actortypes.Version4: - return []Transformer{}, nil - case actortypes.Version5: - return []Transformer{}, nil - case actortypes.Version6: - return []Transformer{}, nil - case actortypes.Version7: - return []Transformer{}, nil - case actortypes.Version8: - return []Transformer{}, nil case actortypes.Version9: - return []Transformer{ - v9.Verifiers{}, - }, nil + return v2.TransformVerifregState(ctx, s, version, current, executed, root) } - return nil, fmt.Errorf("unsupported actor version for market transform: %d", av) + return nil, fmt.Errorf("unsupported version : %d", version) + } diff --git a/pkg/transform/timescale/actors/verifreg/v1/router.go b/pkg/transform/timescale/actors/verifreg/v1/router.go new file mode 100644 index 000000000..6a7ecc4ff --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/router.go @@ -0,0 +1,93 @@ +package v1 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v0" + v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v2" + v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v3" + v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v4" + v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v5" + v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v6" + v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v7" + v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1/v8" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { + transformers, err := LookupVerifregStateTransformer(version) + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, root, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, verifrefStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupVerifregStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version0: + return []Transformer{ + v1_0.Clients{}, + v1_0.Verifiers{}, + }, nil + case actortypes.Version2: + return []Transformer{ + v1_2.Clients{}, + v1_2.Verifiers{}, + }, nil + case actortypes.Version3: + return []Transformer{ + v1_3.Clients{}, + v1_3.Verifiers{}, + }, nil + case actortypes.Version4: + return []Transformer{ + v1_4.Clients{}, + v1_4.Verifiers{}, + }, nil + case actortypes.Version5: + return []Transformer{ + v1_5.Clients{}, + v1_5.Verifiers{}, + }, nil + case actortypes.Version6: + return []Transformer{ + v1_6.Clients{}, + v1_6.Verifiers{}, + }, nil + case actortypes.Version7: + return []Transformer{ + v1_7.Clients{}, + v1_7.Verifiers{}, + }, nil + case actortypes.Version8: + return []Transformer{ + v1_8.Clients{}, + v1_8.Verifiers{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for verifreg transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/verifreg/v0/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v0/clients.go similarity index 99% rename from pkg/transform/timescale/actors/verifreg/v0/clients.go rename to pkg/transform/timescale/actors/verifreg/v1/v0/clients.go index e96ad4fa4..fbcde6557 100644 --- a/pkg/transform/timescale/actors/verifreg/v0/clients.go +++ b/pkg/transform/timescale/actors/verifreg/v1/v0/clients.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" "github.com/filecoin-project/lily/pkg/core" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" ) type Clients struct{} diff --git a/pkg/transform/timescale/actors/verifreg/v0/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go similarity index 99% rename from pkg/transform/timescale/actors/verifreg/v0/verifiers.go rename to pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go index 9f9c64701..c747a726e 100644 --- a/pkg/transform/timescale/actors/verifreg/v0/verifiers.go +++ b/pkg/transform/timescale/actors/verifreg/v1/v0/verifiers.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" "github.com/filecoin-project/lily/pkg/core" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v0" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" ) type Verifiers struct{} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go new file mode 100644 index 000000000..9295043e8 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v2/clients.go @@ -0,0 +1,62 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go new file mode 100644 index 000000000..5daf24325 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v2/verifiers.go @@ -0,0 +1,62 @@ +package v2 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v2/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go new file mode 100644 index 000000000..2984ee80c --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v3/clients.go @@ -0,0 +1,62 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go new file mode 100644 index 000000000..e23b97c65 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v3/verifiers.go @@ -0,0 +1,62 @@ +package v3 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v3/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go new file mode 100644 index 000000000..734b4d6ad --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v4/clients.go @@ -0,0 +1,62 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go new file mode 100644 index 000000000..cc3f371f7 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v4/verifiers.go @@ -0,0 +1,62 @@ +package v4 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v4/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go new file mode 100644 index 000000000..c8beba8ee --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v5/clients.go @@ -0,0 +1,62 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go new file mode 100644 index 000000000..401845f74 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v5/verifiers.go @@ -0,0 +1,62 @@ +package v5 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v5/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go new file mode 100644 index 000000000..41017dca6 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v6/clients.go @@ -0,0 +1,62 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go new file mode 100644 index 000000000..456879a97 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v6/verifiers.go @@ -0,0 +1,62 @@ +package v6 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v6/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go new file mode 100644 index 000000000..c3ba6088d --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v7/clients.go @@ -0,0 +1,62 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go new file mode 100644 index 000000000..4e5341784 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v7/verifiers.go @@ -0,0 +1,62 @@ +package v7 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/specs-actors/v7/actors/builtin/verifreg" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go b/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go new file mode 100644 index 000000000..e78844829 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v8/clients.go @@ -0,0 +1,62 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Clients struct{} + +func (Clients) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiedClientsList, len(changes.ClientChanges)) + for i, change := range changes.ClientChanges { + addr, err := address.NewFromBytes(change.Client) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap removed is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifiedClient{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go b/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go new file mode 100644 index 000000000..1b939e195 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v1/v8/verifiers.go @@ -0,0 +1,62 @@ +package v8 + +import ( + "bytes" + "context" + + "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/builtin/v8/verifreg" + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" + "github.com/filecoin-project/lily/pkg/core" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" +) + +type Verifiers struct{} + +func (Verifiers) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { + out := make(verifregmodel.VerifiedRegistryVerifiersList, len(changes.VerifierChanges)) + for i, change := range changes.VerifierChanges { + addr, err := address.NewFromBytes(change.Verifier) + if err != nil { + return nil, err + } + switch change.Change { + case core.ChangeTypeRemove: + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Removed, + DataCap: "0", // data cap remove is zero + } + case core.ChangeTypeAdd: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Added, + DataCap: dcap.String(), + } + case core.ChangeTypeModify: + dcap := new(verifreg.DataCap) + if err := dcap.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + return nil, err + } + out[i] = &verifregmodel.VerifiedRegistryVerifier{ + Height: int64(current.Height()), + StateRoot: current.ParentState().String(), + Address: addr.String(), + Event: verifregmodel.Modified, + DataCap: dcap.String(), + } + } + } + return out, nil +} diff --git a/pkg/transform/timescale/actors/verifreg/v2/router.go b/pkg/transform/timescale/actors/verifreg/v2/router.go new file mode 100644 index 000000000..171e1c9d9 --- /dev/null +++ b/pkg/transform/timescale/actors/verifreg/v2/router.go @@ -0,0 +1,52 @@ +package v2 + +import ( + "context" + "fmt" + + actortypes "github.com/filecoin-project/go-state-types/actors" + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lily/model" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" + v2_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v2/v9" +) + +type Transformer interface { + Transform(ctx context.Context, current, parent *types.TipSet, change *verifregdiff.StateDiffResult) (model.Persistable, error) +} + +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid, transformers ...Transformer) (model.Persistable, error) { + transformers, err := LookupVerifregStateTransformer(version) + verifregState := new(verifregdiff.StateChange) + if err := s.Get(ctx, root, verifregState); err != nil { + return nil, err + } + verifrefStateDiff, err := verifregState.ToStateDiffResult(ctx, s) + if err != nil { + return nil, err + } + out := model.PersistableList{} + for _, t := range transformers { + m, err := t.Transform(ctx, current, executed, verifrefStateDiff) + if err != nil { + return nil, err + } + out = append(out, m) + } + return out, nil +} + +func LookupVerifregStateTransformer(av actortypes.Version) ([]Transformer, error) { + switch av { + case actortypes.Version9: + return []Transformer{ + v2_9.Verifiers{}, + //v2_9.Claims{}, + //v2_9.Allocations{}, + }, nil + } + return nil, fmt.Errorf("unsupported actor version for verifreg transform: %d", av) +} diff --git a/pkg/transform/timescale/actors/verifreg/v9/allocations.go b/pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go similarity index 57% rename from pkg/transform/timescale/actors/verifreg/v9/allocations.go rename to pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go index 39bc88671..8da2488aa 100644 --- a/pkg/transform/timescale/actors/verifreg/v9/allocations.go +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/allocations.go @@ -6,11 +6,11 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" ) type Allocations struct{} -func (Allocations) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { +func (Allocations) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { panic("TODO") } diff --git a/pkg/transform/timescale/actors/verifreg/v9/claims.go b/pkg/transform/timescale/actors/verifreg/v2/v9/claims.go similarity index 58% rename from pkg/transform/timescale/actors/verifreg/v9/claims.go rename to pkg/transform/timescale/actors/verifreg/v2/v9/claims.go index ce6a3a42f..d9bfe4a23 100644 --- a/pkg/transform/timescale/actors/verifreg/v9/claims.go +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/claims.go @@ -6,11 +6,11 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lily/model" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" ) type Claims struct{} -func (Claims) Extract(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *verifregdiff.StateDiffResult) (model.Persistable, error) { panic("TODO") } diff --git a/pkg/transform/timescale/actors/verifreg/v9/verifiers.go b/pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go similarity index 99% rename from pkg/transform/timescale/actors/verifreg/v9/verifiers.go rename to pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go index 89a1b4461..b62b73925 100644 --- a/pkg/transform/timescale/actors/verifreg/v9/verifiers.go +++ b/pkg/transform/timescale/actors/verifreg/v2/v9/verifiers.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" verifregmodel "github.com/filecoin-project/lily/model/actors/verifreg" "github.com/filecoin-project/lily/pkg/core" - verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v9" + verifregdiff "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" ) type Verifiers struct{} diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go index af5b1b7cd..f936ca786 100644 --- a/pkg/transform/timescale/processor.go +++ b/pkg/transform/timescale/processor.go @@ -27,6 +27,7 @@ import ( init_ "github.com/filecoin-project/lily/pkg/transform/timescale/actors/init" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" + "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg" @@ -95,14 +96,6 @@ func ProcessInitAddresses(ctx context.Context, s store.Store, current, executed return init_.InitHandler(ctx, s, current, executed, root) } -func ProcessVerifregActor(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { - verifregHandler, err := verifreg.MakeVerifregProcessor(av) - if err != nil { - return nil, err - } - return verifregHandler(ctx, s, current, executed, root) -} - func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { actorIPLDContainer := new(actors.ActorStateChangesIPLD) if err := s.Get(ctx, root, actorIPLDContainer); err != nil { @@ -111,11 +104,7 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) out := model.PersistableList{} if actorIPLDContainer.MarketActor != nil { - transformers, err := market.LookupMarketStateTransformer(av) - if err != nil { - return nil, err - } - marketModels, err := market.TransformMarketState(ctx, s, current, parent, *actorIPLDContainer.MarketActor, transformers...) + marketModels, err := market.TransformMarketState(ctx, s, av, current, parent, *actorIPLDContainer.MarketActor) if err != nil { return nil, err } @@ -123,11 +112,7 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent } if actorIPLDContainer.MinerActors != nil { - transformers, err := miner.LookupMinerStateTransformer(av) - if err != nil { - return nil, err - } - minerModels, err := miner.TransformMinerStates(ctx, s, current, parent, *actorIPLDContainer.MinerActors, transformers...) + minerModels, err := miner.TransformMinerState(ctx, s, av, current, parent, *actorIPLDContainer.MinerActors) if err != nil { return nil, err } @@ -151,13 +136,21 @@ func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent } if actorIPLDContainer.VerifregActor != nil { - verifregModels, err := ProcessVerifregActor(ctx, s, current, parent, av, *actorIPLDContainer.VerifregActor) + verifregModels, err := verifreg.TransformVerifregState(ctx, s, av, current, parent, *actorIPLDContainer.MarketActor) if err != nil { return nil, err } out = append(out, verifregModels) } + if actorIPLDContainer.PowerActor != nil { + powerModels, err := power.TransformPowerState(ctx, s, av, current, parent, *actorIPLDContainer.PowerActor) + if err != nil { + return nil, err + } + out = append(out, powerModels) + } + return out, nil } @@ -236,6 +229,14 @@ func ProcessActorStates(ctx context.Context, s store.Store, current, executed *t } out = append(out, m) } + + if core.PowerCodes.Has(actorState.Actor.Code) { + m, err := power.HandlePower(ctx, current, executed, addr, actorState, av) + if err != nil { + return err + } + out = append(out, m) + } return nil }); err != nil { From 8205fe300ec472e293f5310e9166fe1c84df70e2 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 26 Jan 2023 11:32:08 -0800 Subject: [PATCH 37/48] tmp: add database tracking of seralized state --- go.mod | 20 ++++--- go.sum | 43 ++++++++++---- pkg/transform/cbor/actors/serialize.go | 39 +++++++++++++ pkg/transform/cbor/router.go | 77 +++++++++++++++++++++++++- 4 files changed, 158 insertions(+), 21 deletions(-) diff --git a/go.mod b/go.mod index 2acefecca..25f62cf68 100644 --- a/go.mod +++ b/go.mod @@ -54,8 +54,8 @@ require ( go.uber.org/fx v1.15.0 go.uber.org/multierr v1.8.0 go.uber.org/zap v1.22.0 - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 - golang.org/x/text v0.3.7 + golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 + golang.org/x/text v0.5.0 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f gopkg.in/cheggaaa/pb.v1 v1.0.28 ) @@ -69,10 +69,11 @@ require ( github.com/ipfs/go-ipld-format v0.4.0 github.com/jedib0t/go-pretty/v6 v6.2.7 github.com/libp2p/go-libp2p v0.22.0 - github.com/libp2p/go-libp2p-peer v0.2.0 github.com/multiformats/go-varint v0.0.6 go.opentelemetry.io/otel/trace v1.7.0 go.uber.org/atomic v1.10.0 + gorm.io/driver/postgres v1.4.6 + gorm.io/gorm v1.24.3 ) require ( @@ -214,12 +215,16 @@ require ( github.com/ipld/go-ipld-prime v0.19.0 // indirect github.com/ipld/go-ipld-selector-text-lite v0.0.1 // indirect github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgx/v5 v5.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect github.com/jessevdk/go-flags v1.4.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect github.com/jpillora/backoff v1.0.0 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect @@ -232,7 +237,6 @@ require ( github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect github.com/libp2p/go-libp2p-connmgr v0.4.0 // indirect - github.com/libp2p/go-libp2p-crypto v0.1.0 // indirect github.com/libp2p/go-libp2p-gostream v0.4.0 // indirect github.com/libp2p/go-libp2p-kad-dht v0.18.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect @@ -321,12 +325,12 @@ require ( go.opentelemetry.io/otel/sdk/export/metric v0.25.0 // indirect go.uber.org/dig v1.12.0 // indirect go4.org v0.0.0-20200411211856-f5505b9728dd // indirect - golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e // indirect + golang.org/x/crypto v0.4.0 // indirect golang.org/x/exp v0.0.0-20220426173459-3bcf042a4bf5 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect - golang.org/x/net v0.0.0-20220812174116-3211cb980234 // indirect - golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab // indirect - golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect + golang.org/x/net v0.3.0 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/term v0.3.0 // indirect golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect golang.org/x/tools v0.1.12 // indirect google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 // indirect diff --git a/go.sum b/go.sum index f1561827e..b934af554 100644 --- a/go.sum +++ b/go.sum @@ -841,6 +841,14 @@ github.com/ipld/go-ipld-selector-text-lite v0.0.1 h1:lNqFsQpBHc3p5xHob2KvEg/iM5d github.com/ipld/go-ipld-selector-text-lite v0.0.1/go.mod h1:U2CQmFb+uWzfIEF3I1arrDa5rwtj00PrpiwwCO+k1RM= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52 h1:QG4CGBqCeuBo6aZlGAamSkxWdgWfZGeE49eUOWJPA4c= github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.2.0 h1:NdPpngX0Y6z6XDFKqmFQaE+bCtkqzvQIOt1wvBlAqs8= +github.com/jackc/pgx/v5 v5.2.0/go.mod h1:Ptn7zmohNsWEsdxRawMzk3gaKma2obW+NWTnKa0S4nk= +github.com/jackc/puddle/v2 v2.1.2/go.mod h1:2lpufsF5mRHO6SuZkm0fNYxM6SWHfvyFj62KwNzgels= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -864,6 +872,9 @@ github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGAR github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8l6qbCUTSiRLG/iKnW3K3/QfPPuSsBt4= @@ -1018,7 +1029,6 @@ github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQR github.com/libp2p/go-libp2p-core v0.13.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.20.0 h1:PGKM74+T+O/FaZNARNW32i90RMBHCcgd/hkum2UQ5eY= github.com/libp2p/go-libp2p-core v0.20.0/go.mod h1:6zR8H7CvQWgYLsbG4on6oLNSGcyKaYFSEYyDt51+bIY= -github.com/libp2p/go-libp2p-crypto v0.1.0 h1:k9MFy+o2zGDNGsaoZl0MA3iZ75qXxr9OOoAZF+sD5OQ= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= @@ -1047,7 +1057,6 @@ github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhU github.com/libp2p/go-libp2p-noise v0.3.0/go.mod h1:JNjHbociDJKHD64KTkzGnzqJ0FEV5gHJa6AB00kbCNQ= github.com/libp2p/go-libp2p-noise v0.5.0 h1:gwJZ/3iH3MRnBrLIyr/YLdCOnmqfJMptlsFFUIc3j0Y= github.com/libp2p/go-libp2p-noise v0.5.0/go.mod h1:CYYpJ+T4pox1d1J/hhUVxuMf6b2s3c41hFUgS8/yFQw= -github.com/libp2p/go-libp2p-peer v0.2.0 h1:EQ8kMjaCUwt/Y5uLgjT8iY2qg0mGUT0N1zUjer50DsY= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= @@ -1724,6 +1733,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.0/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266 h1:O9XLFXGkVswDFmH9LaYpqu+r/AAFWqr0DL6V00KEVFg= github.com/zondax/hid v0.9.1-0.20220302062450-5552068d2266/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= @@ -1860,8 +1870,9 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210813211128-0a44fdfbc16e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= -golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -1973,8 +1984,9 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.0.0-20220812174116-3211cb980234 h1:RDqmgfe7SvlMWoqC3xwQ2blLO3fcWcxMa3eBLRdRW7E= -golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.3.0 h1:VWL6FNY2bEEmsGVKabSlHu5Irp34xmMRoqb/9lF9lxk= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1996,8 +2008,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2104,13 +2117,16 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab h1:2QkjZIsXupsJbJIdSjjUOgWK3aEtzyuh2mPt3l/CkeU= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2118,8 +2134,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2339,6 +2357,11 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/postgres v1.4.6 h1:1FPESNXqIKG5JmraaH2bfCVlMQ7paLoCreFxDtqzwdc= +gorm.io/driver/postgres v1.4.6/go.mod h1:UJChCNLFKeBqQRE+HrkFUbKbq9idPXmTOk2u4Wok8S4= +gorm.io/gorm v1.24.2/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= +gorm.io/gorm v1.24.3 h1:WL2ifUmzR/SLp85CSURAfybcHnGZ+yLSGSxgYXlFBHg= +gorm.io/gorm v1.24.3/go.mod h1:DVrVomtaYTbqs7gB/x2uVvqnXzv0nqjB396B8cG4dBA= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 8c4a30792..944438ef5 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -10,6 +10,7 @@ import ( "github.com/ipfs/go-cid" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" + "gorm.io/gorm" "github.com/filecoin-project/lily/pkg/extract" "github.com/filecoin-project/lily/pkg/extract/actors" @@ -26,6 +27,44 @@ type ActorStateChangesIPLD struct { VerifregActor *cid.Cid `cborgen:"verifreg"` // Veriferg } +func (a *ActorStateChangesIPLD) AsModel() *ActorStateModel { + out := &ActorStateModel{} + if a.DataCapActor != nil { + out.DataCapActor = a.DataCapActor.String() + } + if a.InitActor != nil { + out.InitActor = a.InitActor.String() + } + if a.MarketActor != nil { + out.MarketActor = a.MarketActor.String() + } + if a.MinerActors != nil { + out.MinerActors = a.MinerActors.String() + } + if a.PowerActor != nil { + out.PowerActor = a.PowerActor.String() + } + if a.RawActors != nil { + out.RawActors = a.RawActors.String() + } + if a.VerifregActor != nil { + out.VerifregActor = a.VerifregActor.String() + } + return out +} + +type ActorStateModel struct { + gorm.Model + Height uint64 + DataCapActor string + InitActor string + MarketActor string + MinerActors string + PowerActor string + RawActors string + VerifregActor string +} + func (a *ActorStateChangesIPLD) Attributes() []attribute.KeyValue { var out []attribute.KeyValue if a.DataCapActor != nil { diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 2f2be3816..be8eceb1b 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -15,10 +15,14 @@ import ( "github.com/ipld/go-car/util" "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" + "gorm.io/gorm/schema" + + "gorm.io/driver/postgres" + "gorm.io/gorm" "github.com/filecoin-project/lily/pkg/extract" cboractors "github.com/filecoin-project/lily/pkg/transform/cbor/actors" - messages2 "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + cbormessages "github.com/filecoin-project/lily/pkg/transform/cbor/messages" ) var log = logging.Logger("lily/transform/cbor") @@ -32,6 +36,16 @@ type RootStateIPLD struct { State cid.Cid `cborgen:"state"` // StateExtractionIPLD } +type RootStateModel struct { + gorm.Model + Height uint64 + Cid string + StateVersion uint64 + NetworkName string + NetworkVersion uint64 + StateExtraction string +} + func (r *RootStateIPLD) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ attribute.Int64("state_version", int64(r.StateVersion)), @@ -59,6 +73,17 @@ type StateExtractionIPLD struct { Actors cid.Cid `cborgen:"actors"` } +type StateExtractionModel struct { + gorm.Model + Height uint64 + CurrentTipSet string + ParentTipSet string + BaseFee string + FullBlocks string + ImplicitMessages string + Actors string +} + func (s *StateExtractionIPLD) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ attribute.String("current_tipset", s.Current.Key().String()), @@ -113,12 +138,12 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec return cid.Undef, fmt.Errorf("actor and message executed tipset does not match") } - implicitMsgsAMT, err := messages2.MakeImplicitMessagesHAMT(ctx, store, chainState.Message.ImplicitMessages) + implicitMsgsAMT, err := cbormessages.MakeImplicitMessagesHAMT(ctx, store, chainState.Message.ImplicitMessages) if err != nil { return cid.Undef, err } - fullBlkHAMT, err := messages2.MakeFullBlockHAMT(ctx, store, chainState.Message.FullBlocks) + fullBlkHAMT, err := cbormessages.MakeFullBlockHAMT(ctx, store, chainState.Message.FullBlocks) if err != nil { return cid.Undef, err } @@ -158,5 +183,51 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec if err != nil { return cid.Undef, err } + dsn := "host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable" + db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "lily_cbor", + SingularTable: false, + }, + }) + if err != nil { + return cid.Undef, err + } + if err := db.AutoMigrate(&RootStateModel{}, &StateExtractionModel{}, &cboractors.ActorStateModel{}); err != nil { + return cid.Undef, err + } + rootModel := RootStateModel{ + Height: uint64(current.Height()), + Cid: root.String(), + StateVersion: rootState.StateVersion, + NetworkName: rootState.NetworkName, + NetworkVersion: rootState.NetworkVersion, + StateExtraction: rootState.State.String(), + } + stateModel := StateExtractionModel{ + Height: uint64(current.Height()), + CurrentTipSet: current.String(), + ParentTipSet: executed.String(), + BaseFee: chainState.Message.BaseFee.String(), + FullBlocks: fullBlkHAMT.String(), + ImplicitMessages: implicitMsgsAMT.String(), + Actors: actorStatesRoot.String(), + } + if err := db.Transaction(func(tx *gorm.DB) error { + if err := tx.Create(&rootModel).Error; err != nil { + return err + } + if err := tx.Create(&stateModel).Error; err != nil { + return err + } + as := actorStateContainer.AsModel() + as.Height = uint64(current.Height()) + if err := tx.Create(as).Error; err != nil { + return err + } + return nil + }); err != nil { + return cid.Undef, err + } return root, nil } From 34739c0624a320e7a4a95d8e58c40b77f39efc32 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 26 Jan 2023 13:57:12 -0800 Subject: [PATCH 38/48] version extractors and add a simple binary to show example of extract --- .../datacapdiff/{v9 => v1}/allowances.go | 2 +- .../actors/datacapdiff/{v9 => v1}/balances.go | 2 +- .../actors/datacapdiff/{v9 => v1}/cbor_gen.go | 2 +- .../actors/datacapdiff/{v9 => v1}/load.go | 2 +- .../actors/datacapdiff/{v9 => v1}/state.go | 2 +- pkg/extract/actors/datacapdiff/version.go | 8 +- .../actors/initdiff/{v0 => v1}/addresses.go | 2 +- .../actors/initdiff/{v0 => v1}/cbor_gen.go | 2 +- .../actors/initdiff/{v0 => v1}/load.go | 2 +- .../actors/initdiff/{v0 => v1}/state.go | 2 +- pkg/extract/actors/initdiff/v2/addresses.go | 7 - pkg/extract/actors/initdiff/v2/state.go | 7 - pkg/extract/actors/initdiff/v3/addresses.go | 7 - pkg/extract/actors/initdiff/v3/state.go | 7 - pkg/extract/actors/initdiff/v4/addresses.go | 7 - pkg/extract/actors/initdiff/v4/state.go | 7 - pkg/extract/actors/initdiff/v5/addresses.go | 7 - pkg/extract/actors/initdiff/v5/state.go | 7 - pkg/extract/actors/initdiff/v6/addresses.go | 7 - pkg/extract/actors/initdiff/v6/state.go | 7 - pkg/extract/actors/initdiff/v7/addresses.go | 7 - pkg/extract/actors/initdiff/v7/state.go | 7 - pkg/extract/actors/initdiff/v8/addresses.go | 7 - pkg/extract/actors/initdiff/v8/state.go | 7 - pkg/extract/actors/initdiff/v9/addresses.go | 7 - pkg/extract/actors/initdiff/v9/state.go | 7 - pkg/extract/actors/initdiff/version.go | 57 +--- .../actors/marketdiff/{v0 => v1}/cbor_gen.go | 2 +- .../actors/marketdiff/{v0 => v1}/deals.go | 2 +- .../actors/marketdiff/{v0 => v1}/load.go | 2 +- .../actors/marketdiff/{v0 => v1}/proposals.go | 2 +- .../actors/marketdiff/{v0 => v1}/state.go | 2 +- pkg/extract/actors/marketdiff/v2/deals.go | 7 - pkg/extract/actors/marketdiff/v2/proposals.go | 7 - pkg/extract/actors/marketdiff/v2/state.go | 7 - pkg/extract/actors/marketdiff/v3/deals.go | 7 - pkg/extract/actors/marketdiff/v3/proposals.go | 7 - pkg/extract/actors/marketdiff/v3/state.go | 7 - pkg/extract/actors/marketdiff/v4/deals.go | 7 - pkg/extract/actors/marketdiff/v4/proposals.go | 7 - pkg/extract/actors/marketdiff/v4/state.go | 7 - pkg/extract/actors/marketdiff/v5/deals.go | 7 - pkg/extract/actors/marketdiff/v5/proposals.go | 7 - pkg/extract/actors/marketdiff/v5/state.go | 7 - pkg/extract/actors/marketdiff/v6/deals.go | 7 - pkg/extract/actors/marketdiff/v6/proposals.go | 7 - pkg/extract/actors/marketdiff/v6/state.go | 7 - pkg/extract/actors/marketdiff/v7/deals.go | 7 - pkg/extract/actors/marketdiff/v7/proposals.go | 7 - pkg/extract/actors/marketdiff/v7/state.go | 7 - pkg/extract/actors/marketdiff/v8/deals.go | 7 - pkg/extract/actors/marketdiff/v8/proposals.go | 7 - pkg/extract/actors/marketdiff/v8/state.go | 7 - pkg/extract/actors/marketdiff/v9/deals.go | 7 - pkg/extract/actors/marketdiff/v9/proposals.go | 7 - pkg/extract/actors/marketdiff/v9/state.go | 7 - pkg/extract/actors/marketdiff/version.go | 67 +--- .../actors/minerdiff/{v0 => v1}/cbor_gen.go | 2 +- .../actors/minerdiff/{v0 => v1}/infos.go | 2 +- .../actors/minerdiff/{v0 => v1}/load.go | 2 +- .../actors/minerdiff/{v0 => v1}/precommits.go | 2 +- .../minerdiff/{v0 => v1}/sector_status.go | 2 +- .../actors/minerdiff/{v0 => v1}/sectors.go | 2 +- .../actors/minerdiff/{v0 => v1}/state.go | 2 +- pkg/extract/actors/minerdiff/v2/info.go | 6 - pkg/extract/actors/minerdiff/v2/precommits.go | 7 - .../actors/minerdiff/v2/sector_status.go | 6 - pkg/extract/actors/minerdiff/v2/sectors.go | 7 - pkg/extract/actors/minerdiff/v2/state.go | 7 - pkg/extract/actors/minerdiff/v3/info.go | 6 - pkg/extract/actors/minerdiff/v3/precommits.go | 7 - .../actors/minerdiff/v3/sector_status.go | 6 - pkg/extract/actors/minerdiff/v3/sectors.go | 7 - pkg/extract/actors/minerdiff/v3/state.go | 7 - pkg/extract/actors/minerdiff/v4/info.go | 6 - pkg/extract/actors/minerdiff/v4/precommits.go | 7 - .../actors/minerdiff/v4/sector_status.go | 6 - pkg/extract/actors/minerdiff/v4/sectors.go | 7 - pkg/extract/actors/minerdiff/v4/state.go | 7 - pkg/extract/actors/minerdiff/v5/info.go | 6 - pkg/extract/actors/minerdiff/v5/precommits.go | 7 - .../actors/minerdiff/v5/sector_status.go | 6 - pkg/extract/actors/minerdiff/v5/sectors.go | 7 - pkg/extract/actors/minerdiff/v5/state.go | 7 - pkg/extract/actors/minerdiff/v6/info.go | 6 - pkg/extract/actors/minerdiff/v6/precommits.go | 7 - .../actors/minerdiff/v6/sector_status.go | 6 - pkg/extract/actors/minerdiff/v6/sectors.go | 7 - pkg/extract/actors/minerdiff/v6/state.go | 7 - pkg/extract/actors/minerdiff/v7/info.go | 6 - pkg/extract/actors/minerdiff/v7/precommits.go | 7 - .../actors/minerdiff/v7/sector_status.go | 6 - pkg/extract/actors/minerdiff/v7/sectors.go | 7 - pkg/extract/actors/minerdiff/v7/state.go | 7 - pkg/extract/actors/minerdiff/v8/info.go | 6 - pkg/extract/actors/minerdiff/v8/precommits.go | 7 - .../actors/minerdiff/v8/sector_status.go | 6 - pkg/extract/actors/minerdiff/v8/sectors.go | 7 - pkg/extract/actors/minerdiff/v8/state.go | 7 - pkg/extract/actors/minerdiff/v9/info.go | 6 - pkg/extract/actors/minerdiff/v9/precommits.go | 7 - .../actors/minerdiff/v9/sector_status.go | 6 - pkg/extract/actors/minerdiff/v9/sectors.go | 7 - pkg/extract/actors/minerdiff/v9/state.go | 7 - pkg/extract/actors/minerdiff/version.go | 89 +----- .../actors/powerdiff/{v0 => v1}/cbor_gen.go | 2 +- .../actors/powerdiff/{v0 => v1}/claims.go | 2 +- .../actors/powerdiff/{v0 => v1}/load.go | 2 +- .../actors/powerdiff/{v0 => v1}/state.go | 2 +- pkg/extract/actors/powerdiff/v2/claims.go | 7 - pkg/extract/actors/powerdiff/v2/state.go | 7 - pkg/extract/actors/powerdiff/v3/claims.go | 7 - pkg/extract/actors/powerdiff/v3/state.go | 7 - pkg/extract/actors/powerdiff/v4/claims.go | 7 - pkg/extract/actors/powerdiff/v4/state.go | 7 - pkg/extract/actors/powerdiff/v5/claims.go | 7 - pkg/extract/actors/powerdiff/v5/state.go | 7 - pkg/extract/actors/powerdiff/v6/claims.go | 7 - pkg/extract/actors/powerdiff/v6/state.go | 7 - pkg/extract/actors/powerdiff/v7/claims.go | 7 - pkg/extract/actors/powerdiff/v7/state.go | 7 - pkg/extract/actors/powerdiff/v8/claims.go | 7 - pkg/extract/actors/powerdiff/v8/state.go | 7 - pkg/extract/actors/powerdiff/v9/claims.go | 7 - pkg/extract/actors/powerdiff/v9/state.go | 7 - pkg/extract/actors/powerdiff/version.go | 57 +--- pkg/extract/chain.go | 38 ++- pkg/gen/main.go | 38 +-- pkg/transform/cbor/cbor_gen.go | 158 +++++++++- pkg/transform/cbor/router.go | 26 +- .../timescale/actors/init/addresses.go | 2 +- .../timescale/actors/init/processor.go | 2 +- .../timescale/actors/market/v1/router.go | 2 +- .../timescale/actors/market/v1/v0/deals.go | 2 +- .../actors/market/v1/v0/proposals.go | 2 +- .../timescale/actors/market/v1/v2/deals.go | 2 +- .../actors/market/v1/v2/proposals.go | 2 +- .../timescale/actors/market/v1/v3/deals.go | 2 +- .../actors/market/v1/v3/proposals.go | 2 +- .../timescale/actors/market/v1/v4/deals.go | 2 +- .../actors/market/v1/v4/proposals.go | 2 +- .../timescale/actors/market/v1/v5/deals.go | 2 +- .../actors/market/v1/v5/proposals.go | 2 +- .../timescale/actors/market/v1/v6/deals.go | 2 +- .../actors/market/v1/v6/proposals.go | 2 +- .../timescale/actors/market/v1/v7/deals.go | 2 +- .../actors/market/v1/v7/proposals.go | 2 +- .../timescale/actors/market/v1/v8/deals.go | 2 +- .../actors/market/v1/v8/proposals.go | 2 +- .../timescale/actors/market/v1/v9/deals.go | 2 +- .../actors/market/v1/v9/proposals.go | 2 +- .../timescale/actors/miner/v1/router.go | 2 +- .../timescale/actors/miner/v1/v0/info.go | 2 +- .../actors/miner/v1/v0/precommits.go | 2 +- .../actors/miner/v1/v0/sector_deals.go | 2 +- .../actors/miner/v1/v0/sector_events.go | 2 +- .../timescale/actors/miner/v1/v0/sectors.go | 2 +- .../timescale/actors/miner/v1/v2/info.go | 2 +- .../actors/miner/v1/v2/precommits.go | 2 +- .../actors/miner/v1/v2/sector_deals.go | 2 +- .../actors/miner/v1/v2/sector_events.go | 2 +- .../timescale/actors/miner/v1/v2/sectors.go | 2 +- .../timescale/actors/miner/v1/v3/info.go | 2 +- .../actors/miner/v1/v3/precommits.go | 2 +- .../actors/miner/v1/v3/sector_deals.go | 2 +- .../actors/miner/v1/v3/sector_events.go | 2 +- .../timescale/actors/miner/v1/v3/sectors.go | 2 +- .../timescale/actors/miner/v1/v4/info.go | 2 +- .../actors/miner/v1/v4/precommits.go | 2 +- .../actors/miner/v1/v4/sector_deals.go | 2 +- .../actors/miner/v1/v4/sector_events.go | 2 +- .../timescale/actors/miner/v1/v4/sectors.go | 2 +- .../timescale/actors/miner/v1/v5/info.go | 2 +- .../actors/miner/v1/v5/precommits.go | 2 +- .../actors/miner/v1/v5/sector_deals.go | 2 +- .../actors/miner/v1/v5/sector_events.go | 2 +- .../timescale/actors/miner/v1/v5/sectors.go | 2 +- .../timescale/actors/miner/v1/v6/info.go | 2 +- .../actors/miner/v1/v6/precommits.go | 2 +- .../actors/miner/v1/v6/sector_deals.go | 2 +- .../actors/miner/v1/v6/sector_events.go | 2 +- .../timescale/actors/miner/v1/v6/sectors.go | 2 +- .../timescale/actors/miner/v1/v7/info.go | 2 +- .../actors/miner/v1/v7/precommits.go | 2 +- .../actors/miner/v1/v7/sector_deals.go | 2 +- .../actors/miner/v1/v7/sector_events.go | 2 +- .../timescale/actors/miner/v1/v7/sectors.go | 2 +- .../timescale/actors/miner/v1/v8/info.go | 2 +- .../actors/miner/v1/v8/precommits.go | 2 +- .../actors/miner/v1/v8/sector_deals.go | 2 +- .../actors/miner/v1/v8/sector_events.go | 2 +- .../timescale/actors/miner/v1/v8/sectors.go | 2 +- .../timescale/actors/miner/v1/v9/info.go | 2 +- .../actors/miner/v1/v9/precommits.go | 2 +- .../actors/miner/v1/v9/sector_deals.go | 2 +- .../actors/miner/v1/v9/sector_events.go | 2 +- .../timescale/actors/miner/v1/v9/sectors.go | 2 +- .../timescale/actors/power/v1/router.go | 2 +- .../timescale/actors/power/v1/v0/claims.go | 2 +- .../timescale/actors/power/v1/v2/claims.go | 2 +- .../timescale/actors/power/v1/v3/claims.go | 2 +- .../timescale/actors/power/v1/v4/claims.go | 2 +- .../timescale/actors/power/v1/v5/claims.go | 2 +- .../timescale/actors/power/v1/v6/claims.go | 2 +- .../timescale/actors/power/v1/v7/claims.go | 2 +- .../timescale/actors/power/v1/v8/claims.go | 2 +- .../timescale/actors/power/v1/v9/claims.go | 2 +- pkg/transform/timescale/fullblock/blocks.go | 22 ++ shed/newschema/main.go | 288 ++++++++++++++++++ 209 files changed, 666 insertions(+), 1042 deletions(-) rename pkg/extract/actors/datacapdiff/{v9 => v1}/allowances.go (99%) rename pkg/extract/actors/datacapdiff/{v9 => v1}/balances.go (99%) rename pkg/extract/actors/datacapdiff/{v9 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/datacapdiff/{v9 => v1}/load.go (98%) rename pkg/extract/actors/datacapdiff/{v9 => v1}/state.go (99%) rename pkg/extract/actors/initdiff/{v0 => v1}/addresses.go (99%) rename pkg/extract/actors/initdiff/{v0 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/initdiff/{v0 => v1}/load.go (98%) rename pkg/extract/actors/initdiff/{v0 => v1}/state.go (99%) delete mode 100644 pkg/extract/actors/initdiff/v2/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v2/state.go delete mode 100644 pkg/extract/actors/initdiff/v3/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v3/state.go delete mode 100644 pkg/extract/actors/initdiff/v4/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v4/state.go delete mode 100644 pkg/extract/actors/initdiff/v5/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v5/state.go delete mode 100644 pkg/extract/actors/initdiff/v6/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v6/state.go delete mode 100644 pkg/extract/actors/initdiff/v7/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v7/state.go delete mode 100644 pkg/extract/actors/initdiff/v8/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v8/state.go delete mode 100644 pkg/extract/actors/initdiff/v9/addresses.go delete mode 100644 pkg/extract/actors/initdiff/v9/state.go rename pkg/extract/actors/marketdiff/{v0 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/marketdiff/{v0 => v1}/deals.go (99%) rename pkg/extract/actors/marketdiff/{v0 => v1}/load.go (98%) rename pkg/extract/actors/marketdiff/{v0 => v1}/proposals.go (99%) rename pkg/extract/actors/marketdiff/{v0 => v1}/state.go (99%) delete mode 100644 pkg/extract/actors/marketdiff/v2/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v2/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v2/state.go delete mode 100644 pkg/extract/actors/marketdiff/v3/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v3/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v3/state.go delete mode 100644 pkg/extract/actors/marketdiff/v4/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v4/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v4/state.go delete mode 100644 pkg/extract/actors/marketdiff/v5/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v5/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v5/state.go delete mode 100644 pkg/extract/actors/marketdiff/v6/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v6/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v6/state.go delete mode 100644 pkg/extract/actors/marketdiff/v7/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v7/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v7/state.go delete mode 100644 pkg/extract/actors/marketdiff/v8/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v8/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v8/state.go delete mode 100644 pkg/extract/actors/marketdiff/v9/deals.go delete mode 100644 pkg/extract/actors/marketdiff/v9/proposals.go delete mode 100644 pkg/extract/actors/marketdiff/v9/state.go rename pkg/extract/actors/minerdiff/{v0 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/minerdiff/{v0 => v1}/infos.go (99%) rename pkg/extract/actors/minerdiff/{v0 => v1}/load.go (98%) rename pkg/extract/actors/minerdiff/{v0 => v1}/precommits.go (99%) rename pkg/extract/actors/minerdiff/{v0 => v1}/sector_status.go (99%) rename pkg/extract/actors/minerdiff/{v0 => v1}/sectors.go (99%) rename pkg/extract/actors/minerdiff/{v0 => v1}/state.go (99%) delete mode 100644 pkg/extract/actors/minerdiff/v2/info.go delete mode 100644 pkg/extract/actors/minerdiff/v2/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v2/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v2/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v2/state.go delete mode 100644 pkg/extract/actors/minerdiff/v3/info.go delete mode 100644 pkg/extract/actors/minerdiff/v3/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v3/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v3/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v3/state.go delete mode 100644 pkg/extract/actors/minerdiff/v4/info.go delete mode 100644 pkg/extract/actors/minerdiff/v4/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v4/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v4/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v4/state.go delete mode 100644 pkg/extract/actors/minerdiff/v5/info.go delete mode 100644 pkg/extract/actors/minerdiff/v5/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v5/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v5/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v5/state.go delete mode 100644 pkg/extract/actors/minerdiff/v6/info.go delete mode 100644 pkg/extract/actors/minerdiff/v6/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v6/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v6/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v6/state.go delete mode 100644 pkg/extract/actors/minerdiff/v7/info.go delete mode 100644 pkg/extract/actors/minerdiff/v7/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v7/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v7/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v7/state.go delete mode 100644 pkg/extract/actors/minerdiff/v8/info.go delete mode 100644 pkg/extract/actors/minerdiff/v8/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v8/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v8/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v8/state.go delete mode 100644 pkg/extract/actors/minerdiff/v9/info.go delete mode 100644 pkg/extract/actors/minerdiff/v9/precommits.go delete mode 100644 pkg/extract/actors/minerdiff/v9/sector_status.go delete mode 100644 pkg/extract/actors/minerdiff/v9/sectors.go delete mode 100644 pkg/extract/actors/minerdiff/v9/state.go rename pkg/extract/actors/powerdiff/{v0 => v1}/cbor_gen.go (99%) rename pkg/extract/actors/powerdiff/{v0 => v1}/claims.go (99%) rename pkg/extract/actors/powerdiff/{v0 => v1}/load.go (98%) rename pkg/extract/actors/powerdiff/{v0 => v1}/state.go (99%) delete mode 100644 pkg/extract/actors/powerdiff/v2/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v2/state.go delete mode 100644 pkg/extract/actors/powerdiff/v3/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v3/state.go delete mode 100644 pkg/extract/actors/powerdiff/v4/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v4/state.go delete mode 100644 pkg/extract/actors/powerdiff/v5/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v5/state.go delete mode 100644 pkg/extract/actors/powerdiff/v6/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v6/state.go delete mode 100644 pkg/extract/actors/powerdiff/v7/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v7/state.go delete mode 100644 pkg/extract/actors/powerdiff/v8/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v8/state.go delete mode 100644 pkg/extract/actors/powerdiff/v9/claims.go delete mode 100644 pkg/extract/actors/powerdiff/v9/state.go create mode 100644 shed/newschema/main.go diff --git a/pkg/extract/actors/datacapdiff/v9/allowances.go b/pkg/extract/actors/datacapdiff/v1/allowances.go similarity index 99% rename from pkg/extract/actors/datacapdiff/v9/allowances.go rename to pkg/extract/actors/datacapdiff/v1/allowances.go index 8509e5cb5..406d6b07e 100644 --- a/pkg/extract/actors/datacapdiff/v9/allowances.go +++ b/pkg/extract/actors/datacapdiff/v1/allowances.go @@ -1,4 +1,4 @@ -package v9 +package v1 import ( "context" diff --git a/pkg/extract/actors/datacapdiff/v9/balances.go b/pkg/extract/actors/datacapdiff/v1/balances.go similarity index 99% rename from pkg/extract/actors/datacapdiff/v9/balances.go rename to pkg/extract/actors/datacapdiff/v1/balances.go index 3ba32bbf0..4e11e167a 100644 --- a/pkg/extract/actors/datacapdiff/v9/balances.go +++ b/pkg/extract/actors/datacapdiff/v1/balances.go @@ -1,4 +1,4 @@ -package v9 +package v1 import ( "context" diff --git a/pkg/extract/actors/datacapdiff/v9/cbor_gen.go b/pkg/extract/actors/datacapdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/datacapdiff/v9/cbor_gen.go rename to pkg/extract/actors/datacapdiff/v1/cbor_gen.go index eed251d33..af7fa4862 100644 --- a/pkg/extract/actors/datacapdiff/v9/cbor_gen.go +++ b/pkg/extract/actors/datacapdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v9 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/datacapdiff/v9/load.go b/pkg/extract/actors/datacapdiff/v1/load.go similarity index 98% rename from pkg/extract/actors/datacapdiff/v9/load.go rename to pkg/extract/actors/datacapdiff/v1/load.go index 67cc77862..681c646a1 100644 --- a/pkg/extract/actors/datacapdiff/v9/load.go +++ b/pkg/extract/actors/datacapdiff/v1/load.go @@ -1,4 +1,4 @@ -package v9 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/datacapdiff/v9/state.go b/pkg/extract/actors/datacapdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/datacapdiff/v9/state.go rename to pkg/extract/actors/datacapdiff/v1/state.go index eff42b34e..38ce36e8d 100644 --- a/pkg/extract/actors/datacapdiff/v9/state.go +++ b/pkg/extract/actors/datacapdiff/v1/state.go @@ -1,4 +1,4 @@ -package v9 +package v1 import ( "context" diff --git a/pkg/extract/actors/datacapdiff/version.go b/pkg/extract/actors/datacapdiff/version.go index 056816862..e6b8e5042 100644 --- a/pkg/extract/actors/datacapdiff/version.go +++ b/pkg/extract/actors/datacapdiff/version.go @@ -6,16 +6,16 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v9" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { switch av { case actortypes.Version9: - return &v9.StateDiff{ + return &v1.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v9.Allowance{}, - v9.Balance{}, + v1.Allowance{}, + v1.Balance{}, }, }, nil case actortypes.Version10: diff --git a/pkg/extract/actors/initdiff/v0/addresses.go b/pkg/extract/actors/initdiff/v1/addresses.go similarity index 99% rename from pkg/extract/actors/initdiff/v0/addresses.go rename to pkg/extract/actors/initdiff/v1/addresses.go index 7885a2a09..95f13a1bc 100644 --- a/pkg/extract/actors/initdiff/v0/addresses.go +++ b/pkg/extract/actors/initdiff/v1/addresses.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/initdiff/v0/cbor_gen.go b/pkg/extract/actors/initdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/initdiff/v0/cbor_gen.go rename to pkg/extract/actors/initdiff/v1/cbor_gen.go index a6c29c139..e5c286636 100644 --- a/pkg/extract/actors/initdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/initdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v0 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/initdiff/v0/load.go b/pkg/extract/actors/initdiff/v1/load.go similarity index 98% rename from pkg/extract/actors/initdiff/v0/load.go rename to pkg/extract/actors/initdiff/v1/load.go index ce98e79d4..22d3edd85 100644 --- a/pkg/extract/actors/initdiff/v0/load.go +++ b/pkg/extract/actors/initdiff/v1/load.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/initdiff/v0/state.go b/pkg/extract/actors/initdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/initdiff/v0/state.go rename to pkg/extract/actors/initdiff/v1/state.go index ac7b3bd76..f8a05f8b3 100644 --- a/pkg/extract/actors/initdiff/v0/state.go +++ b/pkg/extract/actors/initdiff/v1/state.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/initdiff/v2/addresses.go b/pkg/extract/actors/initdiff/v2/addresses.go deleted file mode 100644 index 2bf020b19..000000000 --- a/pkg/extract/actors/initdiff/v2/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v2/state.go b/pkg/extract/actors/initdiff/v2/state.go deleted file mode 100644 index b0076ae9d..000000000 --- a/pkg/extract/actors/initdiff/v2/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v3/addresses.go b/pkg/extract/actors/initdiff/v3/addresses.go deleted file mode 100644 index a1369dc92..000000000 --- a/pkg/extract/actors/initdiff/v3/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v3/state.go b/pkg/extract/actors/initdiff/v3/state.go deleted file mode 100644 index 1b76abdf1..000000000 --- a/pkg/extract/actors/initdiff/v3/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v4/addresses.go b/pkg/extract/actors/initdiff/v4/addresses.go deleted file mode 100644 index 04a790c2c..000000000 --- a/pkg/extract/actors/initdiff/v4/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v4/state.go b/pkg/extract/actors/initdiff/v4/state.go deleted file mode 100644 index a5afc65b0..000000000 --- a/pkg/extract/actors/initdiff/v4/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v5/addresses.go b/pkg/extract/actors/initdiff/v5/addresses.go deleted file mode 100644 index 504767884..000000000 --- a/pkg/extract/actors/initdiff/v5/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v5/state.go b/pkg/extract/actors/initdiff/v5/state.go deleted file mode 100644 index 507a44a1b..000000000 --- a/pkg/extract/actors/initdiff/v5/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v6/addresses.go b/pkg/extract/actors/initdiff/v6/addresses.go deleted file mode 100644 index 01a52c23a..000000000 --- a/pkg/extract/actors/initdiff/v6/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v6/state.go b/pkg/extract/actors/initdiff/v6/state.go deleted file mode 100644 index ac102b082..000000000 --- a/pkg/extract/actors/initdiff/v6/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v7/addresses.go b/pkg/extract/actors/initdiff/v7/addresses.go deleted file mode 100644 index fdff66564..000000000 --- a/pkg/extract/actors/initdiff/v7/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v7/state.go b/pkg/extract/actors/initdiff/v7/state.go deleted file mode 100644 index 4d520263a..000000000 --- a/pkg/extract/actors/initdiff/v7/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v8/addresses.go b/pkg/extract/actors/initdiff/v8/addresses.go deleted file mode 100644 index 8bcb05bfc..000000000 --- a/pkg/extract/actors/initdiff/v8/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v8/state.go b/pkg/extract/actors/initdiff/v8/state.go deleted file mode 100644 index d720d448f..000000000 --- a/pkg/extract/actors/initdiff/v8/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/v9/addresses.go b/pkg/extract/actors/initdiff/v9/addresses.go deleted file mode 100644 index fdeb6c47c..000000000 --- a/pkg/extract/actors/initdiff/v9/addresses.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type AddressesChange = v0.AddressChange -type AddressesChangeList = v0.AddressChangeList -type Addresses = v0.Addresses diff --git a/pkg/extract/actors/initdiff/v9/state.go b/pkg/extract/actors/initdiff/v9/state.go deleted file mode 100644 index 55056ae74..000000000 --- a/pkg/extract/actors/initdiff/v9/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - -type StateDiff = v0.StateDiff -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange diff --git a/pkg/extract/actors/initdiff/version.go b/pkg/extract/actors/initdiff/version.go index 63f61d12d..be83b722b 100644 --- a/pkg/extract/actors/initdiff/version.go +++ b/pkg/extract/actors/initdiff/version.go @@ -6,63 +6,16 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - v2 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v2" - v3 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v3" - v4 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v4" - v5 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v5" - v6 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v6" - v7 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v7" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v8" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v9" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { switch av { - case actortypes.Version0: - return &v0.StateDiff{ + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return &v1.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v0.Addresses{}, - }}, nil - case actortypes.Version2: - return &v2.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v2.Addresses{}, - }}, nil - case actortypes.Version3: - return &v3.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v3.Addresses{}, - }}, nil - case actortypes.Version4: - return &v4.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v4.Addresses{}, - }}, nil - case actortypes.Version5: - return &v5.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v5.Addresses{}, - }}, nil - case actortypes.Version6: - return &v6.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v6.Addresses{}, - }}, nil - case actortypes.Version7: - return &v7.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v7.Addresses{}, - }}, nil - case actortypes.Version8: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v8.Addresses{}, - }}, nil - case actortypes.Version9: - return &v9.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v9.Addresses{}, + v1.Addresses{}, }}, nil case actortypes.Version10: panic("NYI") diff --git a/pkg/extract/actors/marketdiff/v0/cbor_gen.go b/pkg/extract/actors/marketdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/marketdiff/v0/cbor_gen.go rename to pkg/extract/actors/marketdiff/v1/cbor_gen.go index e11c76bfa..e578e4b0d 100644 --- a/pkg/extract/actors/marketdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/marketdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v0 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/marketdiff/v0/deals.go b/pkg/extract/actors/marketdiff/v1/deals.go similarity index 99% rename from pkg/extract/actors/marketdiff/v0/deals.go rename to pkg/extract/actors/marketdiff/v1/deals.go index f69b9c1d0..19e1acad8 100644 --- a/pkg/extract/actors/marketdiff/v0/deals.go +++ b/pkg/extract/actors/marketdiff/v1/deals.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/marketdiff/v0/load.go b/pkg/extract/actors/marketdiff/v1/load.go similarity index 98% rename from pkg/extract/actors/marketdiff/v0/load.go rename to pkg/extract/actors/marketdiff/v1/load.go index 07dbefdc9..2395f3305 100644 --- a/pkg/extract/actors/marketdiff/v0/load.go +++ b/pkg/extract/actors/marketdiff/v1/load.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/marketdiff/v0/proposals.go b/pkg/extract/actors/marketdiff/v1/proposals.go similarity index 99% rename from pkg/extract/actors/marketdiff/v0/proposals.go rename to pkg/extract/actors/marketdiff/v1/proposals.go index 191b978f1..b9831dcb6 100644 --- a/pkg/extract/actors/marketdiff/v0/proposals.go +++ b/pkg/extract/actors/marketdiff/v1/proposals.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/marketdiff/v0/state.go b/pkg/extract/actors/marketdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/marketdiff/v0/state.go rename to pkg/extract/actors/marketdiff/v1/state.go index d9b3dbb4d..13bf0026c 100644 --- a/pkg/extract/actors/marketdiff/v0/state.go +++ b/pkg/extract/actors/marketdiff/v1/state.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/marketdiff/v2/deals.go b/pkg/extract/actors/marketdiff/v2/deals.go deleted file mode 100644 index d22d33dfb..000000000 --- a/pkg/extract/actors/marketdiff/v2/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v2/proposals.go b/pkg/extract/actors/marketdiff/v2/proposals.go deleted file mode 100644 index a973d6369..000000000 --- a/pkg/extract/actors/marketdiff/v2/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v2/state.go b/pkg/extract/actors/marketdiff/v2/state.go deleted file mode 100644 index bd66d0927..000000000 --- a/pkg/extract/actors/marketdiff/v2/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v3/deals.go b/pkg/extract/actors/marketdiff/v3/deals.go deleted file mode 100644 index 713acc401..000000000 --- a/pkg/extract/actors/marketdiff/v3/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v3/proposals.go b/pkg/extract/actors/marketdiff/v3/proposals.go deleted file mode 100644 index 1c07caf57..000000000 --- a/pkg/extract/actors/marketdiff/v3/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v3/state.go b/pkg/extract/actors/marketdiff/v3/state.go deleted file mode 100644 index c03e05cba..000000000 --- a/pkg/extract/actors/marketdiff/v3/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v4/deals.go b/pkg/extract/actors/marketdiff/v4/deals.go deleted file mode 100644 index eacc1bb7c..000000000 --- a/pkg/extract/actors/marketdiff/v4/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v4/proposals.go b/pkg/extract/actors/marketdiff/v4/proposals.go deleted file mode 100644 index b0d1bb3aa..000000000 --- a/pkg/extract/actors/marketdiff/v4/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v4/state.go b/pkg/extract/actors/marketdiff/v4/state.go deleted file mode 100644 index f9171ada2..000000000 --- a/pkg/extract/actors/marketdiff/v4/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v5/deals.go b/pkg/extract/actors/marketdiff/v5/deals.go deleted file mode 100644 index b8d8e5010..000000000 --- a/pkg/extract/actors/marketdiff/v5/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v5/proposals.go b/pkg/extract/actors/marketdiff/v5/proposals.go deleted file mode 100644 index 035a9c6c3..000000000 --- a/pkg/extract/actors/marketdiff/v5/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v5/state.go b/pkg/extract/actors/marketdiff/v5/state.go deleted file mode 100644 index 25c60e343..000000000 --- a/pkg/extract/actors/marketdiff/v5/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v6/deals.go b/pkg/extract/actors/marketdiff/v6/deals.go deleted file mode 100644 index 52d05f04e..000000000 --- a/pkg/extract/actors/marketdiff/v6/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v6/proposals.go b/pkg/extract/actors/marketdiff/v6/proposals.go deleted file mode 100644 index ba831d475..000000000 --- a/pkg/extract/actors/marketdiff/v6/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v6/state.go b/pkg/extract/actors/marketdiff/v6/state.go deleted file mode 100644 index ba9540cb3..000000000 --- a/pkg/extract/actors/marketdiff/v6/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v7/deals.go b/pkg/extract/actors/marketdiff/v7/deals.go deleted file mode 100644 index c3af174c8..000000000 --- a/pkg/extract/actors/marketdiff/v7/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v7/proposals.go b/pkg/extract/actors/marketdiff/v7/proposals.go deleted file mode 100644 index 312694afd..000000000 --- a/pkg/extract/actors/marketdiff/v7/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v7/state.go b/pkg/extract/actors/marketdiff/v7/state.go deleted file mode 100644 index 710e10d11..000000000 --- a/pkg/extract/actors/marketdiff/v7/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v8/deals.go b/pkg/extract/actors/marketdiff/v8/deals.go deleted file mode 100644 index 64257a047..000000000 --- a/pkg/extract/actors/marketdiff/v8/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v8/proposals.go b/pkg/extract/actors/marketdiff/v8/proposals.go deleted file mode 100644 index f884016a9..000000000 --- a/pkg/extract/actors/marketdiff/v8/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v8/state.go b/pkg/extract/actors/marketdiff/v8/state.go deleted file mode 100644 index 174d1bac5..000000000 --- a/pkg/extract/actors/marketdiff/v8/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/v9/deals.go b/pkg/extract/actors/marketdiff/v9/deals.go deleted file mode 100644 index 5f0685444..000000000 --- a/pkg/extract/actors/marketdiff/v9/deals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type DealChange = v0.DealChange -type DealChangeList = v0.DealChangeList -type Deals = v0.Deals diff --git a/pkg/extract/actors/marketdiff/v9/proposals.go b/pkg/extract/actors/marketdiff/v9/proposals.go deleted file mode 100644 index b7e27f8f0..000000000 --- a/pkg/extract/actors/marketdiff/v9/proposals.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type ProposalChange = v0.ProposalChange -type ProposalChangeList = v0.ProposalChangeList -type Proposals = v0.Proposals diff --git a/pkg/extract/actors/marketdiff/v9/state.go b/pkg/extract/actors/marketdiff/v9/state.go deleted file mode 100644 index fa3914096..000000000 --- a/pkg/extract/actors/marketdiff/v9/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/marketdiff/version.go b/pkg/extract/actors/marketdiff/version.go index 98e5123ab..448d077f8 100644 --- a/pkg/extract/actors/marketdiff/version.go +++ b/pkg/extract/actors/marketdiff/version.go @@ -6,72 +6,17 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - v2 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" - v3 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" - v4 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" - v5 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" - v6 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" - v7 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { switch av { - case actortypes.Version0: - return &v0.StateDiff{ + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return &v1.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v0.Deals{}, - v0.Proposals{}, - }}, nil - case actortypes.Version2: - return &v2.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v2.Deals{}, - v2.Proposals{}, - }}, nil - case actortypes.Version3: - return &v3.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v3.Deals{}, - v3.Proposals{}, - }}, nil - case actortypes.Version4: - return &v4.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v4.Deals{}, - v4.Proposals{}, - }}, nil - case actortypes.Version5: - return &v5.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v5.Deals{}, - v5.Proposals{}, - }}, nil - case actortypes.Version6: - return &v6.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v6.Deals{}, - v6.Proposals{}, - }}, nil - case actortypes.Version7: - return &v7.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v7.Deals{}, - v7.Proposals{}, - }}, nil - case actortypes.Version8: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v8.Deals{}, - v8.Proposals{}, - }}, nil - case actortypes.Version9: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v9.Deals{}, - v9.Proposals{}, + v1.Deals{}, + v1.Proposals{}, }}, nil case actortypes.Version10: panic("NYI") diff --git a/pkg/extract/actors/minerdiff/v0/cbor_gen.go b/pkg/extract/actors/minerdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/cbor_gen.go rename to pkg/extract/actors/minerdiff/v1/cbor_gen.go index 208a34a87..522da21fa 100644 --- a/pkg/extract/actors/minerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/minerdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v0 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/minerdiff/v0/infos.go b/pkg/extract/actors/minerdiff/v1/infos.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/infos.go rename to pkg/extract/actors/minerdiff/v1/infos.go index 32406e226..bc952764b 100644 --- a/pkg/extract/actors/minerdiff/v0/infos.go +++ b/pkg/extract/actors/minerdiff/v1/infos.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/minerdiff/v0/load.go b/pkg/extract/actors/minerdiff/v1/load.go similarity index 98% rename from pkg/extract/actors/minerdiff/v0/load.go rename to pkg/extract/actors/minerdiff/v1/load.go index 7eefb1990..46b41c7d4 100644 --- a/pkg/extract/actors/minerdiff/v0/load.go +++ b/pkg/extract/actors/minerdiff/v1/load.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/minerdiff/v0/precommits.go b/pkg/extract/actors/minerdiff/v1/precommits.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/precommits.go rename to pkg/extract/actors/minerdiff/v1/precommits.go index d9277aa22..becfe1e33 100644 --- a/pkg/extract/actors/minerdiff/v0/precommits.go +++ b/pkg/extract/actors/minerdiff/v1/precommits.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/minerdiff/v0/sector_status.go b/pkg/extract/actors/minerdiff/v1/sector_status.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/sector_status.go rename to pkg/extract/actors/minerdiff/v1/sector_status.go index d62e2035f..c0e12bea3 100644 --- a/pkg/extract/actors/minerdiff/v0/sector_status.go +++ b/pkg/extract/actors/minerdiff/v1/sector_status.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/minerdiff/v0/sectors.go b/pkg/extract/actors/minerdiff/v1/sectors.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/sectors.go rename to pkg/extract/actors/minerdiff/v1/sectors.go index 679441852..b26333437 100644 --- a/pkg/extract/actors/minerdiff/v0/sectors.go +++ b/pkg/extract/actors/minerdiff/v1/sectors.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/minerdiff/v0/state.go b/pkg/extract/actors/minerdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/minerdiff/v0/state.go rename to pkg/extract/actors/minerdiff/v1/state.go index 2d3294f29..5cfe8f49c 100644 --- a/pkg/extract/actors/minerdiff/v0/state.go +++ b/pkg/extract/actors/minerdiff/v1/state.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/minerdiff/v2/info.go b/pkg/extract/actors/minerdiff/v2/info.go deleted file mode 100644 index 9d76c0ed4..000000000 --- a/pkg/extract/actors/minerdiff/v2/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v2/precommits.go b/pkg/extract/actors/minerdiff/v2/precommits.go deleted file mode 100644 index 518943873..000000000 --- a/pkg/extract/actors/minerdiff/v2/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v2/sector_status.go b/pkg/extract/actors/minerdiff/v2/sector_status.go deleted file mode 100644 index 1f247e995..000000000 --- a/pkg/extract/actors/minerdiff/v2/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v2/sectors.go b/pkg/extract/actors/minerdiff/v2/sectors.go deleted file mode 100644 index 9e76fd2c9..000000000 --- a/pkg/extract/actors/minerdiff/v2/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v2/state.go b/pkg/extract/actors/minerdiff/v2/state.go deleted file mode 100644 index 69c74dac0..000000000 --- a/pkg/extract/actors/minerdiff/v2/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v3/info.go b/pkg/extract/actors/minerdiff/v3/info.go deleted file mode 100644 index 4c0ae4f5f..000000000 --- a/pkg/extract/actors/minerdiff/v3/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v3/precommits.go b/pkg/extract/actors/minerdiff/v3/precommits.go deleted file mode 100644 index eb6127fd6..000000000 --- a/pkg/extract/actors/minerdiff/v3/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v3/sector_status.go b/pkg/extract/actors/minerdiff/v3/sector_status.go deleted file mode 100644 index df36d2bca..000000000 --- a/pkg/extract/actors/minerdiff/v3/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v3/sectors.go b/pkg/extract/actors/minerdiff/v3/sectors.go deleted file mode 100644 index cae0b0584..000000000 --- a/pkg/extract/actors/minerdiff/v3/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v3/state.go b/pkg/extract/actors/minerdiff/v3/state.go deleted file mode 100644 index d3dd5a70b..000000000 --- a/pkg/extract/actors/minerdiff/v3/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v4/info.go b/pkg/extract/actors/minerdiff/v4/info.go deleted file mode 100644 index 67260a2d8..000000000 --- a/pkg/extract/actors/minerdiff/v4/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v4/precommits.go b/pkg/extract/actors/minerdiff/v4/precommits.go deleted file mode 100644 index f07b79a0f..000000000 --- a/pkg/extract/actors/minerdiff/v4/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v4/sector_status.go b/pkg/extract/actors/minerdiff/v4/sector_status.go deleted file mode 100644 index c62dd8f86..000000000 --- a/pkg/extract/actors/minerdiff/v4/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v4/sectors.go b/pkg/extract/actors/minerdiff/v4/sectors.go deleted file mode 100644 index 6756b06cd..000000000 --- a/pkg/extract/actors/minerdiff/v4/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v4/state.go b/pkg/extract/actors/minerdiff/v4/state.go deleted file mode 100644 index ea583d7fe..000000000 --- a/pkg/extract/actors/minerdiff/v4/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v5/info.go b/pkg/extract/actors/minerdiff/v5/info.go deleted file mode 100644 index 45135d5c7..000000000 --- a/pkg/extract/actors/minerdiff/v5/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v5/precommits.go b/pkg/extract/actors/minerdiff/v5/precommits.go deleted file mode 100644 index e7ff3e384..000000000 --- a/pkg/extract/actors/minerdiff/v5/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v5/sector_status.go b/pkg/extract/actors/minerdiff/v5/sector_status.go deleted file mode 100644 index b01fd8186..000000000 --- a/pkg/extract/actors/minerdiff/v5/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v5/sectors.go b/pkg/extract/actors/minerdiff/v5/sectors.go deleted file mode 100644 index f1ce97e65..000000000 --- a/pkg/extract/actors/minerdiff/v5/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v5/state.go b/pkg/extract/actors/minerdiff/v5/state.go deleted file mode 100644 index 95b671e65..000000000 --- a/pkg/extract/actors/minerdiff/v5/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v6/info.go b/pkg/extract/actors/minerdiff/v6/info.go deleted file mode 100644 index c38170de7..000000000 --- a/pkg/extract/actors/minerdiff/v6/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v6/precommits.go b/pkg/extract/actors/minerdiff/v6/precommits.go deleted file mode 100644 index fdee78a84..000000000 --- a/pkg/extract/actors/minerdiff/v6/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v6/sector_status.go b/pkg/extract/actors/minerdiff/v6/sector_status.go deleted file mode 100644 index ddf9a6762..000000000 --- a/pkg/extract/actors/minerdiff/v6/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v6/sectors.go b/pkg/extract/actors/minerdiff/v6/sectors.go deleted file mode 100644 index 409ec8488..000000000 --- a/pkg/extract/actors/minerdiff/v6/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v6/state.go b/pkg/extract/actors/minerdiff/v6/state.go deleted file mode 100644 index c3545513b..000000000 --- a/pkg/extract/actors/minerdiff/v6/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v7/info.go b/pkg/extract/actors/minerdiff/v7/info.go deleted file mode 100644 index 63c42fc76..000000000 --- a/pkg/extract/actors/minerdiff/v7/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v7/precommits.go b/pkg/extract/actors/minerdiff/v7/precommits.go deleted file mode 100644 index 1b8ef337e..000000000 --- a/pkg/extract/actors/minerdiff/v7/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v7/sector_status.go b/pkg/extract/actors/minerdiff/v7/sector_status.go deleted file mode 100644 index fc3fb8ea2..000000000 --- a/pkg/extract/actors/minerdiff/v7/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v7/sectors.go b/pkg/extract/actors/minerdiff/v7/sectors.go deleted file mode 100644 index 2a2697b3f..000000000 --- a/pkg/extract/actors/minerdiff/v7/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v7/state.go b/pkg/extract/actors/minerdiff/v7/state.go deleted file mode 100644 index dac14967c..000000000 --- a/pkg/extract/actors/minerdiff/v7/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v8/info.go b/pkg/extract/actors/minerdiff/v8/info.go deleted file mode 100644 index 38690dca1..000000000 --- a/pkg/extract/actors/minerdiff/v8/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v8/precommits.go b/pkg/extract/actors/minerdiff/v8/precommits.go deleted file mode 100644 index be9925bab..000000000 --- a/pkg/extract/actors/minerdiff/v8/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v8/sector_status.go b/pkg/extract/actors/minerdiff/v8/sector_status.go deleted file mode 100644 index 53f1f5cdb..000000000 --- a/pkg/extract/actors/minerdiff/v8/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v8/sectors.go b/pkg/extract/actors/minerdiff/v8/sectors.go deleted file mode 100644 index bd706479f..000000000 --- a/pkg/extract/actors/minerdiff/v8/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v8/state.go b/pkg/extract/actors/minerdiff/v8/state.go deleted file mode 100644 index d2a72308d..000000000 --- a/pkg/extract/actors/minerdiff/v8/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/v9/info.go b/pkg/extract/actors/minerdiff/v9/info.go deleted file mode 100644 index e1c74d061..000000000 --- a/pkg/extract/actors/minerdiff/v9/info.go +++ /dev/null @@ -1,6 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type InfoChange = v0.InfoChange -type Info = v0.Info diff --git a/pkg/extract/actors/minerdiff/v9/precommits.go b/pkg/extract/actors/minerdiff/v9/precommits.go deleted file mode 100644 index d65a4cf36..000000000 --- a/pkg/extract/actors/minerdiff/v9/precommits.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type PreCommitChange = v0.PreCommitChange -type PreCommitChangeList = v0.PreCommitChangeList -type PreCommit = v0.PreCommit diff --git a/pkg/extract/actors/minerdiff/v9/sector_status.go b/pkg/extract/actors/minerdiff/v9/sector_status.go deleted file mode 100644 index 4286eb9fc..000000000 --- a/pkg/extract/actors/minerdiff/v9/sector_status.go +++ /dev/null @@ -1,6 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorStatusChange = v0.SectorStatusChange -type SectorStatus = v0.SectorStatus diff --git a/pkg/extract/actors/minerdiff/v9/sectors.go b/pkg/extract/actors/minerdiff/v9/sectors.go deleted file mode 100644 index 2961334e3..000000000 --- a/pkg/extract/actors/minerdiff/v9/sectors.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type SectorChange = v0.SectorChange -type SectorChangeList = v0.SectorChangeList -type Sectors = v0.Sectors diff --git a/pkg/extract/actors/minerdiff/v9/state.go b/pkg/extract/actors/minerdiff/v9/state.go deleted file mode 100644 index 287cffc2c..000000000 --- a/pkg/extract/actors/minerdiff/v9/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/minerdiff/version.go b/pkg/extract/actors/minerdiff/version.go index 4de371d8b..71b0ba3e8 100644 --- a/pkg/extract/actors/minerdiff/version.go +++ b/pkg/extract/actors/minerdiff/version.go @@ -6,90 +6,19 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - v2 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" - v3 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" - v4 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" - v5 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" - v6 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" - v7 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { switch av { - case actortypes.Version0: - return &v0.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v0.Info{}, - v0.PreCommit{}, - v0.SectorStatus{}, - v0.Sectors{}, - }}, nil - case actortypes.Version2: - return &v2.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v2.Info{}, - v2.PreCommit{}, - v2.SectorStatus{}, - v2.Sectors{}, - }}, nil - case actortypes.Version3: - return &v3.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v3.Info{}, - v3.PreCommit{}, - v3.SectorStatus{}, - v3.Sectors{}, - }}, nil - case actortypes.Version4: - return &v4.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v4.Info{}, - v4.PreCommit{}, - v4.SectorStatus{}, - v4.Sectors{}, - }}, nil - case actortypes.Version5: - return &v5.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v5.Info{}, - v5.PreCommit{}, - v5.SectorStatus{}, - v5.Sectors{}, - }}, nil - case actortypes.Version6: - return &v6.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v6.Info{}, - v6.PreCommit{}, - v6.SectorStatus{}, - v6.Sectors{}, - }}, nil - case actortypes.Version7: - return &v7.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v7.Info{}, - v7.PreCommit{}, - v7.SectorStatus{}, - v7.Sectors{}, - }}, nil - case actortypes.Version8: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v8.Info{}, - v8.PreCommit{}, - v8.SectorStatus{}, - v8.Sectors{}, - }}, nil - case actortypes.Version9: - return &v9.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v9.Info{}, - v9.PreCommit{}, - v9.SectorStatus{}, - v9.Sectors{}, + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return &v1.StateDiff{ + DiffMethods: []actors.ActorStateDiff{ + v1.Info{}, + v1.PreCommit{}, + v1.SectorStatus{}, + v1.Sectors{}, }}, nil case actortypes.Version10: panic("Not yet implemented") diff --git a/pkg/extract/actors/powerdiff/v0/cbor_gen.go b/pkg/extract/actors/powerdiff/v1/cbor_gen.go similarity index 99% rename from pkg/extract/actors/powerdiff/v0/cbor_gen.go rename to pkg/extract/actors/powerdiff/v1/cbor_gen.go index f27bf776c..c2e3a33ef 100644 --- a/pkg/extract/actors/powerdiff/v0/cbor_gen.go +++ b/pkg/extract/actors/powerdiff/v1/cbor_gen.go @@ -1,6 +1,6 @@ // Code generated by github.com/whyrusleeping/cbor-gen. DO NOT EDIT. -package v0 +package v1 import ( "fmt" diff --git a/pkg/extract/actors/powerdiff/v0/claims.go b/pkg/extract/actors/powerdiff/v1/claims.go similarity index 99% rename from pkg/extract/actors/powerdiff/v0/claims.go rename to pkg/extract/actors/powerdiff/v1/claims.go index 9c858452e..ce6d91055 100644 --- a/pkg/extract/actors/powerdiff/v0/claims.go +++ b/pkg/extract/actors/powerdiff/v1/claims.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/powerdiff/v0/load.go b/pkg/extract/actors/powerdiff/v1/load.go similarity index 98% rename from pkg/extract/actors/powerdiff/v0/load.go rename to pkg/extract/actors/powerdiff/v1/load.go index 1ac9026af..f49b979ef 100644 --- a/pkg/extract/actors/powerdiff/v0/load.go +++ b/pkg/extract/actors/powerdiff/v1/load.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "github.com/filecoin-project/lotus/chain/types" diff --git a/pkg/extract/actors/powerdiff/v0/state.go b/pkg/extract/actors/powerdiff/v1/state.go similarity index 99% rename from pkg/extract/actors/powerdiff/v0/state.go rename to pkg/extract/actors/powerdiff/v1/state.go index 6e517611b..15a2dcb96 100644 --- a/pkg/extract/actors/powerdiff/v0/state.go +++ b/pkg/extract/actors/powerdiff/v1/state.go @@ -1,4 +1,4 @@ -package v0 +package v1 import ( "context" diff --git a/pkg/extract/actors/powerdiff/v2/claims.go b/pkg/extract/actors/powerdiff/v2/claims.go deleted file mode 100644 index bf063933b..000000000 --- a/pkg/extract/actors/powerdiff/v2/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v2/state.go b/pkg/extract/actors/powerdiff/v2/state.go deleted file mode 100644 index 08bb44e09..000000000 --- a/pkg/extract/actors/powerdiff/v2/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v2 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v3/claims.go b/pkg/extract/actors/powerdiff/v3/claims.go deleted file mode 100644 index 29bff2844..000000000 --- a/pkg/extract/actors/powerdiff/v3/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v3/state.go b/pkg/extract/actors/powerdiff/v3/state.go deleted file mode 100644 index f79f6693e..000000000 --- a/pkg/extract/actors/powerdiff/v3/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v3 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v4/claims.go b/pkg/extract/actors/powerdiff/v4/claims.go deleted file mode 100644 index 8cf17b525..000000000 --- a/pkg/extract/actors/powerdiff/v4/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v4/state.go b/pkg/extract/actors/powerdiff/v4/state.go deleted file mode 100644 index 42701a660..000000000 --- a/pkg/extract/actors/powerdiff/v4/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v4 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v5/claims.go b/pkg/extract/actors/powerdiff/v5/claims.go deleted file mode 100644 index 4fc36881f..000000000 --- a/pkg/extract/actors/powerdiff/v5/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v5/state.go b/pkg/extract/actors/powerdiff/v5/state.go deleted file mode 100644 index 702bc2916..000000000 --- a/pkg/extract/actors/powerdiff/v5/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v5 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v6/claims.go b/pkg/extract/actors/powerdiff/v6/claims.go deleted file mode 100644 index 776615258..000000000 --- a/pkg/extract/actors/powerdiff/v6/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v6/state.go b/pkg/extract/actors/powerdiff/v6/state.go deleted file mode 100644 index 43c9922df..000000000 --- a/pkg/extract/actors/powerdiff/v6/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v6 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v7/claims.go b/pkg/extract/actors/powerdiff/v7/claims.go deleted file mode 100644 index e9b8d03c7..000000000 --- a/pkg/extract/actors/powerdiff/v7/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v7/state.go b/pkg/extract/actors/powerdiff/v7/state.go deleted file mode 100644 index 0b4811439..000000000 --- a/pkg/extract/actors/powerdiff/v7/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v7 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v8/claims.go b/pkg/extract/actors/powerdiff/v8/claims.go deleted file mode 100644 index b8e9708f0..000000000 --- a/pkg/extract/actors/powerdiff/v8/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v8/state.go b/pkg/extract/actors/powerdiff/v8/state.go deleted file mode 100644 index 21c051563..000000000 --- a/pkg/extract/actors/powerdiff/v8/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v8 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/v9/claims.go b/pkg/extract/actors/powerdiff/v9/claims.go deleted file mode 100644 index 35da6a1d1..000000000 --- a/pkg/extract/actors/powerdiff/v9/claims.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type ClaimsChange = v0.ClaimsChange -type ClaimsChangeList = v0.ClaimsChangeList -type Claims = v0.Claims diff --git a/pkg/extract/actors/powerdiff/v9/state.go b/pkg/extract/actors/powerdiff/v9/state.go deleted file mode 100644 index 1267b3707..000000000 --- a/pkg/extract/actors/powerdiff/v9/state.go +++ /dev/null @@ -1,7 +0,0 @@ -package v9 - -import v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - -type StateDiffResult = v0.StateDiffResult -type StateChange = v0.StateChange -type StateDiff = v0.StateDiff diff --git a/pkg/extract/actors/powerdiff/version.go b/pkg/extract/actors/powerdiff/version.go index 1d74a7696..227c0e582 100644 --- a/pkg/extract/actors/powerdiff/version.go +++ b/pkg/extract/actors/powerdiff/version.go @@ -6,63 +6,16 @@ import ( actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" - v2 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v2" - v3 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v3" - v4 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v4" - v5 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v5" - v6 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v6" - v7 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v7" - v8 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v8" - v9 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v9" + v1 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { switch av { - case actortypes.Version0: - return &v0.StateDiff{ + case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, + actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: + return &v1.StateDiff{ DiffMethods: []actors.ActorStateDiff{ - v0.Claims{}, - }}, nil - case actortypes.Version2: - return &v2.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v2.Claims{}, - }}, nil - case actortypes.Version3: - return &v3.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v3.Claims{}, - }}, nil - case actortypes.Version4: - return &v4.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v4.Claims{}, - }}, nil - case actortypes.Version5: - return &v5.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v5.Claims{}, - }}, nil - case actortypes.Version6: - return &v6.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v6.Claims{}, - }}, nil - case actortypes.Version7: - return &v7.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v7.Claims{}, - }}, nil - case actortypes.Version8: - return &v8.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v8.Claims{}, - }}, nil - case actortypes.Version9: - return &v9.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v9.Claims{}, + v1.Claims{}, }}, nil case actortypes.Version10: panic("NYI") diff --git a/pkg/extract/chain.go b/pkg/extract/chain.go index 0b37b4570..f5a5513ef 100644 --- a/pkg/extract/chain.go +++ b/pkg/extract/chain.go @@ -12,11 +12,21 @@ import ( ) type MessageStateChanges struct { - Current *types.TipSet - Executed *types.TipSet - BaseFee abi.TokenAmount - FullBlocks map[cid.Cid]*chain.FullBlock - ImplicitMessages []*chain.ImplicitMessage + Current *types.TipSet + Executed *types.TipSet + BaseFee abi.TokenAmount + CirculatingSupply CirculatingSupply + FullBlocks map[cid.Cid]*chain.FullBlock + ImplicitMessages []*chain.ImplicitMessage +} + +type CirculatingSupply struct { + FilVested abi.TokenAmount + FilMined abi.TokenAmount + FilBurnt abi.TokenAmount + FilLocked abi.TokenAmount + FilCirculating abi.TokenAmount + FilReserveDisbursed abi.TokenAmount } func FullBlockMessages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (*MessageStateChanges, error) { @@ -24,15 +34,27 @@ func FullBlockMessages(ctx context.Context, api tasks.DataSource, current, execu if err != nil { return nil, err } + cs, err := api.CirculatingSupply(ctx, current) + if err != nil { + return nil, err + } fullBlocks, implicitMessages, err := chain.Messages(ctx, api, current, executed) if err != nil { return nil, err } return &MessageStateChanges{ - Current: current, - Executed: executed, - BaseFee: baseFee, + Current: current, + Executed: executed, + BaseFee: baseFee, + CirculatingSupply: CirculatingSupply{ + FilVested: cs.FilVested, + FilMined: cs.FilMined, + FilBurnt: cs.FilBurnt, + FilLocked: cs.FilLocked, + FilCirculating: cs.FilCirculating, + FilReserveDisbursed: cs.FilReserveDisbursed, + }, FullBlocks: fullBlocks, ImplicitMessages: implicitMessages, }, nil diff --git a/pkg/gen/main.go b/pkg/gen/main.go index 0ce82ebfe..240978887 100644 --- a/pkg/gen/main.go +++ b/pkg/gen/main.go @@ -3,11 +3,11 @@ package main import ( cbg "github.com/whyrusleeping/cbor-gen" - datacapV9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v9" - initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" - marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" - minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" - powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + datacapV9 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" + initv0 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + marketV0 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + minerV0 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + powerV0 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" verifV0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" verifV9 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" @@ -20,26 +20,26 @@ import ( const actorDiffPath = "pkg/extract/actors/rawdiff/cbor_gen.go" const actorDiffPkg = "rawdiff" -const datacapDiffPath = "pkg/extract/actors/datacapdiff/v9/cbor_gen.go" -const datacapDiffPkg = "v9" +const datacapDiffPath = "pkg/extract/actors/datacapdiff/v1/cbor_gen.go" +const datacapDiffPkg = "v1" -const minerDiffPath = "pkg/extract/actors/minerdiff/v0/cbor_gen.go" -const minerDiffPkg = "v0" +const minerDiffPath = "pkg/extract/actors/minerdiff/v1/cbor_gen.go" +const minerDiffPkg = "v1" -const initDiffPath = "pkg/extract/actors/initdiff/v0/cbor_gen.go" -const initDiffPkg = "v0" +const initDiffPath = "pkg/extract/actors/initdiff/v1/cbor_gen.go" +const initDiffPkg = "v1" -const verifDiffPathV0 = "pkg/extract/actors/verifregdiff/v0/cbor_gen.go" -const verifDiffPkgV0 = "v0" +const verifDiffPathV0 = "pkg/extract/actors/verifregdiff/v1/cbor_gen.go" +const verifDiffPkgV0 = "v1" -const verifDiffPathV9 = "pkg/extract/actors/verifregdiff/v9/cbor_gen.go" -const verifDiffPkgV9 = "v9" +const verifDiffPathV9 = "pkg/extract/actors/verifregdiff/v2/cbor_gen.go" +const verifDiffPkgV9 = "v2" -const marketDiffPath = "pkg/extract/actors/marketdiff/v0/cbor_gen.go" -const marketDiffPkg = "v0" +const marketDiffPath = "pkg/extract/actors/marketdiff/v1/cbor_gen.go" +const marketDiffPkg = "v1" -const powerDiffPath = "pkg/extract/actors/powerdiff/v0/cbor_gen.go" -const powerDiffPkg = "v0" +const powerDiffPath = "pkg/extract/actors/powerdiff/v1/cbor_gen.go" +const powerDiffPkg = "v1" const IPLDActorContainerPath = "pkg/transform/cbor/actors/cbor_gen.go" const IPLDActorContainerPkg = "actors" diff --git a/pkg/transform/cbor/cbor_gen.go b/pkg/transform/cbor/cbor_gen.go index 90c7f3f9e..1b81ca138 100644 --- a/pkg/transform/cbor/cbor_gen.go +++ b/pkg/transform/cbor/cbor_gen.go @@ -213,7 +213,7 @@ func (t *StateExtractionIPLD) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{166}); err != nil { + if _, err := cw.Write([]byte{172}); err != nil { return err } @@ -265,6 +265,102 @@ func (t *StateExtractionIPLD) MarshalCBOR(w io.Writer) error { return err } + // t.FilVested (big.Int) (struct) + if len("filvested") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filvested\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filvested"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filvested")); err != nil { + return err + } + + if err := t.FilVested.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilMined (big.Int) (struct) + if len("filmined") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filmined\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filmined"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filmined")); err != nil { + return err + } + + if err := t.FilMined.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilBurnt (big.Int) (struct) + if len("filburnt") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filburnt\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filburnt"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filburnt")); err != nil { + return err + } + + if err := t.FilBurnt.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilLocked (big.Int) (struct) + if len("fillocked") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"fillocked\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("fillocked"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("fillocked")); err != nil { + return err + } + + if err := t.FilLocked.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilCirculating (big.Int) (struct) + if len("filcirculating") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filcirculating\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filcirculating"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filcirculating")); err != nil { + return err + } + + if err := t.FilCirculating.MarshalCBOR(cw); err != nil { + return err + } + + // t.FilReserveDisbursed (big.Int) (struct) + if len("filreserveddisbursed") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"filreserveddisbursed\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("filreserveddisbursed"))); err != nil { + return err + } + if _, err := io.WriteString(w, string("filreserveddisbursed")); err != nil { + return err + } + + if err := t.FilReserveDisbursed.MarshalCBOR(cw); err != nil { + return err + } + // t.FullBlocks (cid.Cid) (struct) if len("fullblocks") > cbg.MaxLength { return xerrors.Errorf("Value in field \"fullblocks\" was too long") @@ -383,6 +479,66 @@ func (t *StateExtractionIPLD) UnmarshalCBOR(r io.Reader) (err error) { return xerrors.Errorf("unmarshaling t.BaseFee: %w", err) } + } + // t.FilVested (big.Int) (struct) + case "filvested": + + { + + if err := t.FilVested.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilVested: %w", err) + } + + } + // t.FilMined (big.Int) (struct) + case "filmined": + + { + + if err := t.FilMined.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilMined: %w", err) + } + + } + // t.FilBurnt (big.Int) (struct) + case "filburnt": + + { + + if err := t.FilBurnt.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilBurnt: %w", err) + } + + } + // t.FilLocked (big.Int) (struct) + case "fillocked": + + { + + if err := t.FilLocked.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilLocked: %w", err) + } + + } + // t.FilCirculating (big.Int) (struct) + case "filcirculating": + + { + + if err := t.FilCirculating.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilCirculating: %w", err) + } + + } + // t.FilReserveDisbursed (big.Int) (struct) + case "filreserveddisbursed": + + { + + if err := t.FilReserveDisbursed.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.FilReserveDisbursed: %w", err) + } + } // t.FullBlocks (cid.Cid) (struct) case "fullblocks": diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index be8eceb1b..0a75b2c53 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -66,7 +66,13 @@ type StateExtractionIPLD struct { Current types.TipSet `cborgen:"current"` Parent types.TipSet `cborgen:"parent"` - BaseFee abi.TokenAmount `cborgen:"basefee"` + BaseFee abi.TokenAmount `cborgen:"basefee"` + FilVested abi.TokenAmount `cborgen:"filvested"` + FilMined abi.TokenAmount `cborgen:"filmined"` + FilBurnt abi.TokenAmount `cborgen:"filburnt"` + FilLocked abi.TokenAmount `cborgen:"fillocked"` + FilCirculating abi.TokenAmount `cborgen:"filcirculating"` + FilReserveDisbursed abi.TokenAmount `cborgen:"filreserveddisbursed"` FullBlocks cid.Cid `cborgen:"fullblocks"` ImplicitMessages cid.Cid `cborgen:"implicitmessages"` @@ -159,12 +165,18 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec } extractedState := &StateExtractionIPLD{ - Current: *current, - Parent: *executed, - BaseFee: chainState.Message.BaseFee, - FullBlocks: fullBlkHAMT, - ImplicitMessages: implicitMsgsAMT, - Actors: actorStatesRoot, + Current: *current, + Parent: *executed, + BaseFee: chainState.Message.BaseFee, + FilVested: chainState.Message.CirculatingSupply.FilVested, + FilMined: chainState.Message.CirculatingSupply.FilMined, + FilBurnt: chainState.Message.CirculatingSupply.FilBurnt, + FilLocked: chainState.Message.CirculatingSupply.FilLocked, + FilCirculating: chainState.Message.CirculatingSupply.FilCirculating, + FilReserveDisbursed: chainState.Message.CirculatingSupply.FilReserveDisbursed, + FullBlocks: fullBlkHAMT, + ImplicitMessages: implicitMsgsAMT, + Actors: actorStatesRoot, } extractedStateRoot, err := store.Put(ctx, extractedState) diff --git a/pkg/transform/timescale/actors/init/addresses.go b/pkg/transform/timescale/actors/init/addresses.go index dc8cbccf1..3dbcc2146 100644 --- a/pkg/transform/timescale/actors/init/addresses.go +++ b/pkg/transform/timescale/actors/init/addresses.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" initmodel "github.com/filecoin-project/lily/model/actors/init" "github.com/filecoin-project/lily/pkg/core" - initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" ) type Addresses struct{} diff --git a/pkg/transform/timescale/actors/init/processor.go b/pkg/transform/timescale/actors/init/processor.go index f92561aeb..0738b8bce 100644 --- a/pkg/transform/timescale/actors/init/processor.go +++ b/pkg/transform/timescale/actors/init/processor.go @@ -8,7 +8,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v0" + initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" ) func InitHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, initMapRoot cid.Cid) (model.Persistable, error) { diff --git a/pkg/transform/timescale/actors/market/v1/router.go b/pkg/transform/timescale/actors/market/v1/router.go index 51c0b5e86..7bc0dc7e0 100644 --- a/pkg/transform/timescale/actors/market/v1/router.go +++ b/pkg/transform/timescale/actors/market/v1/router.go @@ -10,7 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v0" v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v2" v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1/v3" diff --git a/pkg/transform/timescale/actors/market/v1/v0/deals.go b/pkg/transform/timescale/actors/market/v1/v0/deals.go index a9d436696..676ffea4e 100644 --- a/pkg/transform/timescale/actors/market/v1/v0/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v0/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v0/proposals.go b/pkg/transform/timescale/actors/market/v1/v0/proposals.go index e193a6415..9e3ffe1d5 100644 --- a/pkg/transform/timescale/actors/market/v1/v0/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v0/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v2/deals.go b/pkg/transform/timescale/actors/market/v1/v2/deals.go index f1de6b3ca..4fcb657bf 100644 --- a/pkg/transform/timescale/actors/market/v1/v2/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v2/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v2/proposals.go b/pkg/transform/timescale/actors/market/v1/v2/proposals.go index ad2c265a5..f52e08175 100644 --- a/pkg/transform/timescale/actors/market/v1/v2/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v2/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v2" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v3/deals.go b/pkg/transform/timescale/actors/market/v1/v3/deals.go index c9994ed7b..a1bd54122 100644 --- a/pkg/transform/timescale/actors/market/v1/v3/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v3/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v3/proposals.go b/pkg/transform/timescale/actors/market/v1/v3/proposals.go index 670082bc6..4a2727297 100644 --- a/pkg/transform/timescale/actors/market/v1/v3/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v3/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v3" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v4/deals.go b/pkg/transform/timescale/actors/market/v1/v4/deals.go index ee54e073b..19a578b31 100644 --- a/pkg/transform/timescale/actors/market/v1/v4/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v4/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v4/proposals.go b/pkg/transform/timescale/actors/market/v1/v4/proposals.go index 7d54d5d8c..01156024e 100644 --- a/pkg/transform/timescale/actors/market/v1/v4/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v4/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v4" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v5/deals.go b/pkg/transform/timescale/actors/market/v1/v5/deals.go index dc6ba07c4..a44ebc0a6 100644 --- a/pkg/transform/timescale/actors/market/v1/v5/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v5/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v5/proposals.go b/pkg/transform/timescale/actors/market/v1/v5/proposals.go index aecc5fa5f..c3339dc1b 100644 --- a/pkg/transform/timescale/actors/market/v1/v5/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v5/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v5" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v6/deals.go b/pkg/transform/timescale/actors/market/v1/v6/deals.go index 51e7f1ca2..8833f9a06 100644 --- a/pkg/transform/timescale/actors/market/v1/v6/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v6/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v6/proposals.go b/pkg/transform/timescale/actors/market/v1/v6/proposals.go index ace360b34..d57534bb5 100644 --- a/pkg/transform/timescale/actors/market/v1/v6/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v6/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v6" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v7/deals.go b/pkg/transform/timescale/actors/market/v1/v7/deals.go index cb6868f33..5a476aede 100644 --- a/pkg/transform/timescale/actors/market/v1/v7/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v7/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v7/proposals.go b/pkg/transform/timescale/actors/market/v1/v7/proposals.go index f89c57292..497bd32b1 100644 --- a/pkg/transform/timescale/actors/market/v1/v7/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v7/proposals.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v7" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v8/deals.go b/pkg/transform/timescale/actors/market/v1/v8/deals.go index dbfa52324..e99c2b716 100644 --- a/pkg/transform/timescale/actors/market/v1/v8/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v8/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v8/proposals.go b/pkg/transform/timescale/actors/market/v1/v8/proposals.go index 182ff09cd..9c45a95d8 100644 --- a/pkg/transform/timescale/actors/market/v1/v8/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v8/proposals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v8" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/market/v1/v9/deals.go b/pkg/transform/timescale/actors/market/v1/v9/deals.go index e84be56a4..dd63989b9 100644 --- a/pkg/transform/timescale/actors/market/v1/v9/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v9/deals.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v9" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) type Deals struct{} diff --git a/pkg/transform/timescale/actors/market/v1/v9/proposals.go b/pkg/transform/timescale/actors/market/v1/v9/proposals.go index 0b81541b2..7778f776a 100644 --- a/pkg/transform/timescale/actors/market/v1/v9/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v9/proposals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" - marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v0" + marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" ) diff --git a/pkg/transform/timescale/actors/miner/v1/router.go b/pkg/transform/timescale/actors/miner/v1/router.go index 1dcdefd52..09078c915 100644 --- a/pkg/transform/timescale/actors/miner/v1/router.go +++ b/pkg/transform/timescale/actors/miner/v1/router.go @@ -12,7 +12,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" diff --git a/pkg/transform/timescale/actors/miner/v1/v0/info.go b/pkg/transform/timescale/actors/miner/v1/v0/info.go index bb51c8370..3130e1abe 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v0/precommits.go b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go index f640aa60e..c05aa060e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go index e78a7b616..928ea15f6 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go index dde7d8fec..344ea24a0 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sectors.go b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go index b27602359..c0ef5ee8f 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v0" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v2/info.go b/pkg/transform/timescale/actors/miner/v1/v2/info.go index 31460cfbb..0a708e85d 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v2/precommits.go b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go index f9fb12cf7..2611ead26 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go index 387fd0fe5..a6d1f81ed 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go index 49d858dbd..da303dc74 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sectors.go b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go index f3d93d306..57fda956c 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v2" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v3/info.go b/pkg/transform/timescale/actors/miner/v1/v3/info.go index 85d694c81..1990fa86e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v3/precommits.go b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go index f65293b57..e572fd787 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go index ec3354c4b..4c0f6e382 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go index 5ca1c8009..ecfa4b1fd 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sectors.go b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go index 751333bad..9d708cc6e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v3" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v4/info.go b/pkg/transform/timescale/actors/miner/v1/v4/info.go index 139533888..80a7835c0 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v4/precommits.go b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go index a22d45420..cdb5782e1 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go index 2047206d4..3ab673034 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go index 064ef33b8..db7a93532 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sectors.go b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go index 0e3208fec..298a9d2e0 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v4" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v5/info.go b/pkg/transform/timescale/actors/miner/v1/v5/info.go index fa635d9bb..a7c0be0b4 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v5/precommits.go b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go index 93dca424d..ef63dd41e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go index 1a90727f7..51a43df21 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go index 2d4445eb7..dee50847d 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sectors.go b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go index bc4e7aff4..522336b8f 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v5" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v6/info.go b/pkg/transform/timescale/actors/miner/v1/v6/info.go index 83fa91815..b35eb7256 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v6/precommits.go b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go index aa8d4e86e..1b3c1d6d0 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go index 72cfceb2d..91ba80017 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go index 1d1e8c998..e420b8d2f 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sectors.go b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go index 233b2cca1..03c80f1f7 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v6" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v7/info.go b/pkg/transform/timescale/actors/miner/v1/v7/info.go index 2bab6e0ad..a5835eac8 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v7/precommits.go b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go index d5c629c2c..b1b613c85 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go index 821f5706a..a0b8d0693 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go index a243a0e1f..b01062518 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sectors.go b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go index a359f8822..73b1f1608 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v7" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v8/info.go b/pkg/transform/timescale/actors/miner/v1/v8/info.go index fae1ab727..c6751ecb9 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v8/precommits.go b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go index 2c85ffd0e..26423d009 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go index e3ef0bcaf..94252c990 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go index 63245ad5f..50298f02d 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sectors.go b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go index 24ba7f2ba..57e40fc37 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v8" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v9/info.go b/pkg/transform/timescale/actors/miner/v1/v9/info.go index e2bae0c99..6dc0f4625 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/info.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v9/precommits.go b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go index 764c9ecfd..948ef49f3 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go index 95cd68998..5e8a0135e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go index 6885913b3..2747cf360 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go @@ -12,7 +12,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sectors.go b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go index 332525ba9..fe5fd04c9 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go @@ -11,7 +11,7 @@ import ( minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v9" + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) diff --git a/pkg/transform/timescale/actors/power/v1/router.go b/pkg/transform/timescale/actors/power/v1/router.go index 1b9716fac..b2b2fcdfb 100644 --- a/pkg/transform/timescale/actors/power/v1/router.go +++ b/pkg/transform/timescale/actors/power/v1/router.go @@ -10,7 +10,7 @@ import ( "github.com/ipfs/go-cid" "github.com/filecoin-project/lily/model" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v0" v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v2" v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v3" diff --git a/pkg/transform/timescale/actors/power/v1/v0/claims.go b/pkg/transform/timescale/actors/power/v1/v0/claims.go index abb264d89..6cb1e42d4 100644 --- a/pkg/transform/timescale/actors/power/v1/v0/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v0/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v2/claims.go b/pkg/transform/timescale/actors/power/v1/v2/claims.go index 215f42df9..8465c4169 100644 --- a/pkg/transform/timescale/actors/power/v1/v2/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v2/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v3/claims.go b/pkg/transform/timescale/actors/power/v1/v3/claims.go index 1018fdf8f..fbaf0278a 100644 --- a/pkg/transform/timescale/actors/power/v1/v3/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v3/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v4/claims.go b/pkg/transform/timescale/actors/power/v1/v4/claims.go index 514314669..e0f1b5c64 100644 --- a/pkg/transform/timescale/actors/power/v1/v4/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v4/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v5/claims.go b/pkg/transform/timescale/actors/power/v1/v5/claims.go index 054b80d4c..c33f906fc 100644 --- a/pkg/transform/timescale/actors/power/v1/v5/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v5/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v6/claims.go b/pkg/transform/timescale/actors/power/v1/v6/claims.go index f33b85f88..2107ff220 100644 --- a/pkg/transform/timescale/actors/power/v1/v6/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v6/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v7/claims.go b/pkg/transform/timescale/actors/power/v1/v7/claims.go index ff445d1aa..40b2f41e7 100644 --- a/pkg/transform/timescale/actors/power/v1/v7/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v7/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v8/claims.go b/pkg/transform/timescale/actors/power/v1/v8/claims.go index 7f1382778..e4426b0ad 100644 --- a/pkg/transform/timescale/actors/power/v1/v8/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v8/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/actors/power/v1/v9/claims.go b/pkg/transform/timescale/actors/power/v1/v9/claims.go index ef6ad4923..4784aeda2 100644 --- a/pkg/transform/timescale/actors/power/v1/v9/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v9/claims.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" - powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v0" + powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) type Claims struct{} diff --git a/pkg/transform/timescale/fullblock/blocks.go b/pkg/transform/timescale/fullblock/blocks.go index ea5d5d04c..83c98171a 100644 --- a/pkg/transform/timescale/fullblock/blocks.go +++ b/pkg/transform/timescale/fullblock/blocks.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/model/blocks" + messagemodel "github.com/filecoin-project/lily/model/messages" "github.com/filecoin-project/lily/pkg/extract/chain" ) @@ -41,3 +42,24 @@ func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*chain.Full } return out, nil } + +func ExtractBlockMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { + out := messagemodel.BlockMessages{} + for _, fb := range fullBlocks { + for _, msg := range fb.BlsMessages { + out = append(out, &messagemodel.BlockMessage{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Message: msg.Message.Cid().String(), + }) + } + for _, msg := range fb.SecpMessages { + out = append(out, &messagemodel.BlockMessage{ + Height: int64(fb.Block.Height), + Block: fb.Block.Cid().String(), + Message: msg.Message.Cid().String(), + }) + } + } + return out, nil +} diff --git a/shed/newschema/main.go b/shed/newschema/main.go new file mode 100644 index 000000000..5859b6aa7 --- /dev/null +++ b/shed/newschema/main.go @@ -0,0 +1,288 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + "github.com/urfave/cli/v2" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/schema" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" +) + +func main() { + app := &cli.App{ + Name: "transform", + Commands: []*cli.Command{ + TransformCmd, + }, + } + app.Setup() + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stdout, err.Error()) + os.Exit(1) + } +} + +var TransformCmd = &cli.Command{ + Name: "delta", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "filename", + Usage: "Name of the file to transform", + }, + &cli.StringFlag{ + Name: "db", + Value: "host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable", + }, + }, + Action: func(cctx *cli.Context) error { + // Open the delta file (car file), this contains the filecoin state + f, err := os.OpenFile(cctx.String("filename"), os.O_RDONLY, 0o644) + defer f.Close() + if err != nil { + return err + } + + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open(cctx.String("db")), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return err + } + + // extract the delta state to the database and exit. + return Run(cctx.Context, f, db) + }, +} + +func Run(ctx context.Context, r io.Reader, db *gorm.DB) error { + // create a blockstore and load the contents of the car file (reader is a pointer to the car file) into it. + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + // expect to have a single root (cid) pointing to content + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + // we need to wrap the blockstore to meet some dumb interface. + s := store.WrapBlockStore(ctx, bs) + + // load the root container, this contains netwwork metadata and the root (cid) of the extracted state. + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := s.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { + return err + } + + // load the extracted state, this contains links to fullblocks, implicit message, network economics, and actor states + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := s.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + + // ohh and it also contains the tipset the extraction was performed over. + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent + + // for now we will only handle the fullblock dag. + if err := HandleFullBlocks(ctx, db, s, current, parent, stateExtractionIPLDContainer.FullBlocks); err != nil { + return err + } + + return nil +} + +func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, parent *types.TipSet, root cid.Cid) error { + // decode the content at the root (cid) into a concrete type we can inspect, a map of block CID to the block, its messages, their receipts and vm messages. + fullBlocks, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return err + } + // migrate the database, this creates new tables or updates existing ones with new fields + if err := db.AutoMigrate(&BlockHeaderModel{}, &ChainMessage{}, &ChainMessageReceipt{}); err != nil { + return err + } + // make some blockheaders and plop em in the database + if err := db.Create(MakeBlockHeaderModels(ctx, fullBlocks)).Error; err != nil { + return err + } + // now do messages + if err := db.Create(MakeMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + // finally receipts + if err := db.Create(MakeReceipts(ctx, fullBlocks)).Error; err != nil { + return err + } + + // okay all done. + return nil +} + +type BlockHeaderModel struct { + Cid string `gorm:"primaryKey"` + StateRoot string + Height int64 + Miner string + ParentWeight string `gorm:"bigint"` + TimeStamp uint64 + ForkSignaling uint64 + BaseFee string `gorm:"bigint"` + WinCount int64 +} + +func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*BlockHeaderModel { + out := make([]*BlockHeaderModel, 0, len(fullBlocks)) + for _, fb := range fullBlocks { + out = append(out, &BlockHeaderModel{ + Cid: fb.Block.Cid().String(), + StateRoot: fb.Block.ParentStateRoot.String(), + Height: int64(fb.Block.Height), + Miner: fb.Block.Miner.String(), + ParentWeight: fb.Block.ParentWeight.String(), + TimeStamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + BaseFee: fb.Block.ParentBaseFee.String(), + WinCount: fb.Block.ElectionProof.WinCount, + }) + } + return out +} + +type ChainMessage struct { + Cid string `gorm:"primaryKey"` + Version int64 + To string + From string + Nonce uint64 + Value string `gorm:"bigint"` + GasLimit int64 + GasFeeCap string `gorm:"bigint"` + GasPremium string `gorm:"bigint"` + Method uint64 + Params []byte +} + +func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*ChainMessage { + // messages can be contained in more than 1 block, this is used to prevent persisting them twice + seen := cid.NewSet() + var out []*ChainMessage + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + if !seen.Visit(smsg.Message.Cid()) { + continue + } + out = append(out, &ChainMessage{ + Cid: smsg.Message.Cid().String(), + Version: int64(smsg.Message.Message.Version), + To: smsg.Message.Message.To.String(), + From: smsg.Message.Message.From.String(), + Nonce: smsg.Message.Message.Nonce, + Value: smsg.Message.Message.Value.String(), + GasLimit: smsg.Message.Message.GasLimit, + GasFeeCap: smsg.Message.Message.GasFeeCap.String(), + GasPremium: smsg.Message.Message.GasPremium.String(), + Method: uint64(smsg.Message.Message.Method), + Params: smsg.Message.Message.Params, + }) + } + for _, msg := range fb.BlsMessages { + if !seen.Visit(msg.Message.Cid()) { + continue + } + out = append(out, &ChainMessage{ + Cid: msg.Message.Cid().String(), + Version: int64(msg.Message.Version), + To: msg.Message.To.String(), + From: msg.Message.From.String(), + Nonce: msg.Message.Nonce, + Value: msg.Message.Value.String(), + GasLimit: msg.Message.GasLimit, + GasFeeCap: msg.Message.GasFeeCap.String(), + GasPremium: msg.Message.GasPremium.String(), + Method: uint64(msg.Message.Method), + Params: msg.Message.Params, + }) + } + } + return out +} + +type ChainMessageReceipt struct { + MessageCid string `gorm:"primaryKey"` + Index int64 `gorm:"primaryKey"` + ExitCode int64 + GasUsed int64 + Return []byte + + BaseFeeBurn string `gorm:"bigint"` + OverEstimationBurn string `gorm:"bigint"` + MinerPenalty string `gorm:"bigint"` + MinerTip string `gorm:"bigint"` + Refund string `gorm:"bigint"` + GasRefund int64 + GasBurned int64 +} + +func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*ChainMessageReceipt { + var out []*ChainMessageReceipt + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + if smsg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &ChainMessageReceipt{ + MessageCid: smsg.Message.Cid().String(), + Index: smsg.Receipt.Index, + ExitCode: int64(smsg.Receipt.Receipt.ExitCode), + GasUsed: smsg.Receipt.Receipt.GasUsed, + Return: smsg.Receipt.Receipt.Return, + BaseFeeBurn: smsg.Receipt.GasOutputs.BaseFeeBurn.String(), + OverEstimationBurn: smsg.Receipt.GasOutputs.OverEstimationBurn.String(), + MinerPenalty: smsg.Receipt.GasOutputs.MinerPenalty.String(), + MinerTip: smsg.Receipt.GasOutputs.MinerTip.String(), + Refund: smsg.Receipt.GasOutputs.Refund.String(), + GasRefund: smsg.Receipt.GasOutputs.GasRefund, + GasBurned: smsg.Receipt.GasOutputs.GasBurned, + }) + } + for _, msg := range fb.BlsMessages { + if msg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &ChainMessageReceipt{ + MessageCid: msg.Message.Cid().String(), + Index: msg.Receipt.Index, + ExitCode: int64(msg.Receipt.Receipt.ExitCode), + GasUsed: msg.Receipt.Receipt.GasUsed, + Return: msg.Receipt.Receipt.Return, + BaseFeeBurn: msg.Receipt.GasOutputs.BaseFeeBurn.String(), + OverEstimationBurn: msg.Receipt.GasOutputs.OverEstimationBurn.String(), + MinerPenalty: msg.Receipt.GasOutputs.MinerPenalty.String(), + MinerTip: msg.Receipt.GasOutputs.MinerTip.String(), + Refund: msg.Receipt.GasOutputs.Refund.String(), + GasRefund: msg.Receipt.GasOutputs.GasRefund, + GasBurned: msg.Receipt.GasOutputs.GasBurned, + }) + } + } + return out +} From 0cddcdee847a97b9e10611f6da2d8e0a7b6fcad7 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 26 Jan 2023 18:26:44 -0800 Subject: [PATCH 39/48] extractio logic clean up --- .../timescale/actors/init/addresses.go | 19 +- .../timescale/actors/init/processor.go | 15 +- .../timescale/actors/market/router.go | 19 +- .../timescale/actors/market/v1/router.go | 7 +- .../timescale/actors/market/v1/v0/deals.go | 14 +- .../actors/market/v1/v0/proposals.go | 14 +- .../timescale/actors/market/v1/v2/deals.go | 14 +- .../actors/market/v1/v2/proposals.go | 14 +- .../timescale/actors/market/v1/v3/deals.go | 14 +- .../actors/market/v1/v3/proposals.go | 14 +- .../timescale/actors/market/v1/v4/deals.go | 14 +- .../actors/market/v1/v4/proposals.go | 14 +- .../timescale/actors/market/v1/v5/deals.go | 14 +- .../actors/market/v1/v5/proposals.go | 14 +- .../timescale/actors/market/v1/v6/deals.go | 14 +- .../actors/market/v1/v6/proposals.go | 14 +- .../timescale/actors/market/v1/v7/deals.go | 14 +- .../actors/market/v1/v7/proposals.go | 14 +- .../timescale/actors/market/v1/v8/deals.go | 14 +- .../actors/market/v1/v8/proposals.go | 14 +- .../timescale/actors/market/v1/v9/deals.go | 16 +- .../actors/market/v1/v9/proposals.go | 14 +- .../timescale/actors/miner/router.go | 71 +++--- .../timescale/actors/miner/v1/router.go | 41 +-- .../timescale/actors/miner/v1/types/types.go | 12 + .../timescale/actors/miner/v1/v0/info.go | 49 ++-- .../actors/miner/v1/v0/precommits.go | 37 +-- .../actors/miner/v1/v0/sector_deals.go | 87 ++++--- .../actors/miner/v1/v0/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v0/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v0/state.go | 63 ----- .../timescale/actors/miner/v1/v2/info.go | 48 ++-- .../actors/miner/v1/v2/precommits.go | 38 +-- .../actors/miner/v1/v2/sector_deals.go | 87 ++++--- .../actors/miner/v1/v2/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v2/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v2/state.go | 70 ------ .../timescale/actors/miner/v1/v3/info.go | 48 ++-- .../actors/miner/v1/v3/precommits.go | 38 +-- .../actors/miner/v1/v3/sector_deals.go | 87 ++++--- .../actors/miner/v1/v3/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v3/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v3/state.go | 70 ------ .../timescale/actors/miner/v1/v4/info.go | 48 ++-- .../actors/miner/v1/v4/precommits.go | 38 +-- .../actors/miner/v1/v4/sector_deals.go | 87 ++++--- .../actors/miner/v1/v4/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v4/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v4/state.go | 70 ------ .../timescale/actors/miner/v1/v5/info.go | 48 ++-- .../actors/miner/v1/v5/precommits.go | 37 +-- .../actors/miner/v1/v5/sector_deals.go | 87 ++++--- .../actors/miner/v1/v5/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v5/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v5/state.go | 70 ------ .../timescale/actors/miner/v1/v6/info.go | 48 ++-- .../actors/miner/v1/v6/precommits.go | 37 +-- .../actors/miner/v1/v6/sector_deals.go | 87 ++++--- .../actors/miner/v1/v6/sector_events.go | 224 +++++++++-------- .../timescale/actors/miner/v1/v6/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v6/state.go | 70 ------ .../timescale/actors/miner/v1/v7/info.go | 48 ++-- .../actors/miner/v1/v7/precommits.go | 38 +-- .../actors/miner/v1/v7/sector_deals.go | 87 ++++--- .../actors/miner/v1/v7/sector_events.go | 233 ++++++++--------- .../timescale/actors/miner/v1/v7/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v7/state.go | 70 ------ .../timescale/actors/miner/v1/v8/info.go | 48 ++-- .../actors/miner/v1/v8/precommits.go | 37 +-- .../actors/miner/v1/v8/sector_deals.go | 87 ++++--- .../actors/miner/v1/v8/sector_events.go | 233 ++++++++--------- .../timescale/actors/miner/v1/v8/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v8/state.go | 70 ------ .../timescale/actors/miner/v1/v9/info.go | 48 ++-- .../actors/miner/v1/v9/precommits.go | 37 +-- .../actors/miner/v1/v9/sector_deals.go | 87 ++++--- .../actors/miner/v1/v9/sector_events.go | 235 +++++++++--------- .../timescale/actors/miner/v1/v9/sectors.go | 43 ++-- .../timescale/actors/miner/v1/v9/state.go | 70 ------ .../timescale/actors/power/router.go | 53 +--- .../timescale/actors/power/v1/router.go | 7 +- .../timescale/actors/power/v1/v0/claims.go | 16 +- .../timescale/actors/power/v1/v0/state.go | 38 --- .../timescale/actors/power/v1/v2/claims.go | 16 +- .../timescale/actors/power/v1/v2/state.go | 38 --- .../timescale/actors/power/v1/v3/claims.go | 16 +- .../timescale/actors/power/v1/v3/state.go | 38 --- .../timescale/actors/power/v1/v4/claims.go | 16 +- .../timescale/actors/power/v1/v4/state.go | 38 --- .../timescale/actors/power/v1/v5/claims.go | 16 +- .../timescale/actors/power/v1/v5/state.go | 38 --- .../timescale/actors/power/v1/v6/claims.go | 16 +- .../timescale/actors/power/v1/v6/state.go | 38 --- .../timescale/actors/power/v1/v7/claims.go | 16 +- .../timescale/actors/power/v1/v7/state.go | 38 --- .../timescale/actors/power/v1/v8/claims.go | 16 +- .../timescale/actors/power/v1/v8/state.go | 38 --- .../timescale/actors/power/v1/v9/claims.go | 16 +- .../timescale/actors/power/v1/v9/state.go | 38 --- pkg/transform/timescale/actors/raw/state.go | 108 ++++++-- .../timescale/actors/reward/router.go | 49 ---- .../timescale/actors/reward/v0/state.go | 39 --- .../timescale/actors/reward/v2/state.go | 39 --- .../timescale/actors/reward/v3/state.go | 39 --- .../timescale/actors/reward/v4/state.go | 39 --- .../timescale/actors/reward/v5/state.go | 39 --- .../timescale/actors/reward/v6/state.go | 39 --- .../timescale/actors/reward/v7/state.go | 39 --- .../timescale/actors/reward/v8/state.go | 39 --- .../timescale/actors/reward/v9/state.go | 39 --- .../timescale/actors/verifreg/router.go | 37 ++- pkg/transform/timescale/data/report.go | 64 +++++ pkg/transform/timescale/fullblock/blocks.go | 41 ++- pkg/transform/timescale/fullblock/messages.go | 34 +-- pkg/transform/timescale/processor.go | 146 +++-------- 115 files changed, 2775 insertions(+), 3647 deletions(-) create mode 100644 pkg/transform/timescale/actors/miner/v1/types/types.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v0/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v2/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v3/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v4/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v5/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v6/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v7/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v8/state.go delete mode 100644 pkg/transform/timescale/actors/miner/v1/v9/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v0/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v2/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v3/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v4/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v5/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v6/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v7/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v8/state.go delete mode 100644 pkg/transform/timescale/actors/power/v1/v9/state.go delete mode 100644 pkg/transform/timescale/actors/reward/router.go delete mode 100644 pkg/transform/timescale/actors/reward/v0/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v2/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v3/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v4/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v5/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v6/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v7/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v8/state.go delete mode 100644 pkg/transform/timescale/actors/reward/v9/state.go create mode 100644 pkg/transform/timescale/data/report.go diff --git a/pkg/transform/timescale/actors/init/addresses.go b/pkg/transform/timescale/actors/init/addresses.go index 3dbcc2146..533993b1c 100644 --- a/pkg/transform/timescale/actors/init/addresses.go +++ b/pkg/transform/timescale/actors/init/addresses.go @@ -8,31 +8,36 @@ import ( "github.com/filecoin-project/lotus/chain/types" cbg "github.com/whyrusleeping/cbor-gen" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" initmodel "github.com/filecoin-project/lily/model/actors/init" "github.com/filecoin-project/lily/pkg/core" initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Addresses struct{} -func (a Addresses) Extract(ctx context.Context, current, executed *types.TipSet, changes *initdiff.StateDiffResult) (model.Persistable, error) { - out := initmodel.IDAddressList{} +func (a Addresses) Extract(ctx context.Context, current, executed *types.TipSet, changes *initdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.IDAddress, current) for _, change := range changes.AddressesChanges { if change.Change == core.ChangeTypeAdd || change.Change == core.ChangeTypeModify { robustAddr, err := address.NewFromBytes(change.Address) if err != nil { - return nil, err + report.AddError(err) + continue } var actorID cbg.CborInt if err := actorID.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } idAddr, err := address.NewIDAddress(uint64(actorID)) if err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &initmodel.IDAddress{ + report.AddModels(&initmodel.IDAddress{ Height: int64(current.Height()), StateRoot: current.ParentState().String(), ID: idAddr.String(), @@ -40,5 +45,5 @@ func (a Addresses) Extract(ctx context.Context, current, executed *types.TipSet, }) } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/init/processor.go b/pkg/transform/timescale/actors/init/processor.go index 0738b8bce..36326c6cd 100644 --- a/pkg/transform/timescale/actors/init/processor.go +++ b/pkg/transform/timescale/actors/init/processor.go @@ -7,13 +7,22 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" initdiff "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func InitHandler(ctx context.Context, s store.Store, current, executed *types.TipSet, initMapRoot cid.Cid) (model.Persistable, error) { +func TransformInitState(ctx context.Context, s store.Store, current, executed *types.TipSet, initMapRoot *cid.Cid) (model.Persistable, error) { + if initMapRoot == nil { + return data.StartProcessingReport(tasktype.IDAddress, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), nil + } initState := new(initdiff.StateChange) - if err := s.Get(ctx, initMapRoot, initState); err != nil { + if err := s.Get(ctx, *initMapRoot, initState); err != nil { return nil, err } @@ -22,5 +31,5 @@ func InitHandler(ctx context.Context, s store.Store, current, executed *types.Ti return nil, err } - return Addresses{}.Extract(ctx, current, executed, initStateDiff) + return Addresses{}.Extract(ctx, current, executed, initStateDiff), nil } diff --git a/pkg/transform/timescale/actors/market/router.go b/pkg/transform/timescale/actors/market/router.go index 8ca8c6d80..4a2979cd2 100644 --- a/pkg/transform/timescale/actors/market/router.go +++ b/pkg/transform/timescale/actors/market/router.go @@ -9,11 +9,26 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { +func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.MarketDealState, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MarketDealProposal, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } switch version { case actortypes.Version0, actortypes.Version2, @@ -24,7 +39,7 @@ func TransformMarketState(ctx context.Context, s store.Store, version actortypes actortypes.Version7, actortypes.Version8, actortypes.Version9: - return v1.TransformMarketState(ctx, s, version, current, executed, root) + return v1.TransformMarketState(ctx, s, version, current, executed, *root) } return nil, fmt.Errorf("unsupported version : %d", version) diff --git a/pkg/transform/timescale/actors/market/v1/router.go b/pkg/transform/timescale/actors/market/v1/router.go index 7bc0dc7e0..66d340d82 100644 --- a/pkg/transform/timescale/actors/market/v1/router.go +++ b/pkg/transform/timescale/actors/market/v1/router.go @@ -23,7 +23,7 @@ import ( ) type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) + Transform(ctx context.Context, current, parent *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable } func TransformMarketState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { @@ -41,10 +41,7 @@ func TransformMarketState(ctx context.Context, s store.Store, version actortypes } out := model.PersistableList{} for _, t := range transformers { - m, err := t.Transform(ctx, current, executed, marketStateDiff) - if err != nil { - return nil, err - } + m := t.Transform(ctx, current, executed, marketStateDiff) out = append(out, m) } return out, nil diff --git a/pkg/transform/timescale/actors/market/v1/v0/deals.go b/pkg/transform/timescale/actors/market/v1/v0/deals.go index 676ffea4e..53a279736 100644 --- a/pkg/transform/timescale/actors/market/v1/v0/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v0/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v0/proposals.go b/pkg/transform/timescale/actors/market/v1/v0/proposals.go index 9e3ffe1d5..75ddd4a91 100644 --- a/pkg/transform/timescale/actors/market/v1/v0/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v0/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v2/deals.go b/pkg/transform/timescale/actors/market/v1/v2/deals.go index 4fcb657bf..ac3a661c1 100644 --- a/pkg/transform/timescale/actors/market/v1/v2/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v2/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v2/proposals.go b/pkg/transform/timescale/actors/market/v1/v2/proposals.go index f52e08175..b771b392d 100644 --- a/pkg/transform/timescale/actors/market/v1/v2/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v2/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v2/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v3/deals.go b/pkg/transform/timescale/actors/market/v1/v3/deals.go index a1bd54122..56ac5f168 100644 --- a/pkg/transform/timescale/actors/market/v1/v3/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v3/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v3/proposals.go b/pkg/transform/timescale/actors/market/v1/v3/proposals.go index 4a2727297..af5be3976 100644 --- a/pkg/transform/timescale/actors/market/v1/v3/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v3/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v3/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v4/deals.go b/pkg/transform/timescale/actors/market/v1/v4/deals.go index 19a578b31..6dd82d7d5 100644 --- a/pkg/transform/timescale/actors/market/v1/v4/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v4/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v4/proposals.go b/pkg/transform/timescale/actors/market/v1/v4/proposals.go index 01156024e..f27c635a2 100644 --- a/pkg/transform/timescale/actors/market/v1/v4/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v4/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v4/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v5/deals.go b/pkg/transform/timescale/actors/market/v1/v5/deals.go index a44ebc0a6..97b745ce0 100644 --- a/pkg/transform/timescale/actors/market/v1/v5/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v5/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v5/proposals.go b/pkg/transform/timescale/actors/market/v1/v5/proposals.go index c3339dc1b..a02b55905 100644 --- a/pkg/transform/timescale/actors/market/v1/v5/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v5/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v5/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v6/deals.go b/pkg/transform/timescale/actors/market/v1/v6/deals.go index 8833f9a06..b742734e7 100644 --- a/pkg/transform/timescale/actors/market/v1/v6/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v6/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v6/proposals.go b/pkg/transform/timescale/actors/market/v1/v6/proposals.go index d57534bb5..f30bc6232 100644 --- a/pkg/transform/timescale/actors/market/v1/v6/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v6/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v6/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v7/deals.go b/pkg/transform/timescale/actors/market/v1/v7/deals.go index 5a476aede..e9a4ad05b 100644 --- a/pkg/transform/timescale/actors/market/v1/v7/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v7/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v7/proposals.go b/pkg/transform/timescale/actors/market/v1/v7/proposals.go index 497bd32b1..c2e7fa115 100644 --- a/pkg/transform/timescale/actors/market/v1/v7/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v7/proposals.go @@ -8,16 +8,19 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v7/actors/builtin/market" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -26,14 +29,15 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + return report.AddModels(MarketDealProposalChangesAsModel(ctx, current, marketProposals)).Finish() } type proposal struct { @@ -41,7 +45,7 @@ type proposal struct { State *market.DealProposal } -func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) (model.Persistable, error) { +func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet, dealProps []*proposal) model.Persistable { dealPropsModel := make(marketmodel.MarketDealProposals, len(dealProps)) for i, prop := range dealProps { label := base64.StdEncoding.EncodeToString([]byte(util.SanitizeLabel(prop.State.Label))) @@ -64,5 +68,5 @@ func MarketDealProposalChangesAsModel(ctx context.Context, current *types.TipSet IsString: false, } } - return dealPropsModel, nil + return dealPropsModel } diff --git a/pkg/transform/timescale/actors/market/v1/v8/deals.go b/pkg/transform/timescale/actors/market/v1/v8/deals.go index e99c2b716..d1a5054e5 100644 --- a/pkg/transform/timescale/actors/market/v1/v8/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v8/deals.go @@ -7,15 +7,18 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v8/market" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals for _, change := range changes.DealStateChanges { // only care about new and modified deal states @@ -24,14 +27,15 @@ func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, cha } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v8/proposals.go b/pkg/transform/timescale/actors/market/v1/v8/proposals.go index 9c45a95d8..a7ce8d949 100644 --- a/pkg/transform/timescale/actors/market/v1/v8/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v8/proposals.go @@ -9,16 +9,19 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v8/market" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -27,14 +30,19 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + m, err := MarketDealProposalChangesAsModel(ctx, current, marketProposals) + if err != nil { + return report.AddError(err).Finish() + } + return report.AddModels(m).Finish() } type proposal struct { diff --git a/pkg/transform/timescale/actors/market/v1/v9/deals.go b/pkg/transform/timescale/actors/market/v1/v9/deals.go index dd63989b9..95a4358f4 100644 --- a/pkg/transform/timescale/actors/market/v1/v9/deals.go +++ b/pkg/transform/timescale/actors/market/v1/v9/deals.go @@ -7,31 +7,35 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Deals struct{} -func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Deals) Transform(ctx context.Context, current, executed *types.TipSet, changes *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealState, current) var marketDeals []*deals - for _, change := range change.DealStateChanges { + for _, change := range changes.DealStateChanges { // only care about new and modified deal states if change.Change == core.ChangeTypeRemove { continue } dealState := new(market.DealState) if err := dealState.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketDeals = append(marketDeals, &deals{ DealID: change.DealID, State: dealState, }) } - return MarketDealStateChangesAsModel(ctx, current, marketDeals) + return report.AddModels(MarketDealStateChangesAsModel(ctx, current, marketDeals)).Finish() } type deals struct { @@ -39,7 +43,7 @@ type deals struct { State *market.DealState } -func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) (model.Persistable, error) { +func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, dealStates []*deals) model.Persistable { dealStateModel := make(marketmodel.MarketDealStates, len(dealStates)) for i, deal := range dealStates { dealStateModel[i] = &marketmodel.MarketDealState{ @@ -51,5 +55,5 @@ func MarketDealStateChangesAsModel(ctx context.Context, current *types.TipSet, d SlashEpoch: int64(deal.State.SlashEpoch), } } - return dealStateModel, nil + return dealStateModel } diff --git a/pkg/transform/timescale/actors/market/v1/v9/proposals.go b/pkg/transform/timescale/actors/market/v1/v9/proposals.go index 7778f776a..e02fd7957 100644 --- a/pkg/transform/timescale/actors/market/v1/v9/proposals.go +++ b/pkg/transform/timescale/actors/market/v1/v9/proposals.go @@ -9,16 +9,19 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/market" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" marketmodel "github.com/filecoin-project/lily/model/actors/market" "github.com/filecoin-project/lily/pkg/core" marketdiff "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/market/util" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Proposals struct{} -func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) (model.Persistable, error) { +func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, change *marketdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.MarketDealProposal, current) var marketProposals []*proposal for _, change := range change.DealProposalChanges { // we only car about new and modified deal proposals @@ -27,14 +30,19 @@ func (Proposals) Transform(ctx context.Context, current, executed *types.TipSet, } dealProp := new(market.DealProposal) if err := dealProp.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } marketProposals = append(marketProposals, &proposal{ DealID: change.DealID, State: dealProp, }) } - return MarketDealProposalChangesAsModel(ctx, current, marketProposals) + m, err := MarketDealProposalChangesAsModel(ctx, current, marketProposals) + if err != nil { + return report.AddError(err).Finish() + } + return report.AddModels(m).Finish() } type proposal struct { diff --git a/pkg/transform/timescale/actors/miner/router.go b/pkg/transform/timescale/actors/miner/router.go index e31fb2aa2..503550f18 100644 --- a/pkg/transform/timescale/actors/miner/router.go +++ b/pkg/transform/timescale/actors/miner/router.go @@ -4,54 +4,47 @@ import ( "context" "fmt" - "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + visormodel "github.com/filecoin-project/lily/model/visor" v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1" - v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" - v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" - v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" - v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v4" - v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v5" - v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v6" - v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v7" - v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v8" - v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v9" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func HandleMiner(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { - switch version { - case actortypes.Version0: - return v1_0.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version2: - return v1_2.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version3: - return v1_3.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version4: - return v1_4.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version5: - return v1_5.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version6: - return v1_6.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version7: - return v1_7.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version8: - return v1_8.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version9: - return v1_9.ExtractMinerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version10: - panic("not yet implemented") - default: - return nil, fmt.Errorf("unsupported miner actor version: %d", version) +func TransformMinerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + sectorInfoTaskName := tasktype.MinerSectorInfoV7 + if version < actortypes.Version7 { + sectorInfoTaskName = tasktype.MinerSectorInfoV1_6 + } + return model.PersistableList{ + data.StartProcessingReport(tasktype.MinerInfo, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerPreCommitInfo, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerSectorDeal, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.MinerSectorEvent, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(sectorInfoTaskName, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil } -} - -func TransformMinerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { switch version { case actortypes.Version0, actortypes.Version2, @@ -62,7 +55,7 @@ func TransformMinerState(ctx context.Context, s store.Store, version actortypes. actortypes.Version7, actortypes.Version8, actortypes.Version9: - return v1.TransformMinerStates(ctx, s, version, current, executed, root) + return v1.TransformMinerStates(ctx, s, version, current, executed, *root) } return nil, fmt.Errorf("unsupported version : %d", version) diff --git a/pkg/transform/timescale/actors/miner/v1/router.go b/pkg/transform/timescale/actors/miner/v1/router.go index 09078c915..773ceb00d 100644 --- a/pkg/transform/timescale/actors/miner/v1/router.go +++ b/pkg/transform/timescale/actors/miner/v1/router.go @@ -13,6 +13,7 @@ import ( "github.com/filecoin-project/lily/model" minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v0" v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v2" v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/v3" @@ -25,50 +26,52 @@ import ( ) func TransformMinerStates(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { - out := model.PersistableList{} - // a map of all miners whose state has changes minerMap, err := adt.AsMap(s, root, 5) if err != nil { return nil, err } - // iterate over each miner with a state change - transformers, err := LookupMinerStateTransformer(version) - if err != nil { - return nil, err - } + + // load map entires into a list to process below + var miners []*minertypes.MinerStateChange minerState := new(minerdiff.StateChange) if err := minerMap.ForEach(minerState, func(key string) error { - // the map is keyed by the miner address, the value of the map contains the miners state change addr, err := address.NewFromBytes([]byte(key)) if err != nil { return err } - // decode the miner state change from the IPLD structure to a type we can inspect. minerStateDiff, err := minerState.ToStateDiffResult(ctx, s) if err != nil { return err } - - for _, t := range transformers { - m, err := t.Transform(ctx, current, executed, addr, minerStateDiff) - if err != nil { - return err - } - out = append(out, m) - } - + miners = append(miners, &minertypes.MinerStateChange{ + Address: addr, + StateChange: minerStateDiff, + }) return nil }); err != nil { return nil, err } + + // get a list of transformers capable of handling this actor version + transformers, err := LookupMinerStateTransformer(version) + if err != nil { + return nil, err + } + + // run each transformer keeping its model + out := model.PersistableList{} + for _, t := range transformers { + m := t.Transform(ctx, current, executed, miners) + out = append(out, m) + } return out, nil } type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) + Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable } func LookupMinerStateTransformer(av actortypes.Version) ([]Transformer, error) { diff --git a/pkg/transform/timescale/actors/miner/v1/types/types.go b/pkg/transform/timescale/actors/miner/v1/types/types.go new file mode 100644 index 000000000..7e1c57b07 --- /dev/null +++ b/pkg/transform/timescale/actors/miner/v1/types/types.go @@ -0,0 +1,12 @@ +package types + +import ( + "github.com/filecoin-project/go-address" + + minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" +) + +type MinerStateChange struct { + Address address.Address + StateChange *minerdiff.StateDiffResult +} diff --git a/pkg/transform/timescale/actors/miner/v1/v0/info.go b/pkg/transform/timescale/actors/miner/v1/v0/info.go index 3130e1abe..741e2aa4b 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/info.go @@ -8,37 +8,44 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v0/precommits.go b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go index c05aa060e..ecfe72a87 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,6 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go index 928ea15f6..cfc670ec8 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go index 344ea24a0..f5d04fc44 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v0/sectors.go b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go index c0ef5ee8f..2b36a423b 100644 --- a/pkg/transform/timescale/actors/miner/v1/v0/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v0/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v0/state.go b/pkg/transform/timescale/actors/miner/v1/v0/state.go deleted file mode 100644 index ea943db69..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v0/state.go +++ /dev/null @@ -1,63 +0,0 @@ -package v0 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: big.Zero().String(), // v0 has no fee debt - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledgeRequirement.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledgeRequirement.Equals(previousState.InitialPledgeRequirement) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledgeRequirement.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v2/info.go b/pkg/transform/timescale/actors/miner/v1/v2/info.go index 0a708e85d..a3db072fe 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v2/precommits.go b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go index 2611ead26..5c1cbda50 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,5 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil - + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go index a6d1f81ed..7e2764209 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go index da303dc74..5a530baf9 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v2/sectors.go b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go index 57fda956c..e5975bcb6 100644 --- a/pkg/transform/timescale/actors/miner/v1/v2/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v2/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v2/state.go b/pkg/transform/timescale/actors/miner/v1/v2/state.go deleted file mode 100644 index 955b26818..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v2/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v2 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v3/info.go b/pkg/transform/timescale/actors/miner/v1/v3/info.go index 1990fa86e..029dcd161 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v3/precommits.go b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go index e572fd787..f1d806015 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,5 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil - + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go index 4c0f6e382..9b9f85ee6 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go index ecfa4b1fd..eeee246df 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v3/sectors.go b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go index 9d708cc6e..28f39533e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v3/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v3/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v3/state.go b/pkg/transform/timescale/actors/miner/v1/v3/state.go deleted file mode 100644 index ece3578da..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v3/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v3 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v4/info.go b/pkg/transform/timescale/actors/miner/v1/v4/info.go index 80a7835c0..4fa4fbcc8 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v4/precommits.go b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go index cdb5782e1..deb4dbc2e 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,5 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil - + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go index 3ab673034..ac4f22208 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go index db7a93532..78aff5638 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v4/sectors.go b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go index 298a9d2e0..ff9319c5b 100644 --- a/pkg/transform/timescale/actors/miner/v1/v4/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v4/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v4/state.go b/pkg/transform/timescale/actors/miner/v1/v4/state.go deleted file mode 100644 index 24524696e..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v4/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v4 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v4/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v5/info.go b/pkg/transform/timescale/actors/miner/v1/v5/info.go index a7c0be0b4..5680f54ff 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v5/precommits.go b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go index ef63dd41e..3ef9e30e7 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,6 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go index 51a43df21..dea59441c 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go index dee50847d..b6efa12af 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v5/sectors.go b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go index 522336b8f..a9fd0d7cc 100644 --- a/pkg/transform/timescale/actors/miner/v1/v5/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v5/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v5/state.go b/pkg/transform/timescale/actors/miner/v1/v5/state.go deleted file mode 100644 index d539cf6ba..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v5/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v5 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v6/info.go b/pkg/transform/timescale/actors/miner/v1/v6/info.go index b35eb7256..e3eb10e65 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v6/precommits.go b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go index 1b3c1d6d0..4ae93453b 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,6 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go index 91ba80017..136c6a2fa 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go index e420b8d2f..c4ea8f177 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sector_events.go @@ -4,140 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorExtended, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v6/sectors.go b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go index 03c80f1f7..b9952425f 100644 --- a/pkg/transform/timescale/actors/miner/v1/v6/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v6/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV1_6, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV1_6List, len(sectors)) for i, sector := range sectors { sectorModel[i] = &minermodel.MinerSectorInfoV1_6{ @@ -55,5 +60,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v6/state.go b/pkg/transform/timescale/actors/miner/v1/v6/state.go deleted file mode 100644 index 45aba28c8..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v6/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v6 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v6/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v7/info.go b/pkg/transform/timescale/actors/miner/v1/v7/info.go index a5835eac8..48a34b213 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v7/precommits.go b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go index b1b613c85..9d21eb9de 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,5 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil - + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go index a0b8d0693..1112c71db 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go index b01062518..7cb5af61f 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sector_events.go @@ -4,149 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sector.SectorNumber, - StateRoot: stateRoot, - Event: minermodel.SectorExtended, - }) - } - if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorSnapped, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v7/sectors.go b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go index 73b1f1608..8fb67b2fa 100644 --- a/pkg/transform/timescale/actors/miner/v1/v7/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v7/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) for i, sector := range sectors { sectorKeyCID := "" @@ -60,5 +65,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v7/state.go b/pkg/transform/timescale/actors/miner/v1/v7/state.go deleted file mode 100644 index 16541595d..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v7/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v7 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/specs-actors/v7/actors/builtin/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v8/info.go b/pkg/transform/timescale/actors/miner/v1/v8/info.go index c6751ecb9..3507616a4 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v8/precommits.go b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go index 26423d009..0eee24323 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoList, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,6 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go index 94252c990..5c6893f0a 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go index 50298f02d..e30d1d6c8 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sector_events.go @@ -4,149 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sector.SectorNumber, - StateRoot: stateRoot, - Event: minermodel.SectorExtended, - }) - } - if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorSnapped, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v8/sectors.go b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go index 57e40fc37..f93cd8d1c 100644 --- a/pkg/transform/timescale/actors/miner/v1/v8/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v8/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) for i, sector := range sectors { sectorKeyCID := "" @@ -60,5 +65,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v8/state.go b/pkg/transform/timescale/actors/miner/v1/v8/state.go deleted file mode 100644 index 0f1ad68d9..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v8/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v8 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/go-state-types/builtin/v8/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/miner/v1/v9/info.go b/pkg/transform/timescale/actors/miner/v1/v9/info.go index 6dc0f4625..5d35a6179 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/info.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/info.go @@ -8,37 +8,43 @@ import ( "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type Info struct{} -func (i Info) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - // if there is no info nothing changed. - if change.InfoChange == nil { - return nil, nil - } - // if the info was removed there is nothing to record - if change.InfoChange.Change == core.ChangeTypeRemove { - return nil, nil - } - // unmarshal the miners info to its type - minerInfo := new(miner.MinerInfo) - if err := minerInfo.UnmarshalCBOR(bytes.NewReader(change.InfoChange.Info.Raw)); err != nil { - return nil, err - } - // wrap the versioned miner info type in an interface for reusable extraction - out, err := util.ExtractMinerInfo(ctx, current, executed, addr, &InfoWrapper{info: minerInfo}) - if err != nil { - return nil, err +func (i Info) Transform(ctx context.Context, current, parent *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerInfo, current) + for _, m := range miners { + // if there is no info nothing changed. + if m.StateChange.InfoChange == nil { + continue + } + // if the info was removed there is nothing to record + if m.StateChange.InfoChange.Change == core.ChangeTypeRemove { + continue + } + // unmarshal the miners info to its type + minerInfo := new(miner.MinerInfo) + if err := minerInfo.UnmarshalCBOR(bytes.NewReader(m.StateChange.InfoChange.Info.Raw)); err != nil { + report.AddError(err) + continue + } + // wrap the versioned miner info type in an interface for reusable extraction + infoModel, err := util.ExtractMinerInfo(ctx, current, parent, m.Address, &InfoWrapper{info: minerInfo}) + if err != nil { + report.AddError(err) + } + report.AddModels(infoModel) } - return out, nil + return report.Finish() } // InfoWrapper satisfies the interface required by ExtractMinerInfo. diff --git a/pkg/transform/timescale/actors/miner/v1/v9/precommits.go b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go index 948ef49f3..dc6862349 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/precommits.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/precommits.go @@ -7,34 +7,39 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type PreCommit struct{} -func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var precommits []*miner.SectorPreCommitOnChainInfo - for _, change := range change.PreCommitChanges { - // only care about precommits added - if change.Change != core.ChangeTypeAdd { - continue - } - precommit := new(miner.SectorPreCommitOnChainInfo) - if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err +func (PreCommit) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerPreCommitInfo, current) + for _, m := range miners { + var precommits []*miner.SectorPreCommitOnChainInfo + for _, change := range m.StateChange.PreCommitChanges { + // only care about precommits added + if change.Change != core.ChangeTypeAdd { + continue + } + precommit := new(miner.SectorPreCommitOnChainInfo) + if err := precommit.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { + report.AddError(err) + } + precommits = append(precommits, precommit) } - precommits = append(precommits, precommit) + report.AddModels(MinerPreCommitChangesAsModel(ctx, current, m.Address, precommits)) } - return MinerPreCommitChangesAsModel(ctx, current, addr, precommits) + return report.Finish() } -func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) (model.Persistable, error) { +func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, precommits []*miner.SectorPreCommitOnChainInfo) model.Persistable { preCommitModel := make(minermodel.MinerPreCommitInfoV9List, len(precommits)) for i, preCommit := range precommits { deals := make([]uint64, len(preCommit.Info.DealIDs)) @@ -60,6 +65,6 @@ func MinerPreCommitChangesAsModel(ctx context.Context, current *types.TipSet, ad } } - return preCommitModel, nil + return preCommitModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go index 5e8a0135e..c896bdc6b 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_deals.go @@ -4,59 +4,66 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/util" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type SectorDeal struct{} -func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - sectors := change.SectorChanges - out := minermodel.MinerSectorDealList{} - height := int64(current.Height()) - minerAddr := addr.String() - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range s.DealIDs { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(s.SectorNumber), - DealID: uint64(deal), - }) - } - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { - out = append(out, &minermodel.MinerSectorDeal{ - Height: height, - MinerID: minerAddr, - SectorID: uint64(currentSector.SectorNumber), - DealID: uint64(deal), - }) +func (s SectorDeal) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorDeal, current) + for _, m := range miners { + + sectors := m.StateChange.SectorChanges + height := int64(current.Height()) + minerAddr := m.Address.String() + + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range s.DealIDs { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(s.SectorNumber), + DealID: uint64(deal), + }) + } + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + for _, deal := range util.CompareDealIDs(currentSector.DealIDs, previousSector.DealIDs) { + report.AddModels(&minermodel.MinerSectorDeal{ + Height: height, + MinerID: minerAddr, + SectorID: uint64(currentSector.SectorNumber), + DealID: uint64(deal), + }) + } } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go index 2747cf360..0da5cbd6a 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sector_events.go @@ -4,149 +4,150 @@ import ( "bytes" "context" - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" - + "github.com/filecoin-project/go-state-types/abi" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) type SectorEvent struct{} -func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var ( - precommits = change.PreCommitChanges - sectors = change.SectorChanges - sectorstatus = change.SectorStatusChanges - height = int64(current.Height()) - minerAddr = addr.String() - stateRoot = current.ParentState().String() - out = minermodel.MinerSectorEventList{} - ) - for _, precommit := range precommits { - // only care about new precommits - if precommit.Change != core.ChangeTypeAdd { - continue - } - sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) - if err != nil { - return nil, err - } - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sectorID, - StateRoot: stateRoot, - Event: minermodel.PreCommitAdded, - }) - } - for _, sector := range sectors { - switch sector.Change { - case core.ChangeTypeAdd: - event := minermodel.SectorAdded - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s SectorEvent) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorEvent, current) + for _, m := range miners { + var ( + precommits = m.StateChange.PreCommitChanges + sectors = m.StateChange.SectorChanges + sectorstatus = m.StateChange.SectorStatusChanges + height = int64(current.Height()) + minerAddr = m.Address.String() + stateRoot = current.ParentState().String() + ) + for _, precommit := range precommits { + // only care about new precommits + if precommit.Change != core.ChangeTypeAdd { + continue } - if len(s.DealIDs) == 0 { - event = minermodel.CommitCapacityAdded + sectorID, err := abi.ParseUIntKey(string(precommit.SectorNumber)) + if err != nil { + report.AddError(err) + continue } - out = append(out, &minermodel.MinerSectorEvent{ + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, - SectorID: sector.SectorNumber, + SectorID: sectorID, StateRoot: stateRoot, - Event: event, + Event: minermodel.PreCommitAdded, }) - case core.ChangeTypeModify: - previousSector := new(miner.SectorOnChainInfo) - if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { - return nil, err - } - currentSector := new(miner.SectorOnChainInfo) - if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err - } - if previousSector.Expiration != currentSector.Expiration { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: sector.SectorNumber, - StateRoot: stateRoot, - Event: minermodel.SectorExtended, - }) - } - if previousSector.SectorKeyCID == nil && currentSector.SectorKeyCID != nil { - out = append(out, &minermodel.MinerSectorEvent{ + } + for _, sector := range sectors { + switch sector.Change { + case core.ChangeTypeAdd: + event := minermodel.SectorAdded + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if len(s.DealIDs) == 0 { + event = minermodel.CommitCapacityAdded + } + report.AddModels(&minermodel.MinerSectorEvent{ Height: height, MinerID: minerAddr, SectorID: sector.SectorNumber, StateRoot: stateRoot, - Event: minermodel.SectorSnapped, + Event: event, }) + case core.ChangeTypeModify: + previousSector := new(miner.SectorOnChainInfo) + if err := previousSector.UnmarshalCBOR(bytes.NewReader(sector.Previous.Raw)); err != nil { + report.AddError(err) + continue + } + currentSector := new(miner.SectorOnChainInfo) + if err := currentSector.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + if previousSector.Expiration != currentSector.Expiration { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: sector.SectorNumber, + StateRoot: stateRoot, + Event: minermodel.SectorExtended, + }) + } } } - } - if sectorstatus == nil { - return out, nil - } - // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. - if err := sectorstatus.Removed.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorTerminated, - }) - return nil - }); err != nil { - return nil, err - } + if sectorstatus == nil { + continue + } + // all sectors removed this epoch are considered terminated, this includes both early terminations and expirations. + if err := sectorstatus.Removed.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorTerminated, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovering.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovering, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Recovering.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovering, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Faulted.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorFaulted, - }) - return nil - }); err != nil { - return nil, err - } + if err := sectorstatus.Faulted.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorFaulted, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } - if err := sectorstatus.Recovered.ForEach(func(u uint64) error { - out = append(out, &minermodel.MinerSectorEvent{ - Height: height, - MinerID: minerAddr, - SectorID: u, - StateRoot: stateRoot, - Event: minermodel.SectorRecovered, - }) - return nil - }); err != nil { - return nil, err + if err := sectorstatus.Recovered.ForEach(func(u uint64) error { + report.AddModels(&minermodel.MinerSectorEvent{ + Height: height, + MinerID: minerAddr, + SectorID: u, + StateRoot: stateRoot, + Event: minermodel.SectorRecovered, + }) + return nil + }); err != nil { + report.AddError(err) + continue + } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/miner/v1/v9/sectors.go b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go index fe5fd04c9..8a7898939 100644 --- a/pkg/transform/timescale/actors/miner/v1/v9/sectors.go +++ b/pkg/transform/timescale/actors/miner/v1/v9/sectors.go @@ -7,36 +7,41 @@ import ( "github.com/filecoin-project/go-address" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" minermodel "github.com/filecoin-project/lily/model/actors/miner" "github.com/filecoin-project/lily/pkg/core" - - minerdiff "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" + minertypes "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner/v1/types" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" ) -type Sector struct { -} +type Sector struct{} -func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *minerdiff.StateDiffResult) (model.Persistable, error) { - var sectors []*miner.SectorOnChainInfo - changes := change.SectorChanges - for _, sector := range changes { - // only care about modified and added sectors - if sector.Change == core.ChangeTypeRemove { - continue - } - s := new(miner.SectorOnChainInfo) - if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { - return nil, err +func (s Sector) Transform(ctx context.Context, current, executed *types.TipSet, miners []*minertypes.MinerStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.MinerSectorInfoV7, current) + for _, m := range miners { + var sectors []*miner.SectorOnChainInfo + changes := m.StateChange.SectorChanges + for _, sector := range changes { + // only care about modified and added sectors + if sector.Change == core.ChangeTypeRemove { + continue + } + s := new(miner.SectorOnChainInfo) + if err := s.UnmarshalCBOR(bytes.NewReader(sector.Current.Raw)); err != nil { + report.AddError(err) + continue + } + sectors = append(sectors, s) } - sectors = append(sectors, s) + report.AddModels(MinerSectorChangesAsModel(ctx, current, m.Address, sectors)) } - return MinerSectorChangesAsModel(ctx, current, addr, sectors) + return report.Finish() } -func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) (model.Persistable, error) { +func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr address.Address, sectors []*miner.SectorOnChainInfo) model.Persistable { sectorModel := make(minermodel.MinerSectorInfoV7List, len(sectors)) for i, sector := range sectors { sectorKeyCID := "" @@ -60,5 +65,5 @@ func MinerSectorChangesAsModel(ctx context.Context, current *types.TipSet, addr } } - return sectorModel, nil + return sectorModel } diff --git a/pkg/transform/timescale/actors/miner/v1/v9/state.go b/pkg/transform/timescale/actors/miner/v1/v9/state.go deleted file mode 100644 index fbc5e6df3..000000000 --- a/pkg/transform/timescale/actors/miner/v1/v9/state.go +++ /dev/null @@ -1,70 +0,0 @@ -package v9 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - minermodel "github.com/filecoin-project/lily/model/actors/miner" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - - miner "github.com/filecoin-project/go-state-types/builtin/v9/miner" -) - -func ExtractMinerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - return nil, nil - } - var out model.PersistableList - currentState := new(miner.State) - if err := currentState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - if change.Change == core.ChangeTypeAdd { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - if change.Change == core.ChangeTypeModify { - previousState := new(miner.State) - if err := previousState.UnmarshalCBOR(bytes.NewReader(change.Previous)); err != nil { - return nil, err - } - if !currentState.FeeDebt.Equals(previousState.FeeDebt) { - out = append(out, &minermodel.MinerFeeDebt{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - FeeDebt: currentState.FeeDebt.String(), - }) - } - if !currentState.LockedFunds.Equals(previousState.LockedFunds) || - !currentState.InitialPledge.Equals(previousState.InitialPledge) || - !currentState.PreCommitDeposits.Equals(previousState.PreCommitDeposits) { - out = append(out, &minermodel.MinerLockedFund{ - Height: int64(current.Height()), - MinerID: addr.String(), - StateRoot: current.ParentState().String(), - LockedFunds: currentState.LockedFunds.String(), - InitialPledge: currentState.InitialPledge.String(), - PreCommitDeposits: currentState.PreCommitDeposits.String(), - }) - } - } - return out, nil -} diff --git a/pkg/transform/timescale/actors/power/router.go b/pkg/transform/timescale/actors/power/router.go index c22343a4d..1e2322cf3 100644 --- a/pkg/transform/timescale/actors/power/router.go +++ b/pkg/transform/timescale/actors/power/router.go @@ -4,27 +4,27 @@ import ( "context" "fmt" - "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + visormodel "github.com/filecoin-project/lily/model/visor" v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1" - v1_0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v0" - v1_2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v2" - v1_3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v3" - v1_4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v4" - v1_5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v5" - v1_6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v6" - v1_7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v7" - v1_8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v8" - v1_9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power/v1/v9" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { +func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.PowerActorClaim, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } switch version { case actortypes.Version0, actortypes.Version2, @@ -35,34 +35,7 @@ func TransformPowerState(ctx context.Context, s store.Store, version actortypes. actortypes.Version7, actortypes.Version8, actortypes.Version9: - return v1.TransformPowerState(ctx, s, version, current, executed, root) + return v1.TransformPowerState(ctx, s, version, current, executed, *root) } return nil, fmt.Errorf("unsupported version : %d", version) } - -func HandlePower(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { - switch version { - case actortypes.Version0: - return v1_0.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version2: - return v1_2.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version3: - return v1_3.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version4: - return v1_4.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version5: - return v1_5.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version6: - return v1_6.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version7: - return v1_7.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version8: - return v1_8.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version9: - return v1_9.ExtractPowerStateChanges(ctx, current, executed, addr, change) - case actortypes.Version10: - panic("not yet implemented") - default: - return nil, fmt.Errorf("unsupported miner actor version: %d", version) - } -} diff --git a/pkg/transform/timescale/actors/power/v1/router.go b/pkg/transform/timescale/actors/power/v1/router.go index b2b2fcdfb..d030dcc71 100644 --- a/pkg/transform/timescale/actors/power/v1/router.go +++ b/pkg/transform/timescale/actors/power/v1/router.go @@ -23,7 +23,7 @@ import ( ) type Transformer interface { - Transform(ctx context.Context, current, parent *types.TipSet, change *powerdiff.StateDiffResult) (model.Persistable, error) + Transform(ctx context.Context, current, parent *types.TipSet, change *powerdiff.StateDiffResult) model.Persistable } func TransformPowerState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { @@ -41,10 +41,7 @@ func TransformPowerState(ctx context.Context, s store.Store, version actortypes. } out := model.PersistableList{} for _, t := range tramsformers { - m, err := t.Transform(ctx, current, executed, powerStateDiff) - if err != nil { - return nil, err - } + m := t.Transform(ctx, current, executed, powerStateDiff) out = append(out, m) } return out, nil diff --git a/pkg/transform/timescale/actors/power/v1/v0/claims.go b/pkg/transform/timescale/actors/power/v1/v0/claims.go index 6cb1e42d4..fc6a4570f 100644 --- a/pkg/transform/timescale/actors/power/v1/v0/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v0/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v0/state.go b/pkg/transform/timescale/actors/power/v1/v0/state.go deleted file mode 100644 index b3056feb7..000000000 --- a/pkg/transform/timescale/actors/power/v1/v0/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v0 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v2/claims.go b/pkg/transform/timescale/actors/power/v1/v2/claims.go index 8465c4169..6faf16d58 100644 --- a/pkg/transform/timescale/actors/power/v1/v2/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v2/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v2/state.go b/pkg/transform/timescale/actors/power/v1/v2/state.go deleted file mode 100644 index fb61c5f65..000000000 --- a/pkg/transform/timescale/actors/power/v1/v2/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v2 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v2/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v3/claims.go b/pkg/transform/timescale/actors/power/v1/v3/claims.go index fbaf0278a..8506044d4 100644 --- a/pkg/transform/timescale/actors/power/v1/v3/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v3/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v3/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v3/state.go b/pkg/transform/timescale/actors/power/v1/v3/state.go deleted file mode 100644 index d69662fb2..000000000 --- a/pkg/transform/timescale/actors/power/v1/v3/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v3 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v3/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v4/claims.go b/pkg/transform/timescale/actors/power/v1/v4/claims.go index e0f1b5c64..3cbe6f006 100644 --- a/pkg/transform/timescale/actors/power/v1/v4/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v4/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v4/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v4/state.go b/pkg/transform/timescale/actors/power/v1/v4/state.go deleted file mode 100644 index 9fdd86b2f..000000000 --- a/pkg/transform/timescale/actors/power/v1/v4/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v4 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v4/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v5/claims.go b/pkg/transform/timescale/actors/power/v1/v5/claims.go index c33f906fc..a0b95b105 100644 --- a/pkg/transform/timescale/actors/power/v1/v5/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v5/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v5/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v5/state.go b/pkg/transform/timescale/actors/power/v1/v5/state.go deleted file mode 100644 index 7a74a9ae4..000000000 --- a/pkg/transform/timescale/actors/power/v1/v5/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v5 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v5/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v6/claims.go b/pkg/transform/timescale/actors/power/v1/v6/claims.go index 2107ff220..c1333a143 100644 --- a/pkg/transform/timescale/actors/power/v1/v6/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v6/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v6/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v6/state.go b/pkg/transform/timescale/actors/power/v1/v6/state.go deleted file mode 100644 index 763113e9b..000000000 --- a/pkg/transform/timescale/actors/power/v1/v6/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v6 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v6/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v7/claims.go b/pkg/transform/timescale/actors/power/v1/v7/claims.go index 40b2f41e7..ff4d8fbfb 100644 --- a/pkg/transform/timescale/actors/power/v1/v7/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v7/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/specs-actors/v7/actors/builtin/power" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v7/state.go b/pkg/transform/timescale/actors/power/v1/v7/state.go deleted file mode 100644 index 1084caf62..000000000 --- a/pkg/transform/timescale/actors/power/v1/v7/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v7 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - "github.com/filecoin-project/specs-actors/v7/actors/builtin/power" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v8/claims.go b/pkg/transform/timescale/actors/power/v1/v8/claims.go index e4426b0ad..650fbec05 100644 --- a/pkg/transform/timescale/actors/power/v1/v8/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v8/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v8/power" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v8/state.go b/pkg/transform/timescale/actors/power/v1/v8/state.go deleted file mode 100644 index 737b8bc01..000000000 --- a/pkg/transform/timescale/actors/power/v1/v8/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v8 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/builtin/v8/power" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/power/v1/v9/claims.go b/pkg/transform/timescale/actors/power/v1/v9/claims.go index 4784aeda2..678ea9a35 100644 --- a/pkg/transform/timescale/actors/power/v1/v9/claims.go +++ b/pkg/transform/timescale/actors/power/v1/v9/claims.go @@ -8,16 +8,18 @@ import ( "github.com/filecoin-project/go-state-types/builtin/v9/power" "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" powermodel "github.com/filecoin-project/lily/model/actors/power" "github.com/filecoin-project/lily/pkg/core" powerdiff "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) type Claims struct{} -func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) (model.Persistable, error) { - out := make(powermodel.PowerActorClaimList, 0, len(changes.ClaimsChanges)) +func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, changes *powerdiff.StateDiffResult) model.Persistable { + report := data.StartProcessingReport(tasktype.PowerActorClaim, current) for _, change := range changes.ClaimsChanges { // only care about new and modified power entries if change.Change == core.ChangeTypeRemove { @@ -25,13 +27,15 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch } miner, err := address.NewFromBytes(change.Miner) if err != nil { - return nil, err + report.AddError(err) + continue } claim := new(power.Claim) if err := claim.UnmarshalCBOR(bytes.NewReader(change.Current.Raw)); err != nil { - return nil, err + report.AddError(err) + continue } - out = append(out, &powermodel.PowerActorClaim{ + report.AddModels(&powermodel.PowerActorClaim{ Height: int64(current.Height()), MinerID: miner.String(), StateRoot: current.ParentState().String(), @@ -39,5 +43,5 @@ func (Claims) Transform(ctx context.Context, current, executed *types.TipSet, ch QualityAdjPower: claim.QualityAdjPower.String(), }) } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/actors/power/v1/v9/state.go b/pkg/transform/timescale/actors/power/v1/v9/state.go deleted file mode 100644 index 32f82d255..000000000 --- a/pkg/transform/timescale/actors/power/v1/v9/state.go +++ /dev/null @@ -1,38 +0,0 @@ -package v9 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/builtin/v9/power" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - powermodel "github.com/filecoin-project/lily/model/actors/power" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func ExtractPowerStateChanges(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("power actor should never be removed from the state tree") - } - powerState := new(power.State) - if err := powerState.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &powermodel.ChainPower{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - TotalRawBytesPower: powerState.TotalRawBytePower.String(), - TotalQABytesPower: powerState.TotalQualityAdjPower.String(), - TotalRawBytesCommitted: powerState.TotalBytesCommitted.String(), - TotalQABytesCommitted: powerState.TotalQABytesCommitted.String(), - TotalPledgeCollateral: powerState.TotalPledgeCollateral.String(), - QASmoothedPositionEstimate: powerState.ThisEpochQAPowerSmoothed.PositionEstimate.String(), - QASmoothedVelocityEstimate: powerState.ThisEpochQAPowerSmoothed.VelocityEstimate.String(), - MinerCount: uint64(powerState.MinerCount), - ParticipatingMinerCount: uint64(powerState.MinerAboveMinPowerCount), - }, nil -} diff --git a/pkg/transform/timescale/actors/raw/state.go b/pkg/transform/timescale/actors/raw/state.go index e78e8cd3f..ed5e878f7 100644 --- a/pkg/transform/timescale/actors/raw/state.go +++ b/pkg/transform/timescale/actors/raw/state.go @@ -5,36 +5,116 @@ import ( "encoding/json" "github.com/filecoin-project/go-address" + "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/specs-actors/actors/util/adt" + "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/model" commonmodel "github.com/filecoin-project/lily/model/actors/common" + visormodel "github.com/filecoin-project/lily/model/visor" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func RawActorHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - // noop when actor is removed - if change.Change == core.ChangeTypeRemove { - return nil, nil - } +type rawActorStateChange struct { + Address address.Address + StateChange *rawdiff.ActorChange +} - stateDump, err := vm.DumpActorState(util.ActorRegistry, change.Actor, change.Current) +func TransformActorStates(ctx context.Context, s store.Store, current, executed *types.TipSet, actorMapRoot *cid.Cid) (model.Persistable, error) { + if actorMapRoot == nil { + return model.PersistableList{ + data.StartProcessingReport(tasktype.Actor, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.ActorState, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + var out = model.PersistableList{} + actorMap, err := adt.AsMap(s, *actorMapRoot) if err != nil { return nil, err } - state, err := json.Marshal(stateDump) - if err != nil { + var actorChanges []*rawActorStateChange + actorState := new(rawdiff.ActorChange) + if err := actorMap.ForEach(actorState, func(key string) error { + addr, err := address.NewFromBytes([]byte(key)) + if err != nil { + return err + } + val := new(rawdiff.ActorChange) + *val = *actorState + actorChanges = append(actorChanges, &rawActorStateChange{ + Address: addr, + StateChange: val, + }) + return nil + }); err != nil { return nil, err } - return &commonmodel.ActorState{ - Height: int64(current.Height()), - Head: change.Actor.Head.String(), - Code: change.Actor.Code.String(), - State: string(state), - }, nil + out = append(out, ActorStateHandler(ctx, current, executed, actorChanges)) + + out = append(out, ActorHandler(ctx, current, executed, actorChanges)) + + return out, nil +} + +func ActorStateHandler(ctx context.Context, current, executed *types.TipSet, actors []*rawActorStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.ActorState, current) + // noop when actor is removed + for _, actor := range actors { + if actor.StateChange.Change == core.ChangeTypeRemove { + continue + } + + stateDump, err := vm.DumpActorState(util.ActorRegistry, actor.StateChange.Actor, actor.StateChange.Current) + if err != nil { + report.AddError(err) + continue + } + + state, err := json.Marshal(stateDump) + if err != nil { + report.AddError(err) + continue + } + report.AddModels(&commonmodel.ActorState{ + Height: int64(current.Height()), + Head: actor.StateChange.Actor.Head.String(), + Code: actor.StateChange.Actor.Code.String(), + State: string(state), + }) + } + return report.Finish() +} + +func ActorHandler(ctx context.Context, current, executed *types.TipSet, actors []*rawActorStateChange) model.Persistable { + report := data.StartProcessingReport(tasktype.Actor, current) + for _, actor := range actors { + if actor.StateChange.Change == core.ChangeTypeRemove { + continue + } + + report.AddModels(&commonmodel.Actor{ + Height: int64(current.Height()), + ID: actor.Address.String(), + StateRoot: current.ParentState().String(), + Code: actor.StateChange.Actor.Code.String(), + Head: actor.StateChange.Actor.Head.String(), + Balance: actor.StateChange.Actor.Balance.String(), + Nonce: actor.StateChange.Actor.Nonce, + }) + } + return report.Finish() } diff --git a/pkg/transform/timescale/actors/reward/router.go b/pkg/transform/timescale/actors/reward/router.go deleted file mode 100644 index a17a9e2f7..000000000 --- a/pkg/transform/timescale/actors/reward/router.go +++ /dev/null @@ -1,49 +0,0 @@ -package reward - -import ( - "context" - "fmt" - - "github.com/filecoin-project/go-address" - actortypes "github.com/filecoin-project/go-state-types/actors" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" - v0 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v0" - v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v2" - v3 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v3" - v4 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v4" - v5 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v5" - v6 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v6" - v7 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v7" - v8 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v8" - v9 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward/v9" -) - -func HandleReward(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange, version actortypes.Version) (model.Persistable, error) { - switch version { - case actortypes.Version0: - return v0.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version2: - return v2.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version3: - return v3.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version4: - return v4.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version5: - return v5.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version6: - return v6.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version7: - return v7.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version8: - return v8.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version9: - return v9.RewardHandler(ctx, current, executed, addr, change) - case actortypes.Version10: - panic("not yet implemented") - default: - return nil, fmt.Errorf("unsupported reward actor version: %d", version) - } -} diff --git a/pkg/transform/timescale/actors/reward/v0/state.go b/pkg/transform/timescale/actors/reward/v0/state.go deleted file mode 100644 index 1030c7b2a..000000000 --- a/pkg/transform/timescale/actors/reward/v0/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v0 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward0 "github.com/filecoin-project/specs-actors/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward0.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalMined.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v2/state.go b/pkg/transform/timescale/actors/reward/v2/state.go deleted file mode 100644 index fc4bd7968..000000000 --- a/pkg/transform/timescale/actors/reward/v2/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v2 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward2.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v3/state.go b/pkg/transform/timescale/actors/reward/v3/state.go deleted file mode 100644 index ac9de4e7e..000000000 --- a/pkg/transform/timescale/actors/reward/v3/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v3 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward3.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v4/state.go b/pkg/transform/timescale/actors/reward/v4/state.go deleted file mode 100644 index 2240e9d5b..000000000 --- a/pkg/transform/timescale/actors/reward/v4/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v4 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward4 "github.com/filecoin-project/specs-actors/v4/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward4.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v5/state.go b/pkg/transform/timescale/actors/reward/v5/state.go deleted file mode 100644 index 5d53eefc3..000000000 --- a/pkg/transform/timescale/actors/reward/v5/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v5 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward5.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v6/state.go b/pkg/transform/timescale/actors/reward/v6/state.go deleted file mode 100644 index 2c8b693b3..000000000 --- a/pkg/transform/timescale/actors/reward/v6/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v6 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward6 "github.com/filecoin-project/specs-actors/v6/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward6.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v7/state.go b/pkg/transform/timescale/actors/reward/v7/state.go deleted file mode 100644 index 4573d90b7..000000000 --- a/pkg/transform/timescale/actors/reward/v7/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v7 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - "github.com/filecoin-project/lotus/chain/types" - reward7 "github.com/filecoin-project/specs-actors/v7/actors/builtin/reward" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward7.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v8/state.go b/pkg/transform/timescale/actors/reward/v8/state.go deleted file mode 100644 index c928f5c14..000000000 --- a/pkg/transform/timescale/actors/reward/v8/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v8 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - reward8 "github.com/filecoin-project/go-state-types/builtin/v8/reward" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward8.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/reward/v9/state.go b/pkg/transform/timescale/actors/reward/v9/state.go deleted file mode 100644 index 0d0d8990f..000000000 --- a/pkg/transform/timescale/actors/reward/v9/state.go +++ /dev/null @@ -1,39 +0,0 @@ -package v9 - -import ( - "bytes" - "context" - - "github.com/filecoin-project/go-address" - reward9 "github.com/filecoin-project/go-state-types/builtin/v9/reward" - "github.com/filecoin-project/lotus/chain/types" - - "github.com/filecoin-project/lily/model" - rewardmodel "github.com/filecoin-project/lily/model/actors/reward" - "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" -) - -func RewardHandler(ctx context.Context, current, executed *types.TipSet, addr address.Address, change *rawdiff.ActorChange) (model.Persistable, error) { - if change.Change == core.ChangeTypeRemove { - panic("reward is a singleton actor and cannot be removed") - } - state := new(reward9.State) - if err := state.UnmarshalCBOR(bytes.NewReader(change.Current)); err != nil { - return nil, err - } - return &rewardmodel.ChainReward{ - Height: int64(current.Height()), - StateRoot: current.ParentState().String(), - CumSumBaseline: state.CumsumBaseline.String(), - CumSumRealized: state.CumsumRealized.String(), - EffectiveBaselinePower: state.EffectiveBaselinePower.String(), - NewBaselinePower: state.ThisEpochBaselinePower.String(), - NewRewardSmoothedPositionEstimate: state.ThisEpochRewardSmoothed.PositionEstimate.String(), - NewRewardSmoothedVelocityEstimate: state.ThisEpochRewardSmoothed.VelocityEstimate.String(), - TotalMinedReward: state.TotalStoragePowerReward.String(), - NewReward: state.ThisEpochReward.String(), - EffectiveNetworkTime: int64(state.EffectiveNetworkTime), - }, nil - -} diff --git a/pkg/transform/timescale/actors/verifreg/router.go b/pkg/transform/timescale/actors/verifreg/router.go index fc4c4e158..2d3f782b8 100644 --- a/pkg/transform/timescale/actors/verifreg/router.go +++ b/pkg/transform/timescale/actors/verifreg/router.go @@ -9,12 +9,43 @@ import ( "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" v1 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v1" v2 "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg/v2" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root cid.Cid) (model.Persistable, error) { +func TransformVerifregState(ctx context.Context, s store.Store, version actortypes.Version, current, executed *types.TipSet, root *cid.Cid) (model.Persistable, error) { + if root == nil { + if version < actortypes.Version9 { + return model.PersistableList{ + data.StartProcessingReport(tasktype.VerifiedRegistryVerifiedClient, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.VerifiedRegistryVerifier, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } + return model.PersistableList{ + data.StartProcessingReport("verified_registry_claims", current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport("verified_registry_allocation", current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + data.StartProcessingReport(tasktype.VerifiedRegistryVerifier, current). + WithStatus(visormodel.ProcessingStatusInfo). + WithInformation("no change detected"). + Finish(), + }, nil + } switch version { case actortypes.Version0, actortypes.Version2, @@ -24,10 +55,10 @@ func TransformVerifregState(ctx context.Context, s store.Store, version actortyp actortypes.Version6, actortypes.Version7, actortypes.Version8: - return v1.TransformVerifregState(ctx, s, version, current, executed, root) + return v1.TransformVerifregState(ctx, s, version, current, executed, *root) case actortypes.Version9: - return v2.TransformVerifregState(ctx, s, version, current, executed, root) + return v2.TransformVerifregState(ctx, s, version, current, executed, *root) } return nil, fmt.Errorf("unsupported version : %d", version) diff --git a/pkg/transform/timescale/data/report.go b/pkg/transform/timescale/data/report.go new file mode 100644 index 000000000..21178c144 --- /dev/null +++ b/pkg/transform/timescale/data/report.go @@ -0,0 +1,64 @@ +package data + +import ( + "time" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/model" + visormodel "github.com/filecoin-project/lily/model/visor" +) + +type ReportError struct { + Error string +} + +type ProcessingReportBuilder struct { + errors []*ReportError + models model.PersistableList + report *visormodel.ProcessingReport +} + +func (b *ProcessingReportBuilder) WithStatus(status string) *ProcessingReportBuilder { + b.report.Status = status + return b +} + +func (b *ProcessingReportBuilder) WithInformation(info string) *ProcessingReportBuilder { + b.report.StatusInformation = info + return b +} + +func (b *ProcessingReportBuilder) AddError(err error) *ProcessingReportBuilder { + b.errors = append(b.errors, &ReportError{Error: err.Error()}) + return b +} + +func (b *ProcessingReportBuilder) AddModels(m ...model.Persistable) *ProcessingReportBuilder { + b.models = append(b.models, m...) + return b +} + +func (b *ProcessingReportBuilder) Finish() model.PersistableList { + b.report.CompletedAt = time.Now() + if len(b.errors) == 0 { + if b.report.Status == "" { + b.WithStatus(visormodel.ProcessingStatusOK) + } + } else { + b.WithStatus(visormodel.ProcessingStatusError) + } + return model.PersistableList{b.models, b.report} +} + +func StartProcessingReport(task string, ts *types.TipSet) *ProcessingReportBuilder { + return &ProcessingReportBuilder{ + report: &visormodel.ProcessingReport{ + Height: int64(ts.Height()), + StateRoot: ts.ParentState().String(), + Reporter: "Deprecate", + Task: task, + StartedAt: time.Now(), + }, + } +} diff --git a/pkg/transform/timescale/fullblock/blocks.go b/pkg/transform/timescale/fullblock/blocks.go index 83c98171a..f8cd1ae21 100644 --- a/pkg/transform/timescale/fullblock/blocks.go +++ b/pkg/transform/timescale/fullblock/blocks.go @@ -3,18 +3,33 @@ package fullblock import ( "context" + "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/model/blocks" messagemodel "github.com/filecoin-project/lily/model/messages" "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { - out := blocks.BlockHeaders{} +func mustMakeTipsetFromFullBlocks(fullBlocks map[cid.Cid]*chain.FullBlock) *types.TipSet { + var header []*types.BlockHeader for _, fb := range fullBlocks { - out = append(out, &blocks.BlockHeader{ + header = append(header, fb.Block) + } + ts, err := types.NewTipSet(header) + if err != nil { + panic(err) + } + return ts +} + +func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockHeader, mustMakeTipsetFromFullBlocks(fullBlocks)) + for _, fb := range fullBlocks { + report.AddModels(&blocks.BlockHeader{ Height: int64(fb.Block.Height), Cid: fb.Block.Cid().String(), Miner: fb.Block.Miner.String(), @@ -26,40 +41,40 @@ func ExtractBlockHeaders(ctx context.Context, fullBlocks map[cid.Cid]*chain.Full ForkSignaling: fb.Block.ForkSignaling, }) } - return out, nil + return report.Finish() } -func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { - out := blocks.BlockParents{} +func ExtractBlockParents(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockParent, mustMakeTipsetFromFullBlocks(fullBlocks)) for _, fb := range fullBlocks { for _, p := range fb.Block.Parents { - out = append(out, &blocks.BlockParent{ + report.AddModels(&blocks.BlockParent{ Height: int64(fb.Block.Height), Block: fb.Block.Cid().String(), Parent: p.String(), }) } } - return out, nil + return report.Finish() } -func ExtractBlockMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { - out := messagemodel.BlockMessages{} +func ExtractBlockMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.BlockMessage, mustMakeTipsetFromFullBlocks(fullBlocks)) for _, fb := range fullBlocks { for _, msg := range fb.BlsMessages { - out = append(out, &messagemodel.BlockMessage{ + report.AddModels(&messagemodel.BlockMessage{ Height: int64(fb.Block.Height), Block: fb.Block.Cid().String(), Message: msg.Message.Cid().String(), }) } for _, msg := range fb.SecpMessages { - out = append(out, &messagemodel.BlockMessage{ + report.AddModels(&messagemodel.BlockMessage{ Height: int64(fb.Block.Height), Block: fb.Block.Cid().String(), Message: msg.Message.Cid().String(), }) } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/fullblock/messages.go b/pkg/transform/timescale/fullblock/messages.go index e071e9737..b5f263547 100644 --- a/pkg/transform/timescale/fullblock/messages.go +++ b/pkg/transform/timescale/fullblock/messages.go @@ -5,16 +5,18 @@ import ( "github.com/ipfs/go-cid" + "github.com/filecoin-project/lily/chain/indexer/tasktype" "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/model/messages" "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/timescale/data" ) -func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { - out := messages.Messages{} +func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.Message, mustMakeTipsetFromFullBlocks(fullBlocks)) for _, fb := range fullBlocks { for _, msg := range fb.SecpMessages { - out = append(out, &messages.Message{ + report.AddModels(&messages.Message{ Height: int64(fb.Block.Height), Cid: msg.Message.Cid().String(), From: msg.Message.Message.From.String(), @@ -29,7 +31,7 @@ func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBloc }) } for _, msg := range fb.BlsMessages { - out = append(out, &messages.Message{ + report.AddModels(&messages.Message{ Height: int64(fb.Block.Height), Cid: msg.Message.Cid().String(), From: msg.Message.From.String(), @@ -44,15 +46,15 @@ func ExtractMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBloc }) } } - return out, nil + return report.Finish() } -func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) (model.Persistable, error) { - out := messages.VMMessageList{} +func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) model.Persistable { + report := data.StartProcessingReport(tasktype.VMMessage, mustMakeTipsetFromFullBlocks(fullBlocks)) for _, fb := range fullBlocks { for _, msg := range fb.SecpMessages { for _, vm := range msg.VmMessages { - out = append(out, &messages.VMMessage{ + report.AddModels(&messages.VMMessage{ Height: int64(fb.Block.Height), StateRoot: fb.Block.ParentStateRoot.String(), Cid: vm.Message.Cid().String(), @@ -61,17 +63,17 @@ func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBl To: vm.Message.To.String(), Value: vm.Message.Value.String(), Method: uint64(vm.Message.Method), - ActorCode: "TODO", + ActorCode: "Deprecate", ExitCode: int64(vm.Receipt.ExitCode), GasUsed: vm.Receipt.GasUsed, - Params: "", //vm.Message.Params - Returns: "", //vm.Receipt.Return + Params: "TODO make bytes", //vm.Message.Params + Returns: "TODO make bytes", //vm.Receipt.Return }) } } for _, msg := range fb.BlsMessages { for _, vm := range msg.VmMessages { - out = append(out, &messages.VMMessage{ + report.AddModels(&messages.VMMessage{ Height: int64(fb.Block.Height), StateRoot: fb.Block.ParentStateRoot.String(), Cid: vm.Message.Cid().String(), @@ -80,14 +82,14 @@ func ExtractVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBl To: vm.Message.To.String(), Value: vm.Message.Value.String(), Method: uint64(vm.Message.Method), - ActorCode: "TODO", + ActorCode: "Deprecate", ExitCode: int64(vm.Receipt.ExitCode), GasUsed: vm.Receipt.GasUsed, - Params: "", //vm.Message.Params - Returns: "", //vm.Receipt.Return + Params: "TODO make bytes", //vm.Message.Params + Returns: "TODO make bytes", //vm.Receipt.Return }) } } } - return out, nil + return report.Finish() } diff --git a/pkg/transform/timescale/processor.go b/pkg/transform/timescale/processor.go index f936ca786..9b612c278 100644 --- a/pkg/transform/timescale/processor.go +++ b/pkg/transform/timescale/processor.go @@ -5,10 +5,8 @@ import ( "fmt" "io" - "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" actorstypes "github.com/filecoin-project/go-state-types/actors" - "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/blockstore" @@ -20,7 +18,6 @@ import ( "github.com/filecoin-project/lily/model" "github.com/filecoin-project/lily/pkg/core" - "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/actors" "github.com/filecoin-project/lily/pkg/transform/cbor/messages" @@ -29,7 +26,6 @@ import ( "github.com/filecoin-project/lily/pkg/transform/timescale/actors/miner" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/power" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/raw" - "github.com/filecoin-project/lily/pkg/transform/timescale/actors/reward" "github.com/filecoin-project/lily/pkg/transform/timescale/actors/verifreg" "github.com/filecoin-project/lily/pkg/transform/timescale/fullblock" ) @@ -92,64 +88,49 @@ func Process(ctx context.Context, r io.Reader, strg model.Storage, nvg NetworkVe return strg.PersistBatch(ctx, toStorage) } -func ProcessInitAddresses(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { - return init_.InitHandler(ctx, s, current, executed, root) -} - func HandleActorStateChanges(ctx context.Context, s store.Store, current, parent *types.TipSet, av actorstypes.Version, root cid.Cid) (model.Persistable, error) { actorIPLDContainer := new(actors.ActorStateChangesIPLD) if err := s.Get(ctx, root, actorIPLDContainer); err != nil { return nil, err } log.Infow("open actor state changes", zap.Inline(actorIPLDContainer)) + out := model.PersistableList{} - if actorIPLDContainer.MarketActor != nil { - marketModels, err := market.TransformMarketState(ctx, s, av, current, parent, *actorIPLDContainer.MarketActor) - if err != nil { - return nil, err - } - out = append(out, marketModels) + marketModels, err := market.TransformMarketState(ctx, s, av, current, parent, actorIPLDContainer.MarketActor) + if err != nil { + return nil, err } + out = append(out, marketModels) - if actorIPLDContainer.MinerActors != nil { - minerModels, err := miner.TransformMinerState(ctx, s, av, current, parent, *actorIPLDContainer.MinerActors) - if err != nil { - return nil, err - } - out = append(out, minerModels) + minerModels, err := miner.TransformMinerState(ctx, s, av, current, parent, actorIPLDContainer.MinerActors) + if err != nil { + return nil, err } + out = append(out, minerModels) - if actorIPLDContainer.InitActor != nil { - initModels, err := ProcessInitAddresses(ctx, s, current, parent, av, *actorIPLDContainer.InitActor) - if err != nil { - return nil, err - } - out = append(out, initModels) + powerModels, err := power.TransformPowerState(ctx, s, av, current, parent, actorIPLDContainer.PowerActor) + if err != nil { + return nil, err } + out = append(out, powerModels) - if actorIPLDContainer.RawActors != nil { - rawModels, err := ProcessActorStates(ctx, s, current, parent, av, *actorIPLDContainer.RawActors) - if err != nil { - return nil, err - } - out = append(out, rawModels) + initModels, err := init_.TransformInitState(ctx, s, current, parent, actorIPLDContainer.InitActor) + if err != nil { + return nil, err } + out = append(out, initModels) - if actorIPLDContainer.VerifregActor != nil { - verifregModels, err := verifreg.TransformVerifregState(ctx, s, av, current, parent, *actorIPLDContainer.MarketActor) - if err != nil { - return nil, err - } - out = append(out, verifregModels) + verifregModels, err := verifreg.TransformVerifregState(ctx, s, av, current, parent, actorIPLDContainer.VerifregActor) + if err != nil { + return nil, err } + out = append(out, verifregModels) - if actorIPLDContainer.PowerActor != nil { - powerModels, err := power.TransformPowerState(ctx, s, av, current, parent, *actorIPLDContainer.PowerActor) - if err != nil { - return nil, err - } - out = append(out, powerModels) + rawModels, err := raw.TransformActorStates(ctx, s, current, parent, actorIPLDContainer.RawActors) + if err != nil { + return nil, err } + out = append(out, rawModels) return out, nil } @@ -160,26 +141,11 @@ func HandleFullBlocks(ctx context.Context, s store.Store, current, parent *types if err != nil { return nil, err } - bh, err := fullblock.ExtractBlockHeaders(ctx, fullBlockMap) - if err != nil { - return nil, err - } - out = append(out, bh) - bp, err := fullblock.ExtractBlockParents(ctx, fullBlockMap) - if err != nil { - return nil, err - } - out = append(out, bp) - msgs, err := fullblock.ExtractMessages(ctx, fullBlockMap) - if err != nil { - return nil, err - } - out = append(out, msgs) - vm, err := fullblock.ExtractVmMessages(ctx, fullBlockMap) - if err != nil { - return nil, err - } - out = append(out, vm) + out = append(out, fullblock.ExtractBlockHeaders(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractBlockParents(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractBlockMessages(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractMessages(ctx, fullBlockMap)) + out = append(out, fullblock.ExtractVmMessages(ctx, fullBlockMap)) return out, nil } @@ -192,55 +158,3 @@ func HandleImplicitMessages(ctx context.Context, s store.Store, current, parent _ = implicitMessages return nil, nil } - -func ProcessActorStates(ctx context.Context, s store.Store, current, executed *types.TipSet, av actorstypes.Version, actorMapRoot cid.Cid) (model.Persistable, error) { - var out = model.PersistableList{} - actorMap, err := adt.AsMap(s, actorMapRoot, 5) - if err != nil { - return nil, err - } - actorState := new(rawdiff.ActorChange) - if err := actorMap.ForEach(actorState, func(key string) error { - addr, err := address.NewFromBytes([]byte(key)) - if err != nil { - return err - } - - m, err := raw.RawActorHandler(ctx, current, executed, addr, actorState) - if err != nil { - return err - } - if m != nil { - out = append(out, m) - } - - if core.RewardCodes.Has(actorState.Actor.Code) { - m, err := reward.HandleReward(ctx, current, executed, addr, actorState, av) - if err != nil { - return err - } - out = append(out, m) - } - - if core.MinerCodes.Has(actorState.Actor.Code) { - m, err := miner.HandleMiner(ctx, current, executed, addr, actorState, av) - if err != nil { - return err - } - out = append(out, m) - } - - if core.PowerCodes.Has(actorState.Actor.Code) { - m, err := power.HandlePower(ctx, current, executed, addr, actorState, av) - if err != nil { - return err - } - out = append(out, m) - } - return nil - - }); err != nil { - return nil, err - } - return out, nil -} From 6d13e5bb4161d1ca92e4eac38abcdf94785b5770 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 26 Jan 2023 21:02:44 -0800 Subject: [PATCH 40/48] start new schema add lambda example for parsed messages --- lens/lily/impl.go | 18 ++ pkg/transform/gorm/lambda/params.go | 42 ++++ pkg/transform/gorm/models/blockheader.go | 18 ++ pkg/transform/gorm/models/chainmessage.go | 25 +++ pkg/transform/gorm/models/receipt.go | 26 +++ pkg/transform/gorm/models/vmmessages.go | 15 ++ pkg/transform/gorm/run.go | 251 ++++++++++++++++++++++ pkg/transform/gorm/types/address.go | 33 +++ pkg/transform/gorm/types/bigint.go | 35 +++ pkg/transform/gorm/types/cid.go | 57 +++++ pkg/transform/gorm/types/json.go | 28 +++ pkg/transform/gorm/types/token.go | 48 +++++ shed/newschema/main.go | 209 ++++++++++-------- 13 files changed, 714 insertions(+), 91 deletions(-) create mode 100644 pkg/transform/gorm/lambda/params.go create mode 100644 pkg/transform/gorm/models/blockheader.go create mode 100644 pkg/transform/gorm/models/chainmessage.go create mode 100644 pkg/transform/gorm/models/receipt.go create mode 100644 pkg/transform/gorm/models/vmmessages.go create mode 100644 pkg/transform/gorm/run.go create mode 100644 pkg/transform/gorm/types/address.go create mode 100644 pkg/transform/gorm/types/bigint.go create mode 100644 pkg/transform/gorm/types/cid.go create mode 100644 pkg/transform/gorm/types/json.go create mode 100644 pkg/transform/gorm/types/token.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 9f0a58900..1305a0dc0 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -28,6 +28,9 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" + "gorm.io/driver/postgres" + "gorm.io/gorm" + "gorm.io/gorm/schema" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -45,6 +48,7 @@ import ( "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract" "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/gorm/lambda" "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" @@ -153,6 +157,20 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return false, err + } + if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { + return false, err + } + currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) if err != nil { return false, err diff --git a/pkg/transform/gorm/lambda/params.go b/pkg/transform/gorm/lambda/params.go new file mode 100644 index 000000000..7e3914b32 --- /dev/null +++ b/pkg/transform/gorm/lambda/params.go @@ -0,0 +1,42 @@ +package lambda + +import ( + "context" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/lotus/chain/types" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/lens/util" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + "github.com/filecoin-project/lily/tasks" +) + +func ParseParams(ctx context.Context, api tasks.DataSource, db *gorm.DB) error { + var messages []models.Message + res := db.Find(&messages) + if res.Error != nil { + return res.Error + } + out := make([]models.ParsedMessageParams, 0, len(messages)) + for _, msg := range messages { + act, err := api.Actor(ctx, msg.To.Addr, types.EmptyTSK) + if err != nil { + return err + } + params, method, err := util.ParseParams(msg.Params, abi.MethodNum(msg.Method), act.Code) + if err != nil { + // TODO could continue + return err + } + out = append(out, models.ParsedMessageParams{ + Cid: msg.Cid, + Params: params, + Method: method, + }) + } + if err := db.Create(out).Error; err != nil { + return err + } + return nil +} diff --git a/pkg/transform/gorm/models/blockheader.go b/pkg/transform/gorm/models/blockheader.go new file mode 100644 index 000000000..e13411108 --- /dev/null +++ b/pkg/transform/gorm/models/blockheader.go @@ -0,0 +1,18 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type BlockHeaderModel struct { + Cid types.DbCID `gorm:"primaryKey"` + StateRoot types.DbCID + Height int64 + Miner types.DbAddr + ParentWeight types.DbBigInt + TimeStamp uint64 + ForkSignaling uint64 + BaseFee types.DbToken + WinCount int64 + ParentBaseFee types.DbToken +} diff --git a/pkg/transform/gorm/models/chainmessage.go b/pkg/transform/gorm/models/chainmessage.go new file mode 100644 index 000000000..bc39954dc --- /dev/null +++ b/pkg/transform/gorm/models/chainmessage.go @@ -0,0 +1,25 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type Message struct { + Cid types.DbCID `gorm:"primaryKey"` + Version int64 + To types.DbAddr + From types.DbAddr + Nonce uint64 + Value types.DbToken + GasLimit int64 + GasFeeCap types.DbToken + GasPremium types.DbToken + Method uint64 + Params []byte +} + +type ParsedMessageParams struct { + Cid types.DbCID `gorm:"primaryKey"` + Params string `gorm:"jsonb"` + Method string +} diff --git a/pkg/transform/gorm/models/receipt.go b/pkg/transform/gorm/models/receipt.go new file mode 100644 index 000000000..827fa22dd --- /dev/null +++ b/pkg/transform/gorm/models/receipt.go @@ -0,0 +1,26 @@ +package models + +import ( + types "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +type MessageReceipt struct { + MessageCid types.DbCID `gorm:"primaryKey"` + Receipt Receipt `gorm:"embedded"` + + BaseFeeBurn types.DbToken + OverEstimationBurn types.DbToken + MinerPenalty types.DbToken + MinerTip types.DbToken + Refund types.DbToken + GasRefund int64 + GasBurned int64 + Error string +} + +type Receipt struct { + Index int64 `gorm:"primaryKey"` + ExitCode int64 + GasUsed int64 + Return []byte +} diff --git a/pkg/transform/gorm/models/vmmessages.go b/pkg/transform/gorm/models/vmmessages.go new file mode 100644 index 000000000..2b44f7edc --- /dev/null +++ b/pkg/transform/gorm/models/vmmessages.go @@ -0,0 +1,15 @@ +package models + +import "github.com/filecoin-project/lily/pkg/transform/gorm/types" + +type VmMessage struct { + Source types.DbCID `gorm:"primaryKey"` + Cid types.DbCID `gorm:"primaryKey"` + To types.DbAddr + From types.DbAddr + Value types.DbToken + Method uint64 + Params []byte + Receipt Receipt `gorm:"embedded"` + Error string +} diff --git a/pkg/transform/gorm/run.go b/pkg/transform/gorm/run.go new file mode 100644 index 000000000..516512f37 --- /dev/null +++ b/pkg/transform/gorm/run.go @@ -0,0 +1,251 @@ +package gorm + +import ( + "context" + "fmt" + "io" + + "github.com/filecoin-project/go-state-types/store" + "github.com/filecoin-project/lotus/blockstore" + "github.com/filecoin-project/lotus/chain/types" + cid "github.com/ipfs/go-cid" + v1car "github.com/ipld/go-car" + "gorm.io/gorm" + + "github.com/filecoin-project/lily/pkg/extract/chain" + "github.com/filecoin-project/lily/pkg/transform/cbor" + "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + dbtypes "github.com/filecoin-project/lily/pkg/transform/gorm/types" +) + +func Run(ctx context.Context, r io.Reader, db *gorm.DB) error { + // create a blockstore and load the contents of the car file (reader is a pointer to the car file) into it. + bs := blockstore.NewMemorySync() + header, err := v1car.LoadCar(ctx, bs, r) + if err != nil { + return err + } + // expect to have a single root (cid) pointing to content + if len(header.Roots) != 1 { + return fmt.Errorf("invalid header expected 1 root got %d", len(header.Roots)) + } + + // we need to wrap the blockstore to meet some dumb interface. + s := store.WrapBlockStore(ctx, bs) + + // load the root container, this contains netwwork metadata and the root (cid) of the extracted state. + rootIPLDContainer := new(cbor.RootStateIPLD) + if err := s.Get(ctx, header.Roots[0], rootIPLDContainer); err != nil { + return err + } + + // load the extracted state, this contains links to fullblocks, implicit message, network economics, and actor states + stateExtractionIPLDContainer := new(cbor.StateExtractionIPLD) + if err := s.Get(ctx, rootIPLDContainer.State, stateExtractionIPLDContainer); err != nil { + return err + } + + // ohh and it also contains the tipset the extraction was performed over. + current := &stateExtractionIPLDContainer.Current + parent := &stateExtractionIPLDContainer.Parent + + // for now we will only handle the fullblock dag. + if err := HandleFullBlocks(ctx, db, s, current, parent, stateExtractionIPLDContainer.FullBlocks); err != nil { + return err + } + + return nil +} + +func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, parent *types.TipSet, root cid.Cid) error { + // decode the content at the root (cid) into a concrete type we can inspect, a map of block CID to the block, its messages, their receipts and vm messages. + fullBlocks, err := messages.DecodeFullBlockHAMT(ctx, s, root) + if err != nil { + return err + } + // migrate the database, this creates new tables or updates existing ones with new fields + if err := db.AutoMigrate(&models.BlockHeaderModel{}, &models.Message{}, models.MessageReceipt{}, models.VmMessage{}); err != nil { + return err + } + // make some blockheaders and plop em in the database + if err := db.Create(MakeBlockHeaderModels(ctx, fullBlocks)).Error; err != nil { + return err + } + // now do messages + if err := db.Create(MakeMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + // finally receipts + if err := db.Create(MakeReceipts(ctx, fullBlocks)).Error; err != nil { + return err + } + + if err := db.Create(MakeVmMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + + // okay all done. + return nil +} + +func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.BlockHeaderModel { + out := make([]*models.BlockHeaderModel, 0, len(fullBlocks)) + for _, fb := range fullBlocks { + out = append(out, &models.BlockHeaderModel{ + Cid: dbtypes.DbCID{CID: fb.Block.Cid()}, + StateRoot: dbtypes.DbCID{CID: fb.Block.ParentStateRoot}, + Height: int64(fb.Block.Height), + Miner: dbtypes.DbAddr{Addr: fb.Block.Miner}, + ParentWeight: dbtypes.DbBigInt{BigInt: fb.Block.ParentWeight}, + TimeStamp: fb.Block.Timestamp, + ForkSignaling: fb.Block.ForkSignaling, + BaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + WinCount: fb.Block.ElectionProof.WinCount, + ParentBaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, + }) + } + return out +} + +func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.Message { + // messages can be contained in more than 1 block, this is used to prevent persisting them twice + seen := cid.NewSet() + var out []*models.Message + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + if !seen.Visit(smsg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Version: int64(smsg.Message.Message.Version), + To: dbtypes.DbAddr{Addr: smsg.Message.Message.To}, + From: dbtypes.DbAddr{smsg.Message.Message.From}, + Nonce: smsg.Message.Message.Nonce, + Value: dbtypes.DbToken{Token: smsg.Message.Message.Value}, + GasLimit: smsg.Message.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: smsg.Message.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: smsg.Message.Message.GasPremium}, + Method: uint64(smsg.Message.Message.Method), + Params: smsg.Message.Message.Params, + }) + } + for _, msg := range fb.BlsMessages { + if !seen.Visit(msg.Message.Cid()) { + continue + } + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Version: int64(msg.Message.Version), + To: dbtypes.DbAddr{Addr: msg.Message.To}, + From: dbtypes.DbAddr{msg.Message.From}, + Nonce: msg.Message.Nonce, + Value: dbtypes.DbToken{Token: msg.Message.Value}, + GasLimit: msg.Message.GasLimit, + GasFeeCap: dbtypes.DbToken{Token: msg.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: msg.Message.GasPremium}, + Method: uint64(msg.Message.Method), + Params: msg.Message.Params, + }) + } + } + return out +} + +func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.MessageReceipt { + var out []*models.MessageReceipt + for _, fb := range fullBlocks { + for _, smsg := range fb.SecpMessages { + // TODO this is buggy + if smsg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Receipt: models.Receipt{ + Index: smsg.Receipt.Index, + ExitCode: int64(smsg.Receipt.Receipt.ExitCode), + GasUsed: smsg.Receipt.Receipt.GasUsed, + Return: smsg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.Refund}, + GasRefund: smsg.Receipt.GasOutputs.GasRefund, + GasBurned: smsg.Receipt.GasOutputs.GasBurned, + }) + } + for _, msg := range fb.BlsMessages { + if msg.Receipt.GasOutputs == nil { + continue + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Receipt: models.Receipt{ + Index: msg.Receipt.Index, + ExitCode: int64(msg.Receipt.Receipt.ExitCode), + GasUsed: msg.Receipt.Receipt.GasUsed, + Return: msg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.Refund}, + GasRefund: msg.Receipt.GasOutputs.GasRefund, + GasBurned: msg.Receipt.GasOutputs.GasBurned, + }) + } + } + return out +} + +func MakeVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.VmMessage { + var out []*models.VmMessage + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + } + return out +} diff --git a/pkg/transform/gorm/types/address.go b/pkg/transform/gorm/types/address.go new file mode 100644 index 000000000..858933bb2 --- /dev/null +++ b/pkg/transform/gorm/types/address.go @@ -0,0 +1,33 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-address" +) + +//copied from https://github.com/application-research/estuary/blob/f740b018af1d7ef6eaafb51d1c7b18a2bda0b589/util/database.go#L63 + +type DbAddr struct { + Addr address.Address +} + +func (dba *DbAddr) Scan(v interface{}) error { + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbAddrs must be strings") + } + + addr, err := address.NewFromString(s) + if err != nil { + return err + } + + dba.Addr = addr + return nil +} + +func (dba DbAddr) Value() (driver.Value, error) { + return dba.Addr.String(), nil +} diff --git a/pkg/transform/gorm/types/bigint.go b/pkg/transform/gorm/types/bigint.go new file mode 100644 index 000000000..bd7915a23 --- /dev/null +++ b/pkg/transform/gorm/types/bigint.go @@ -0,0 +1,35 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-state-types/big" +) + +type DbBigInt struct { + BigInt big.Int +} + +func (dbb *DbBigInt) Scan(v interface{}) error { + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbBigInts must be strings") + } + + bi, err := big.FromString(s) + if err != nil { + return err + } + + dbb.BigInt = bi + return nil +} + +func (dbb DbBigInt) Value() (driver.Value, error) { + return dbb.BigInt.String(), nil +} + +func (DbBigInt) GormDataType() string { + return "bigint" +} diff --git a/pkg/transform/gorm/types/cid.go b/pkg/transform/gorm/types/cid.go new file mode 100644 index 000000000..649945174 --- /dev/null +++ b/pkg/transform/gorm/types/cid.go @@ -0,0 +1,57 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "fmt" + + "github.com/ipfs/go-cid" +) + +// copied from https://github.com/application-research/estuary/blob/f740b018af1d7ef6eaafb51d1c7b18a2bda0b589/util/database.go#L63 + +type DbCID struct { + CID cid.Cid +} + +func (dbc *DbCID) Scan(v interface{}) error { + b, ok := v.(string) + if !ok { + return fmt.Errorf("dbcids must be strings") + } + + if len(b) == 0 { + return nil + } + + c, err := cid.Decode(b) + if err != nil { + return err + } + + dbc.CID = c + return nil +} + +func (dbc DbCID) Value() (driver.Value, error) { + return dbc.CID.String(), nil +} + +func (dbc DbCID) MarshalJSON() ([]byte, error) { + return json.Marshal(dbc.CID.String()) +} + +func (dbc *DbCID) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + c, err := cid.Decode(s) + if err != nil { + return err + } + + dbc.CID = c + return nil +} diff --git a/pkg/transform/gorm/types/json.go b/pkg/transform/gorm/types/json.go new file mode 100644 index 000000000..1853acfa4 --- /dev/null +++ b/pkg/transform/gorm/types/json.go @@ -0,0 +1,28 @@ +package types + +import ( + "database/sql/driver" + "encoding/json" + "errors" +) + +// JSONB Interface for JSONB Field of yourTableName Table +type DbJson string + +// Value Marshal +func (a DbJson) Value() (driver.Value, error) { + return json.Marshal(a) +} + +// Scan Unmarshal +func (a *DbJson) Scan(value interface{}) error { + b, ok := value.([]byte) + if !ok { + return errors.New("type assertion to []byte failed") + } + return json.Unmarshal(b, &a) +} + +func (DbJson) GormDataType() string { + return "jsonb" +} diff --git a/pkg/transform/gorm/types/token.go b/pkg/transform/gorm/types/token.go new file mode 100644 index 000000000..549fb2800 --- /dev/null +++ b/pkg/transform/gorm/types/token.go @@ -0,0 +1,48 @@ +package types + +import ( + "database/sql/driver" + "fmt" + + "github.com/filecoin-project/go-state-types/abi" + "github.com/filecoin-project/go-state-types/big" +) + +type DbToken struct { + Token abi.TokenAmount +} + +func (dbt *DbToken) Scan(v interface{}) error { + switch v.(type) { + case string: + s, ok := v.(string) + if !ok { + return fmt.Errorf("DbToken must be strings") + } + + token, err := big.FromString(s) + if err != nil { + return err + } + dbt.Token = token + return nil + case int64: + s, ok := v.(int64) + if !ok { + return fmt.Errorf("DbToken must be strings") + } + + token := big.NewInt(s) + dbt.Token = token + return nil + } + panic("here") +} + +func (dbt DbToken) Value() (driver.Value, error) { + return dbt.Token.String(), nil +} + +func (DbToken) GormDataType() string { + return "bigint" +} diff --git a/shed/newschema/main.go b/shed/newschema/main.go index 5859b6aa7..f16e496b8 100644 --- a/shed/newschema/main.go +++ b/shed/newschema/main.go @@ -19,6 +19,8 @@ import ( "github.com/filecoin-project/lily/pkg/extract/chain" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/cbor/messages" + "github.com/filecoin-project/lily/pkg/transform/gorm/models" + dbtypes "github.com/filecoin-project/lily/pkg/transform/gorm/types" ) func main() { @@ -117,7 +119,7 @@ func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, return err } // migrate the database, this creates new tables or updates existing ones with new fields - if err := db.AutoMigrate(&BlockHeaderModel{}, &ChainMessage{}, &ChainMessageReceipt{}); err != nil { + if err := db.AutoMigrate(&models.BlockHeaderModel{}, &models.Message{}, models.MessageReceipt{}, models.VmMessage{}, models.ParsedMessageParams{}); err != nil { return err } // make some blockheaders and plop em in the database @@ -133,73 +135,52 @@ func HandleFullBlocks(ctx context.Context, db *gorm.DB, s store.Store, current, return err } + if err := db.Create(MakeVmMessages(ctx, fullBlocks)).Error; err != nil { + return err + } + // okay all done. return nil } -type BlockHeaderModel struct { - Cid string `gorm:"primaryKey"` - StateRoot string - Height int64 - Miner string - ParentWeight string `gorm:"bigint"` - TimeStamp uint64 - ForkSignaling uint64 - BaseFee string `gorm:"bigint"` - WinCount int64 -} - -func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*BlockHeaderModel { - out := make([]*BlockHeaderModel, 0, len(fullBlocks)) +func MakeBlockHeaderModels(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.BlockHeaderModel { + out := make([]*models.BlockHeaderModel, 0, len(fullBlocks)) for _, fb := range fullBlocks { - out = append(out, &BlockHeaderModel{ - Cid: fb.Block.Cid().String(), - StateRoot: fb.Block.ParentStateRoot.String(), + out = append(out, &models.BlockHeaderModel{ + Cid: dbtypes.DbCID{CID: fb.Block.Cid()}, + StateRoot: dbtypes.DbCID{CID: fb.Block.ParentStateRoot}, Height: int64(fb.Block.Height), - Miner: fb.Block.Miner.String(), - ParentWeight: fb.Block.ParentWeight.String(), + Miner: dbtypes.DbAddr{Addr: fb.Block.Miner}, + ParentWeight: dbtypes.DbBigInt{BigInt: fb.Block.ParentWeight}, TimeStamp: fb.Block.Timestamp, ForkSignaling: fb.Block.ForkSignaling, - BaseFee: fb.Block.ParentBaseFee.String(), + BaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, WinCount: fb.Block.ElectionProof.WinCount, + ParentBaseFee: dbtypes.DbToken{Token: fb.Block.ParentBaseFee}, }) } return out } -type ChainMessage struct { - Cid string `gorm:"primaryKey"` - Version int64 - To string - From string - Nonce uint64 - Value string `gorm:"bigint"` - GasLimit int64 - GasFeeCap string `gorm:"bigint"` - GasPremium string `gorm:"bigint"` - Method uint64 - Params []byte -} - -func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*ChainMessage { +func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.Message { // messages can be contained in more than 1 block, this is used to prevent persisting them twice seen := cid.NewSet() - var out []*ChainMessage + var out []*models.Message for _, fb := range fullBlocks { for _, smsg := range fb.SecpMessages { if !seen.Visit(smsg.Message.Cid()) { continue } - out = append(out, &ChainMessage{ - Cid: smsg.Message.Cid().String(), + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: smsg.Message.Cid()}, Version: int64(smsg.Message.Message.Version), - To: smsg.Message.Message.To.String(), - From: smsg.Message.Message.From.String(), + To: dbtypes.DbAddr{Addr: smsg.Message.Message.To}, + From: dbtypes.DbAddr{smsg.Message.Message.From}, Nonce: smsg.Message.Message.Nonce, - Value: smsg.Message.Message.Value.String(), + Value: dbtypes.DbToken{Token: smsg.Message.Message.Value}, GasLimit: smsg.Message.Message.GasLimit, - GasFeeCap: smsg.Message.Message.GasFeeCap.String(), - GasPremium: smsg.Message.Message.GasPremium.String(), + GasFeeCap: dbtypes.DbToken{Token: smsg.Message.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: smsg.Message.Message.GasPremium}, Method: uint64(smsg.Message.Message.Method), Params: smsg.Message.Message.Params, }) @@ -208,16 +189,16 @@ func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) if !seen.Visit(msg.Message.Cid()) { continue } - out = append(out, &ChainMessage{ - Cid: msg.Message.Cid().String(), + out = append(out, &models.Message{ + Cid: dbtypes.DbCID{CID: msg.Message.Cid()}, Version: int64(msg.Message.Version), - To: msg.Message.To.String(), - From: msg.Message.From.String(), + To: dbtypes.DbAddr{Addr: msg.Message.To}, + From: dbtypes.DbAddr{msg.Message.From}, Nonce: msg.Message.Nonce, - Value: msg.Message.Value.String(), + Value: dbtypes.DbToken{Token: msg.Message.Value}, GasLimit: msg.Message.GasLimit, - GasFeeCap: msg.Message.GasFeeCap.String(), - GasPremium: msg.Message.GasPremium.String(), + GasFeeCap: dbtypes.DbToken{Token: msg.Message.GasFeeCap}, + GasPremium: dbtypes.DbToken{Token: msg.Message.GasPremium}, Method: uint64(msg.Message.Method), Params: msg.Message.Params, }) @@ -226,63 +207,109 @@ func MakeMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) return out } -type ChainMessageReceipt struct { - MessageCid string `gorm:"primaryKey"` - Index int64 `gorm:"primaryKey"` - ExitCode int64 - GasUsed int64 - Return []byte - - BaseFeeBurn string `gorm:"bigint"` - OverEstimationBurn string `gorm:"bigint"` - MinerPenalty string `gorm:"bigint"` - MinerTip string `gorm:"bigint"` - Refund string `gorm:"bigint"` - GasRefund int64 - GasBurned int64 -} - -func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*ChainMessageReceipt { - var out []*ChainMessageReceipt +func MakeReceipts(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.MessageReceipt { + var out []*models.MessageReceipt for _, fb := range fullBlocks { for _, smsg := range fb.SecpMessages { + // TODO this is buggy if smsg.Receipt.GasOutputs == nil { continue } - out = append(out, &ChainMessageReceipt{ - MessageCid: smsg.Message.Cid().String(), - Index: smsg.Receipt.Index, - ExitCode: int64(smsg.Receipt.Receipt.ExitCode), - GasUsed: smsg.Receipt.Receipt.GasUsed, - Return: smsg.Receipt.Receipt.Return, - BaseFeeBurn: smsg.Receipt.GasOutputs.BaseFeeBurn.String(), - OverEstimationBurn: smsg.Receipt.GasOutputs.OverEstimationBurn.String(), - MinerPenalty: smsg.Receipt.GasOutputs.MinerPenalty.String(), - MinerTip: smsg.Receipt.GasOutputs.MinerTip.String(), - Refund: smsg.Receipt.GasOutputs.Refund.String(), + actorError := "" + if smsg.Receipt.ActorError != nil { + actorError = smsg.Receipt.ActorError.Error + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: smsg.Message.Cid()}, + Receipt: models.Receipt{ + Index: smsg.Receipt.Index, + ExitCode: int64(smsg.Receipt.Receipt.ExitCode), + GasUsed: smsg.Receipt.Receipt.GasUsed, + Return: smsg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: smsg.Receipt.GasOutputs.Refund}, GasRefund: smsg.Receipt.GasOutputs.GasRefund, GasBurned: smsg.Receipt.GasOutputs.GasBurned, + Error: actorError, }) } for _, msg := range fb.BlsMessages { if msg.Receipt.GasOutputs == nil { continue } - out = append(out, &ChainMessageReceipt{ - MessageCid: msg.Message.Cid().String(), - Index: msg.Receipt.Index, - ExitCode: int64(msg.Receipt.Receipt.ExitCode), - GasUsed: msg.Receipt.Receipt.GasUsed, - Return: msg.Receipt.Receipt.Return, - BaseFeeBurn: msg.Receipt.GasOutputs.BaseFeeBurn.String(), - OverEstimationBurn: msg.Receipt.GasOutputs.OverEstimationBurn.String(), - MinerPenalty: msg.Receipt.GasOutputs.MinerPenalty.String(), - MinerTip: msg.Receipt.GasOutputs.MinerTip.String(), - Refund: msg.Receipt.GasOutputs.Refund.String(), + actorError := "" + if msg.Receipt.ActorError != nil { + actorError = msg.Receipt.ActorError.Error + } + out = append(out, &models.MessageReceipt{ + MessageCid: dbtypes.DbCID{CID: msg.Message.Cid()}, + Receipt: models.Receipt{ + Index: msg.Receipt.Index, + ExitCode: int64(msg.Receipt.Receipt.ExitCode), + GasUsed: msg.Receipt.Receipt.GasUsed, + Return: msg.Receipt.Receipt.Return, + }, + BaseFeeBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.BaseFeeBurn}, + OverEstimationBurn: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.OverEstimationBurn}, + MinerPenalty: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerPenalty}, + MinerTip: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.MinerTip}, + Refund: dbtypes.DbToken{Token: msg.Receipt.GasOutputs.Refund}, GasRefund: msg.Receipt.GasOutputs.GasRefund, GasBurned: msg.Receipt.GasOutputs.GasBurned, + Error: actorError, }) } } return out } + +func MakeVmMessages(ctx context.Context, fullBlocks map[cid.Cid]*chain.FullBlock) []*models.VmMessage { + var out []*models.VmMessage + for _, fb := range fullBlocks { + for _, msg := range fb.SecpMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + for _, msg := range fb.BlsMessages { + for _, vm := range msg.VmMessages { + out = append(out, &models.VmMessage{ + Source: dbtypes.DbCID{CID: vm.Source}, + Cid: dbtypes.DbCID{CID: vm.Message.Cid()}, + To: dbtypes.DbAddr{Addr: vm.Message.To}, + From: dbtypes.DbAddr{vm.Message.From}, + Value: dbtypes.DbToken{Token: vm.Message.Value}, + Method: uint64(vm.Message.Method), + Params: vm.Message.Params, + Receipt: models.Receipt{ + Index: vm.Index, + ExitCode: int64(vm.Receipt.ExitCode), + GasUsed: vm.Receipt.GasUsed, + Return: vm.Receipt.Return, + }, + Error: vm.Error, + }) + } + } + } + return out +} From 0655c3efdc3628b8d0b1ba1ecbe74e88185b2e5b Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 8 Feb 2023 13:31:29 -0800 Subject: [PATCH 41/48] simplify actor state diffing into single method --- lens/lily/impl.go | 33 ++-- pkg/extract/actors.go | 149 +++++++++++------- .../actors/datacapdiff/v1/allowances.go | 4 + pkg/extract/actors/datacapdiff/v1/balances.go | 4 + pkg/extract/actors/datacapdiff/v1/state.go | 57 ++----- pkg/extract/actors/datacapdiff/version.go | 13 +- pkg/extract/actors/executor.go | 62 +++++--- pkg/extract/actors/initdiff/v1/addresses.go | 4 + pkg/extract/actors/initdiff/v1/state.go | 31 +--- pkg/extract/actors/initdiff/version.go | 12 +- pkg/extract/actors/marketdiff/v1/deals.go | 4 + pkg/extract/actors/marketdiff/v1/proposals.go | 4 + pkg/extract/actors/marketdiff/v1/state.go | 35 ++-- pkg/extract/actors/marketdiff/version.go | 15 +- pkg/extract/actors/minerdiff/v1/infos.go | 6 +- pkg/extract/actors/minerdiff/v1/precommits.go | 4 + .../actors/minerdiff/v1/sector_status.go | 4 + pkg/extract/actors/minerdiff/v1/sectors.go | 4 + pkg/extract/actors/minerdiff/v1/state.go | 44 ++---- pkg/extract/actors/minerdiff/version.go | 19 +-- pkg/extract/actors/powerdiff/v1/claims.go | 4 + pkg/extract/actors/powerdiff/v1/state.go | 31 +--- pkg/extract/actors/powerdiff/version.go | 11 +- pkg/extract/actors/rawdiff/actor.go | 36 +---- pkg/extract/actors/rawdiff/state.go | 55 +------ pkg/extract/actors/rawdiff/version.go | 11 ++ pkg/extract/actors/types.go | 18 ++- pkg/extract/actors/verifregdiff/v1/clients.go | 4 + pkg/extract/actors/verifregdiff/v1/state.go | 36 ++--- .../actors/verifregdiff/v1/verifiers.go | 4 + .../actors/verifregdiff/v2/allocations.go | 4 + pkg/extract/actors/verifregdiff/v2/claims.go | 4 + pkg/extract/actors/verifregdiff/v2/state.go | 42 ++--- pkg/extract/actors/verifregdiff/version.go | 24 ++- 34 files changed, 360 insertions(+), 432 deletions(-) create mode 100644 pkg/extract/actors/rawdiff/version.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index 1305a0dc0..e2eb889cb 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -28,9 +28,6 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" - "gorm.io/driver/postgres" - "gorm.io/gorm" - "gorm.io/gorm/schema" "github.com/filecoin-project/lily/chain/datasource" "github.com/filecoin-project/lily/chain/gap" @@ -48,7 +45,6 @@ import ( "github.com/filecoin-project/lily/network" "github.com/filecoin-project/lily/pkg/extract" "github.com/filecoin-project/lily/pkg/transform/cbor" - "github.com/filecoin-project/lily/pkg/transform/gorm/lambda" "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" "github.com/filecoin-project/lily/storage" @@ -157,20 +153,23 @@ func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) return false, err } - // connect to the database, were gonna write stuff to this. - db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ - NamingStrategy: schema.NamingStrategy{ - TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list - // TODO figure out how to make a new schema with gorm... - }, - }) - if err != nil { - return false, err - } - if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { - return false, err - } + /* + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return false, err + } + if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { + return false, err + } + + */ currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) if err != nil { return false, err diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 5383d7f2a..a9a069a7f 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -2,9 +2,9 @@ package extract import ( "context" + "fmt" "sort" "sync" - "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/abi" @@ -16,16 +16,22 @@ import ( "go.opentelemetry.io/otel/attribute" "go.uber.org/zap/zapcore" - "github.com/filecoin-project/lily/chain/actors/builtin" "github.com/filecoin-project/lily/pkg/core" "github.com/filecoin-project/lily/pkg/extract/actors" "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff" + dcapDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/initdiff" + initDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff" + marketDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff" + minerDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff" + powerDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" "github.com/filecoin-project/lily/pkg/extract/actors/rawdiff" "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff" + verifDiffV1 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" + verifDiffV2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" "github.com/filecoin-project/lily/pkg/extract/statetree" "github.com/filecoin-project/lily/tasks" ) @@ -67,23 +73,6 @@ type StateDiffResult struct { type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version -func DiffActor(ctx context.Context, api tasks.DataSource, version actortypes.Version, act *actors.ActorChange, diffLoader func(av actortypes.Version) (actors.ActorDiff, error)) (actors.ActorDiffResult, error) { - // construct the state differ required by this actor version - diff, err := diffLoader(version) - if err != nil { - return nil, err - } - start := time.Now() - // diff the actors state and collect changes - log.Debugw("diff actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String()) - diffRes, err := diff.State(ctx, api, act) - if err != nil { - return nil, err - } - log.Debugw("diffed actor state", "address", act.Address.String(), "name", builtin.ActorNameByCode(act.Current.Code), "type", act.Type.String(), "duration", time.Since(start)) - return diffRes, nil -} - func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []address.Address { keys := make([]address.Address, 0, len(actors)) @@ -133,7 +122,12 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. } wg.Add(1) pool.Submit(func() { - doActorDiff(ctx, api, act, actorVersion, results) + if err := doRawActorDiff(ctx, api, act, actorVersion, results); err != nil { + log.Fatal(err) + } + if err := doActorDiff(ctx, api, act, actorVersion, results); err != nil { + log.Fatal(err) + } wg.Done() }) @@ -144,68 +138,115 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. }() for stateDiff := range results { if stateDiff.Error != nil { + // TODO bug need to call wg.Done() for remaining open wg functions pool.Stop() return nil, err } - switch stateDiff.ActorDiff.Kind() { - case "actor": - asc.RawActors[stateDiff.Address] = stateDiff.ActorDiff - case "miner": - asc.MinerActors[stateDiff.Address] = stateDiff.ActorDiff - case "verifreg": - asc.VerifregActor = stateDiff.ActorDiff - case "init": - asc.InitActor = stateDiff.ActorDiff - case "power": - asc.PowerActor = stateDiff.ActorDiff - case "market": - asc.MarketActor = stateDiff.ActorDiff - case "datacap": - asc.DatacapActor = stateDiff.ActorDiff + switch v := stateDiff.ActorDiff.(type) { + case *rawdiff.StateDiffResult: + asc.RawActors[stateDiff.Address] = v + case *minerDiffV1.StateDiffResult: + asc.MinerActors[stateDiff.Address] = v + case *initDiffV1.StateDiffResult: + asc.InitActor = v + case *powerDiffV1.StateDiffResult: + asc.PowerActor = v + case *marketDiffV1.StateDiffResult: + asc.MarketActor = v + case *dcapDiffV1.StateDiffResult: + asc.DatacapActor = v + case *verifDiffV1.StateDiffResult, *verifDiffV2.StateDiffResult: + asc.VerifregActor = v default: - panic(stateDiff.ActorDiff.Kind()) + return nil, fmt.Errorf("unknown StateDiffResult type: %T", v) } } return asc, nil } -func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { - actorDiff := &rawdiff.StateDiff{ - DiffMethods: []actors.ActorStateDiff{rawdiff.Actor{}}, - } - actorStateChanges, err := actorDiff.State(ctx, api, act) +func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) error { + methods, handler, err := rawdiff.StateDiffFor(version) + actorDiff := &actors.StateDiffer{ + Methods: methods, + ReportHandler: func(reports []actors.DifferReport) error { + for _, report := range reports { + log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) + } + return nil + }, + ActorHandler: handler, + } + res, err := actorDiff.ActorDiff(ctx, api, act) results <- &StateDiffResult{ - ActorDiff: actorStateChanges, + ActorDiff: res, Address: act.Address, Error: err, } - var actorDiffer func(av actortypes.Version) (actors.ActorDiff, error) + return nil +} + +func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) error { + var ( + methods []actors.ActorDiffMethods + handler actors.ActorHandlerFn + err error + ) if core.DataCapCodes.Has(act.Current.Code) { - actorDiffer = datacapdiff.StateDiffFor + methods, handler, err = datacapdiff.StateDiffFor(version) + if err != nil { + return err + } } if core.MinerCodes.Has(act.Current.Code) { - actorDiffer = minerdiff.StateDiffFor - } - if core.VerifregCodes.Has(act.Current.Code) { - actorDiffer = verifregdiff.StateDiffFor + methods, handler, err = minerdiff.StateDiffFor(version) + if err != nil { + return err + } } if core.InitCodes.Has(act.Current.Code) { - actorDiffer = initdiff.StateDiffFor + methods, handler, err = initdiff.StateDiffFor(version) + if err != nil { + return err + } } if core.PowerCodes.Has(act.Current.Code) { - actorDiffer = powerdiff.StateDiffFor + methods, handler, err = powerdiff.StateDiffFor(version) + if err != nil { + return err + } } if core.MarketCodes.Has(act.Current.Code) { - actorDiffer = marketdiff.StateDiffFor + methods, handler, err = marketdiff.StateDiffFor(version) + if err != nil { + return err + } + } + if core.VerifregCodes.Has(act.Current.Code) { + methods, handler, err = verifregdiff.StateDiffFor(version) + if err != nil { + return err + } } - if actorDiffer == nil { - return + + if methods == nil { + return nil } - res, err := DiffActor(ctx, api, version, act, actorDiffer) + actorDiff := &actors.StateDiffer{ + Methods: methods, + ReportHandler: func(reports []actors.DifferReport) error { + for _, report := range reports { + log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) + } + return nil + }, + ActorHandler: handler, + } + res, err := actorDiff.ActorDiff(ctx, api, act) results <- &StateDiffResult{ ActorDiff: res, Address: act.Address, Error: err, } + return nil } diff --git a/pkg/extract/actors/datacapdiff/v1/allowances.go b/pkg/extract/actors/datacapdiff/v1/allowances.go index 406d6b07e..524b89ef4 100644 --- a/pkg/extract/actors/datacapdiff/v1/allowances.go +++ b/pkg/extract/actors/datacapdiff/v1/allowances.go @@ -64,6 +64,10 @@ func (a AllowanceChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Allowance struct{} +func (a Allowance) Type() string { + return KindDataCapAllowance +} + func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/datacapdiff/v1/balances.go b/pkg/extract/actors/datacapdiff/v1/balances.go index 4e11e167a..7ad45a94f 100644 --- a/pkg/extract/actors/datacapdiff/v1/balances.go +++ b/pkg/extract/actors/datacapdiff/v1/balances.go @@ -49,6 +49,10 @@ func (b BalanceChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Balance struct{} +func (b Balance) Type() string { + return KindDataCapBalance +} + func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/datacapdiff/v1/state.go b/pkg/extract/actors/datacapdiff/v1/state.go index 38ce36e8d..de48293cf 100644 --- a/pkg/extract/actors/datacapdiff/v1/state.go +++ b/pkg/extract/actors/datacapdiff/v1/state.go @@ -2,25 +2,35 @@ package v1 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { + var stateDiff = new(StateDiffResult) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case AllowanceChangeMap: + stateDiff.AllowanceChanges = v + case BalanceChangeList: + stateDiff.BalanceChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) + } + } + return stateDiff, nil +} + type StateDiffResult struct { BalanceChanges BalanceChangeList AllowanceChanges AllowanceChangeMap } -func (sd *StateDiffResult) Kind() string { - return "datacap" -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} @@ -45,38 +55,3 @@ type StateChange struct { Balances *cid.Cid `cborgen:"balances"` Allowances *cid.Cid `cborgen:"allowances"` } - -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - return ChangeHandler( - ctx, - api, - act, - func(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { - var stateDiff = new(StateDiffResult) - for _, stateChange := range changes { - switch stateChange.Kind() { - case KindDataCapAllowance: - stateDiff.AllowanceChanges = stateChange.(AllowanceChangeMap) - case KindDataCapBalance: - stateDiff.BalanceChanges = stateChange.(BalanceChangeList) - } - } - return stateDiff, nil - }, - s.DiffMethods...) -} - -type StateDiffHandlerFn = func(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) - -func ChangeHandler(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, handler StateDiffHandlerFn, differs ...actors.ActorStateDiff) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, differs...) - if err != nil { - return nil, err - } - return handler(results) -} diff --git a/pkg/extract/actors/datacapdiff/version.go b/pkg/extract/actors/datacapdiff/version.go index e6b8e5042..d8640058e 100644 --- a/pkg/extract/actors/datacapdiff/version.go +++ b/pkg/extract/actors/datacapdiff/version.go @@ -9,17 +9,10 @@ import ( v1 "github.com/filecoin-project/lily/pkg/extract/actors/datacapdiff/v1" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { switch av { case actortypes.Version9: - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Allowance{}, - v1.Balance{}, - }, - }, nil - case actortypes.Version10: - panic("Not yet implemented") + return []actors.ActorDiffMethods{v1.Allowance{}, v1.Balance{}}, v1.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index ff2966c75..1ec4eaa21 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -2,30 +2,56 @@ package actors import ( "context" - - "golang.org/x/sync/errgroup" + "time" "github.com/filecoin-project/lily/tasks" ) -func ExecuteStateDiff(ctx context.Context, grp *errgroup.Group, api tasks.DataSource, act *ActorChange, fns ...ActorStateDiff) ([]ActorStateChange, error) { - out := make([]ActorStateChange, len(fns)) +type DifferReport struct { + DiffType string + StartTime time.Time + Duration time.Duration + Error error + Result ActorStateChange +} + +func ExecuteStateDiff(ctx context.Context, api tasks.DataSource, act *ActorChange, fns ...ActorDiffMethods) []DifferReport { + out := make([]DifferReport, len(fns)) for i, fn := range fns { - fn := fn - i := i - - grp.Go(func() error { - res, err := fn.Diff(ctx, api, act) - if err != nil { - return err - } - out[i] = res - return nil - }) + start := time.Now() + res, err := fn.Diff(ctx, api, act) + out[i] = DifferReport{ + DiffType: fn.Type(), + StartTime: start, + Duration: time.Since(start), + Error: err, + Result: res, + } + } + return out +} + +type StateDiffer struct { + Methods []ActorDiffMethods + ReportHandler ReportHandlerFn + ActorHandler ActorHandlerFn +} + +type ReportHandlerFn = func(reports []DifferReport) error +type ActorHandlerFn = func(changes []ActorStateChange) (ActorDiffResult, error) + +func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) { + reports := ExecuteStateDiff(ctx, api, act, s.Methods...) + + if s.ReportHandler != nil { + if err := s.ReportHandler(reports); err != nil { + return nil, err + } } - if err := grp.Wait(); err != nil { - return nil, err + var results []ActorStateChange + for _, report := range reports { + results = append(results, report.Result) } - return out, nil + return s.ActorHandler(results) } diff --git a/pkg/extract/actors/initdiff/v1/addresses.go b/pkg/extract/actors/initdiff/v1/addresses.go index 95f13a1bc..68824c8d7 100644 --- a/pkg/extract/actors/initdiff/v1/addresses.go +++ b/pkg/extract/actors/initdiff/v1/addresses.go @@ -51,6 +51,10 @@ func (a AddressChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Addresses struct{} +func (a Addresses) Type() string { + return KindInitAddresses +} + func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/initdiff/v1/state.go b/pkg/extract/actors/initdiff/v1/state.go index f8a05f8b3..51996010c 100644 --- a/pkg/extract/actors/initdiff/v1/state.go +++ b/pkg/extract/actors/initdiff/v1/state.go @@ -2,35 +2,24 @@ package v1 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindInitAddresses: - stateDiff.AddressesChanges = stateChange.(AddressChangeList) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case AddressChangeList: + stateDiff.AddressesChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) } } return stateDiff, nil @@ -40,10 +29,6 @@ type StateDiffResult struct { AddressesChanges AddressChangeList } -func (s *StateDiffResult) Kind() string { - return "init" -} - func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} diff --git a/pkg/extract/actors/initdiff/version.go b/pkg/extract/actors/initdiff/version.go index be83b722b..c1183ed10 100644 --- a/pkg/extract/actors/initdiff/version.go +++ b/pkg/extract/actors/initdiff/version.go @@ -9,16 +9,12 @@ import ( v1 "github.com/filecoin-project/lily/pkg/extract/actors/initdiff/v1" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { switch av { + // TODO less than eaual to version 9 case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Addresses{}, - }}, nil - case actortypes.Version10: - panic("NYI") + return []actors.ActorDiffMethods{v1.Addresses{}}, v1.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/extract/actors/marketdiff/v1/deals.go b/pkg/extract/actors/marketdiff/v1/deals.go index 19e1acad8..daef1e813 100644 --- a/pkg/extract/actors/marketdiff/v1/deals.go +++ b/pkg/extract/actors/marketdiff/v1/deals.go @@ -47,6 +47,10 @@ func (p DealChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Deals struct{} +func (d Deals) Type() string { + return KindMarketDeal +} + func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/marketdiff/v1/proposals.go b/pkg/extract/actors/marketdiff/v1/proposals.go index b9831dcb6..07ccbf154 100644 --- a/pkg/extract/actors/marketdiff/v1/proposals.go +++ b/pkg/extract/actors/marketdiff/v1/proposals.go @@ -54,6 +54,10 @@ func (p ProposalChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Proposals struct{} +func (p Proposals) Type() string { + return KindMarketProposal +} + func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/marketdiff/v1/state.go b/pkg/extract/actors/marketdiff/v1/state.go index 13bf0026c..73aa88d87 100644 --- a/pkg/extract/actors/marketdiff/v1/state.go +++ b/pkg/extract/actors/marketdiff/v1/state.go @@ -2,40 +2,27 @@ package v1 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpCtx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpCtx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindMarketDeal: - stateDiff.DealStateChanges = stateChange.(DealChangeList) - case KindMarketProposal: - stateDiff.DealProposalChanges = stateChange.(ProposalChangeList) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case DealChangeList: + stateDiff.DealStateChanges = v + case ProposalChangeList: + stateDiff.DealProposalChanges = v default: - panic(stateChange.Kind()) + return nil, fmt.Errorf("unknown state change kind: %T", v) } } return stateDiff, nil @@ -46,10 +33,6 @@ type StateDiffResult struct { DealProposalChanges ProposalChangeList } -func (sd *StateDiffResult) Kind() string { - return "market" -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} diff --git a/pkg/extract/actors/marketdiff/version.go b/pkg/extract/actors/marketdiff/version.go index 448d077f8..610e7bab5 100644 --- a/pkg/extract/actors/marketdiff/version.go +++ b/pkg/extract/actors/marketdiff/version.go @@ -9,17 +9,14 @@ import ( v1 "github.com/filecoin-project/lily/pkg/extract/actors/marketdiff/v1" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { switch av { case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Deals{}, - v1.Proposals{}, - }}, nil - case actortypes.Version10: - panic("NYI") + return []actors.ActorDiffMethods{ + v1.Deals{}, + v1.Proposals{}, + }, v1.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/extract/actors/minerdiff/v1/infos.go b/pkg/extract/actors/minerdiff/v1/infos.go index bc952764b..fa0909544 100644 --- a/pkg/extract/actors/minerdiff/v1/infos.go +++ b/pkg/extract/actors/minerdiff/v1/infos.go @@ -30,10 +30,14 @@ func (i *InfoChange) Kind() actors.ActorStateKind { return KindMinerInfo } -var _ actors.ActorStateDiff = (*Info)(nil) +var _ actors.ActorDiffMethods = (*Info)(nil) type Info struct{} +func (i Info) Type() string { + return KindMinerInfo +} + func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/minerdiff/v1/precommits.go b/pkg/extract/actors/minerdiff/v1/precommits.go index becfe1e33..27abe7765 100644 --- a/pkg/extract/actors/minerdiff/v1/precommits.go +++ b/pkg/extract/actors/minerdiff/v1/precommits.go @@ -55,6 +55,10 @@ func (p PreCommitChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) type PreCommit struct{} +func (c PreCommit) Type() string { + return KindMinerPreCommit +} + func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/minerdiff/v1/sector_status.go b/pkg/extract/actors/minerdiff/v1/sector_status.go index c0e12bea3..50590860e 100644 --- a/pkg/extract/actors/minerdiff/v1/sector_status.go +++ b/pkg/extract/actors/minerdiff/v1/sector_status.go @@ -36,6 +36,10 @@ func (s SectorStatusChange) Kind() actors.ActorStateKind { type SectorStatus struct{} +func (s SectorStatus) Type() string { + return KindMinerSectorStatus +} + func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/minerdiff/v1/sectors.go b/pkg/extract/actors/minerdiff/v1/sectors.go index b26333437..fca4f1eeb 100644 --- a/pkg/extract/actors/minerdiff/v1/sectors.go +++ b/pkg/extract/actors/minerdiff/v1/sectors.go @@ -48,6 +48,10 @@ func (s SectorChangeList) Kind() actors.ActorStateKind { type Sectors struct{} +func (s Sectors) Type() string { + return KindMinerSector +} + func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/minerdiff/v1/state.go b/pkg/extract/actors/minerdiff/v1/state.go index 5cfe8f49c..565eeb27d 100644 --- a/pkg/extract/actors/minerdiff/v1/state.go +++ b/pkg/extract/actors/minerdiff/v1/state.go @@ -9,42 +9,26 @@ import ( "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" typegen "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) var log = logging.Logger("extract/actors/miner") -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } - +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - // some results may be nil, skip those - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindMinerInfo: - stateDiff.InfoChange = stateChange.(*InfoChange) - case KindMinerSector: - stateDiff.SectorChanges = stateChange.(SectorChangeList) - case KindMinerPreCommit: - stateDiff.PreCommitChanges = stateChange.(PreCommitChangeList) - case KindMinerSectorStatus: - stateDiff.SectorStatusChanges = stateChange.(*SectorStatusChange) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case *InfoChange: + stateDiff.InfoChange = v + case SectorChangeList: + stateDiff.SectorChanges = v + case PreCommitChangeList: + stateDiff.PreCommitChanges = v + case *SectorStatusChange: + stateDiff.SectorStatusChanges = v default: - return nil, fmt.Errorf("unknown state change %s", stateChange.Kind()) + return nil, fmt.Errorf("unknown state change %T", v) } } @@ -58,10 +42,6 @@ type StateDiffResult struct { SectorStatusChanges *SectorStatusChange } -func (s *StateDiffResult) Kind() string { - return "miner" -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { out := &StateChange{} diff --git a/pkg/extract/actors/minerdiff/version.go b/pkg/extract/actors/minerdiff/version.go index 71b0ba3e8..4bbcac9cc 100644 --- a/pkg/extract/actors/minerdiff/version.go +++ b/pkg/extract/actors/minerdiff/version.go @@ -9,19 +9,16 @@ import ( v1 "github.com/filecoin-project/lily/pkg/extract/actors/minerdiff/v1" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { switch av { case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Info{}, - v1.PreCommit{}, - v1.SectorStatus{}, - v1.Sectors{}, - }}, nil - case actortypes.Version10: - panic("Not yet implemented") + return []actors.ActorDiffMethods{ + v1.Info{}, + v1.PreCommit{}, + v1.SectorStatus{}, + v1.Sectors{}, + }, v1.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/extract/actors/powerdiff/v1/claims.go b/pkg/extract/actors/powerdiff/v1/claims.go index ce6d91055..e3f662f05 100644 --- a/pkg/extract/actors/powerdiff/v1/claims.go +++ b/pkg/extract/actors/powerdiff/v1/claims.go @@ -48,6 +48,10 @@ func (p ClaimsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Claims struct{} +func (c Claims) Type() string { + return KindPowerClaims +} + func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/powerdiff/v1/state.go b/pkg/extract/actors/powerdiff/v1/state.go index 15a2dcb96..ed1857535 100644 --- a/pkg/extract/actors/powerdiff/v1/state.go +++ b/pkg/extract/actors/powerdiff/v1/state.go @@ -2,37 +2,24 @@ package v1 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpCtx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpCtx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindPowerClaims: - stateDiff.ClaimsChanges = stateChange.(ClaimsChangeList) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case ClaimsChangeList: + stateDiff.ClaimsChanges = v default: - panic(stateChange.Kind()) + return nil, fmt.Errorf("unknown state change kind: %T", v) } } return stateDiff, nil @@ -54,10 +41,6 @@ func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store return out, nil } -func (sd *StateDiffResult) Kind() string { - return "power" -} - type StateChange struct { Claims *cid.Cid `cborgen:"claims"` } diff --git a/pkg/extract/actors/powerdiff/version.go b/pkg/extract/actors/powerdiff/version.go index 227c0e582..31e8d53fb 100644 --- a/pkg/extract/actors/powerdiff/version.go +++ b/pkg/extract/actors/powerdiff/version.go @@ -9,16 +9,11 @@ import ( v1 "github.com/filecoin-project/lily/pkg/extract/actors/powerdiff/v1" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { switch av { case actortypes.Version0, actortypes.Version2, actortypes.Version3, actortypes.Version4, actortypes.Version5, actortypes.Version6, actortypes.Version7, actortypes.Version8, actortypes.Version9: - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Claims{}, - }}, nil - case actortypes.Version10: - panic("NYI") + return []actors.ActorDiffMethods{v1.Claims{}}, v1.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } diff --git a/pkg/extract/actors/rawdiff/actor.go b/pkg/extract/actors/rawdiff/actor.go index 4b77610f7..47c4aacb6 100644 --- a/pkg/extract/actors/rawdiff/actor.go +++ b/pkg/extract/actors/rawdiff/actor.go @@ -1,15 +1,11 @@ package rawdiff import ( - "bytes" "context" "fmt" "time" - "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/lotus/chain/types" - block "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.uber.org/zap" @@ -27,34 +23,6 @@ type ActorChange struct { Change core.ChangeType `cborgen:"change"` } -func (a *ActorChange) Serialize() ([]byte, error) { - buf := new(bytes.Buffer) - if err := a.MarshalCBOR(buf); err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func (a *ActorChange) ToStorageBlock() (block.Block, error) { - data, err := a.Serialize() - if err != nil { - return nil, err - } - c, err := abi.CidBuilder.WithCodec(cid.Raw).Sum(data) - if err != nil { - return nil, err - } - return block.NewBlockWithCid(data, c) -} - -func DecodeActorChange(b []byte) (*ActorChange, error) { - var ac ActorChange - if err := ac.UnmarshalCBOR(bytes.NewReader(b)); err != nil { - return nil, err - } - return &ac, nil -} - const KindActorChange = "actor_change" func (a *ActorChange) Kind() actors.ActorStateKind { @@ -71,6 +39,10 @@ func (Actor) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorCh return ActorDiff(ctx, api, act) } +func (Actor) Type() string { + return KindActorChange +} + func ActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { switch act.Type { case core.ChangeTypeAdd: diff --git a/pkg/extract/actors/rawdiff/state.go b/pkg/extract/actors/rawdiff/state.go index 686fd10f1..d4fc423ee 100644 --- a/pkg/extract/actors/rawdiff/state.go +++ b/pkg/extract/actors/rawdiff/state.go @@ -2,34 +2,23 @@ package rawdiff import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { + for _, change := range changes { + switch change.Kind() { case KindActorChange: - stateDiff.ActorStateChanges = stateChange.(*ActorChange) + stateDiff.ActorStateChanges = change.(*ActorChange) + default: + return nil, fmt.Errorf("unhandledd change kind: %s", change.Kind()) } } return stateDiff, nil @@ -39,10 +28,6 @@ type StateDiffResult struct { ActorStateChanges *ActorChange } -func (sdr *StateDiffResult) Kind() string { - return "actor" -} - func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (cbg.CBORMarshaler, error) { return sdr.ActorStateChanges, nil } @@ -50,29 +35,3 @@ func (sdr *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Stor type StateChange struct { ActorState cid.Cid `cborgen:"actors"` } - -func GenericStateDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, diffFns []actors.ActorStateDiff) ([]actors.ActorStateChange, error) { - grp, grpCtx := errgroup.WithContext(ctx) - out := make([]actors.ActorStateChange, 0, len(diffFns)) - results := make(chan actors.ActorStateChange, len(diffFns)) - for _, diff := range diffFns { - diff := diff - grp.Go(func() error { - res, err := diff.Diff(grpCtx, api, act) - if err != nil { - return err - } - results <- res - return nil - }) - } - if err := grp.Wait(); err != nil { - close(results) - return nil, err - } - close(results) - for res := range results { - out = append(out, res) - } - return out, nil -} diff --git a/pkg/extract/actors/rawdiff/version.go b/pkg/extract/actors/rawdiff/version.go new file mode 100644 index 000000000..831371e95 --- /dev/null +++ b/pkg/extract/actors/rawdiff/version.go @@ -0,0 +1,11 @@ +package rawdiff + +import ( + actortypes "github.com/filecoin-project/go-state-types/actors" + + "github.com/filecoin-project/lily/pkg/extract/actors" +) + +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { + return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, nil +} diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index f7da04f5b..4d3518f78 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -2,6 +2,7 @@ package actors import ( "context" + "time" "github.com/filecoin-project/go-address" "github.com/filecoin-project/go-state-types/store" @@ -15,8 +16,20 @@ import ( "github.com/filecoin-project/lily/tasks" ) +type ExtractionReport struct { + Current *types.TipSet + Executed *types.TipSet + NetworkVersion int // version of the network + ActorVersion int // version of the actor being extracted + ExtractorVersion int // version of the extraction logic + ExtractorKind string // miner/market/init/power/etc. + DifferKind string // miner_info/miner_sectors/market_claims/power_claims/etc. + ChangeType core.ChangeType // Added, Modified, Removed + StartTime time.Time // when the differ started execution + Duration time.Duration // how long the differ took to run +} + type ActorDiffResult interface { - Kind() string MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) } @@ -24,8 +37,9 @@ type ActorDiff interface { State(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) } -type ActorStateDiff interface { +type ActorDiffMethods interface { Diff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorStateChange, error) + Type() string } type ActorStateKind string diff --git a/pkg/extract/actors/verifregdiff/v1/clients.go b/pkg/extract/actors/verifregdiff/v1/clients.go index 52a9bc2ae..f10a8c2b4 100644 --- a/pkg/extract/actors/verifregdiff/v1/clients.go +++ b/pkg/extract/actors/verifregdiff/v1/clients.go @@ -55,6 +55,10 @@ func (v ClientsChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Clients struct{} +func (c Clients) Type() string { + return KindVerifregClients +} + func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/verifregdiff/v1/state.go b/pkg/extract/actors/verifregdiff/v1/state.go index 095cb513b..e708a8b01 100644 --- a/pkg/extract/actors/verifregdiff/v1/state.go +++ b/pkg/extract/actors/verifregdiff/v1/state.go @@ -2,38 +2,26 @@ package v1 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" typegen "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - "github.com/filecoin-project/lily/tasks" ) -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } - +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case KindVerifregClients: - stateDiff.ClientChanges = stateChange.(ClientsChangeList) - case KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case ClientsChangeList: + stateDiff.ClientChanges = v + case VerifiersChangeList: + stateDiff.VerifierChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) } } return stateDiff, nil @@ -44,10 +32,6 @@ type StateDiffResult struct { ClientChanges ClientsChangeList } -func (s *StateDiffResult) Kind() string { - return "verifreg" -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) { out := &StateChange{} diff --git a/pkg/extract/actors/verifregdiff/v1/verifiers.go b/pkg/extract/actors/verifregdiff/v1/verifiers.go index 39d95c2ed..5d794c59a 100644 --- a/pkg/extract/actors/verifregdiff/v1/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v1/verifiers.go @@ -51,6 +51,10 @@ func (v VerifiersChangeList) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) type Verifiers struct{} +func (v Verifiers) Type() string { + return KindVerifregVerifiers +} + func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/verifregdiff/v2/allocations.go b/pkg/extract/actors/verifregdiff/v2/allocations.go index 16f0fce7f..10c0c6898 100644 --- a/pkg/extract/actors/verifregdiff/v2/allocations.go +++ b/pkg/extract/actors/verifregdiff/v2/allocations.go @@ -66,6 +66,10 @@ func (c AllocationsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) type Allocations struct{} +func (a Allocations) Type() string { + return KindVerifregAllocations +} + func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/verifregdiff/v2/claims.go b/pkg/extract/actors/verifregdiff/v2/claims.go index 594b8e5a8..ae8858cc6 100644 --- a/pkg/extract/actors/verifregdiff/v2/claims.go +++ b/pkg/extract/actors/verifregdiff/v2/claims.go @@ -68,6 +68,10 @@ func (c ClaimsChangeMap) ToAdtMap(store adt.Store, bw int) (cid.Cid, error) { type Claims struct{} +func (c Claims) Type() string { + return KindVerifregClaims +} + func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { start := time.Now() defer func() { diff --git a/pkg/extract/actors/verifregdiff/v2/state.go b/pkg/extract/actors/verifregdiff/v2/state.go index b05272bc0..1043ea40c 100644 --- a/pkg/extract/actors/verifregdiff/v2/state.go +++ b/pkg/extract/actors/verifregdiff/v2/state.go @@ -2,47 +2,35 @@ package v2 import ( "context" + "fmt" "github.com/filecoin-project/go-state-types/builtin/v10/util/adt" "github.com/filecoin-project/go-state-types/store" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" - "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/pkg/extract/actors" - v0 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v1" - "github.com/filecoin-project/lily/tasks" ) var log = logging.Logger("extract/actors/verifreg") -type StateDiff struct { - DiffMethods []actors.ActorStateDiff -} - -func (s *StateDiff) State(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorDiffResult, error) { - grp, grpctx := errgroup.WithContext(ctx) - results, err := actors.ExecuteStateDiff(grpctx, grp, api, act, s.DiffMethods...) - if err != nil { - return nil, err - } - +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) - for _, stateChange := range results { - if stateChange == nil { - continue - } - switch stateChange.Kind() { - case v0.KindVerifregVerifiers: - stateDiff.VerifierChanges = stateChange.(VerifiersChangeList) - case KindVerifregClaims: - stateDiff.ClaimChanges = stateChange.(ClaimsChangeMap) - case KindVerifregAllocations: - stateDiff.AllocationsChanges = stateChange.(AllocationsChangeMap) + for _, stateChange := range changes { + switch v := stateChange.(type) { + case VerifiersChangeList: + stateDiff.VerifierChanges = v + case ClaimsChangeMap: + stateDiff.ClaimChanges = v + case AllocationsChangeMap: + stateDiff.AllocationsChanges = v + default: + return nil, fmt.Errorf("unknown state change kind: %T", v) } } return stateDiff, nil + } type StateDiffResult struct { @@ -51,10 +39,6 @@ type StateDiffResult struct { AllocationsChanges AllocationsChangeMap } -func (sd *StateDiffResult) Kind() string { - return "verifreg" -} - func (sd *StateDiffResult) MarshalStateChange(ctx context.Context, store store.Store) (cbg.CBORMarshaler, error) { out := &StateChange{} diff --git a/pkg/extract/actors/verifregdiff/version.go b/pkg/extract/actors/verifregdiff/version.go index 3a9a5fbee..d1ba8c4e1 100644 --- a/pkg/extract/actors/verifregdiff/version.go +++ b/pkg/extract/actors/verifregdiff/version.go @@ -10,20 +10,18 @@ import ( v2 "github.com/filecoin-project/lily/pkg/extract/actors/verifregdiff/v2" ) -func StateDiffFor(av actortypes.Version) (actors.ActorDiff, error) { +func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { if av < actortypes.Version9 { - return &v1.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v1.Clients{}, - v1.Verifiers{}, - }}, nil + return []actors.ActorDiffMethods{ + v1.Clients{}, + v1.Verifiers{}, + }, v1.ActorStateChangeHandler, nil } else if av == actortypes.Version9 { - return &v2.StateDiff{ - DiffMethods: []actors.ActorStateDiff{ - v2.Verifiers{}, - v2.Claims{}, - v2.Allocations{}, - }}, nil + return []actors.ActorDiffMethods{ + v2.Verifiers{}, + v2.Claims{}, + v2.Allocations{}, + }, v2.ActorStateChangeHandler, nil } - return nil, fmt.Errorf("unsupported actor version %d", av) + return nil, nil, fmt.Errorf("unsupported actor version %d", av) } From b97fcd5b3788522ddef8564cf11ae5dea1010baa Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 8 Feb 2023 16:52:11 -0800 Subject: [PATCH 42/48] paralle message extraction --- pkg/extract/chain/messages.go | 46 ++++++++++++++++++++++++++--------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/pkg/extract/chain/messages.go b/pkg/extract/chain/messages.go index 8126a1096..398d0a915 100644 --- a/pkg/extract/chain/messages.go +++ b/pkg/extract/chain/messages.go @@ -7,6 +7,7 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" "github.com/filecoin-project/lotus/chain/types" "github.com/ipfs/go-cid" + "golang.org/x/sync/errgroup" "github.com/filecoin-project/lily/lens" "github.com/filecoin-project/lily/tasks" @@ -129,11 +130,41 @@ type ActorError struct { } func Messages(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet) (map[cid.Cid]*FullBlock, []*ImplicitMessage, error) { - // fist get all messages included in the executed tipset, not all of these messages will have receipts since some were not executed. - blkMsgs, err := api.TipSetBlockMessages(ctx, executed) - if err != nil { + var ( + blkMsgs []*lens.BlockMessages + exeBlkMsgs []*lens.BlockMessageReceipts + msgExe []*lens.MessageExecutionV2 + err error + ) + grp, grpCtx := errgroup.WithContext(ctx) + + grp.Go(func() error { + blkMsgs, err = api.TipSetBlockMessages(grpCtx, executed) + if err != nil { + return err + } + return nil + }) + + grp.Go(func() error { + exeBlkMsgs, err = api.TipSetMessageReceipts(grpCtx, current, executed) + if err != nil { + return err + } + return nil + }) + + grp.Go(func() error { + msgExe, err = api.MessageExecutionsV2(grpCtx, current, executed) + if err != nil { + return err + } + return nil + }) + if err := grp.Wait(); err != nil { return nil, nil, err } + // build two maps containing all signed (secpMsgs) and unsigned (blsMsgs) messages. secpBlkMsgs := make(map[cid.Cid][]*SignedChainMessage) blsBlkMsgs := make(map[cid.Cid][]*ChainMessage) @@ -153,10 +184,6 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type } } - exeBlkMsgs, err := api.TipSetMessageReceipts(ctx, current, executed) - if err != nil { - return nil, nil, err - } for _, ebm := range exeBlkMsgs { itr, err := ebm.Iterator() if err != nil { @@ -180,11 +207,6 @@ func Messages(ctx context.Context, api tasks.DataSource, current, executed *type } } - msgExe, err := api.MessageExecutionsV2(ctx, current, executed) - if err != nil { - return nil, nil, err - } - var im []*ImplicitMessage for _, emsg := range msgExe { vmMsgs, err := ProcessVmMessages(ctx, emsg) From a986e96964f61ef5c7ae77a44e03b84b2317350c Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 8 Feb 2023 19:35:07 -0800 Subject: [PATCH 43/48] can at least exit when there is an error --- pkg/extract/actors.go | 137 +++++++++++++++----------- pkg/extract/actors/executor.go | 4 +- pkg/extract/actors/rawdiff/state.go | 8 +- pkg/extract/actors/rawdiff/version.go | 4 +- pkg/extract/extract.go | 18 +++- 5 files changed, 100 insertions(+), 71 deletions(-) diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index a9a069a7f..a7b100e09 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -94,7 +94,7 @@ func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []add return keys } -func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version) (*ActorStateChanges, error) { +func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version, workers int) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { return nil, err @@ -106,12 +106,13 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } - pool := workerpool.New(8) - results := make(chan *StateDiffResult, len(actorChanges)) + pool := workerpool.New(workers) + results := make(chan *StateDiffResult) wg := sync.WaitGroup{} + workerCtx, cancel := context.WithCancel(ctx) + defer cancel() // sort actors on actor id in ascending order, causes market actor to be differed early, which is the slowest actor to diff. - sortedKeys := sortedActorChangeKeys(actorChanges) - for _, addr := range sortedKeys { + for _, addr := range sortedActorChangeKeys(actorChanges) { addr := addr change := actorChanges[addr] act := &actors.ActorChange{ @@ -122,13 +123,15 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. } wg.Add(1) pool.Submit(func() { - if err := doRawActorDiff(ctx, api, act, actorVersion, results); err != nil { - log.Fatal(err) - } - if err := doActorDiff(ctx, api, act, actorVersion, results); err != nil { - log.Fatal(err) + select { + case <-workerCtx.Done(): + return + default: + } - wg.Done() + defer wg.Done() + doRawActorDiff(workerCtx, api, act, actorVersion, results) + doActorDiff(workerCtx, api, act, actorVersion, results) }) } @@ -136,9 +139,14 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. wg.Wait() close(results) }() + for stateDiff := range results { - if stateDiff.Error != nil { - // TODO bug need to call wg.Done() for remaining open wg functions + if err := stateDiff.Error; err != nil { + // drain any pensing results + for res := range results { + log.Infow("drain result", "actor", res.Address) + } + // stop the pool pool.Stop() return nil, err } @@ -164,28 +172,47 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. return asc, nil } -func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) error { +func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { methods, handler, err := rawdiff.StateDiffFor(version) + if err != nil { + select { + case <-ctx.Done(): + return + default: + results <- &StateDiffResult{ + ActorDiff: nil, + Address: act.Address, + Error: err, + } + } + return + } + actorDiff := &actors.StateDiffer{ - Methods: methods, + Methods: methods, + ActorHandler: handler, ReportHandler: func(reports []actors.DifferReport) error { for _, report := range reports { log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) } return nil }, - ActorHandler: handler, } res, err := actorDiff.ActorDiff(ctx, api, act) - results <- &StateDiffResult{ - ActorDiff: res, - Address: act.Address, - Error: err, + select { + case <-ctx.Done(): + return + default: + results <- &StateDiffResult{ + ActorDiff: res, + Address: act.Address, + Error: err, + } } - return nil + return } -func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) error { +func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { var ( methods []actors.ActorDiffMethods handler actors.ActorHandlerFn @@ -193,60 +220,52 @@ func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorCha ) if core.DataCapCodes.Has(act.Current.Code) { methods, handler, err = datacapdiff.StateDiffFor(version) - if err != nil { - return err - } - } - if core.MinerCodes.Has(act.Current.Code) { + } else if core.MinerCodes.Has(act.Current.Code) { methods, handler, err = minerdiff.StateDiffFor(version) - if err != nil { - return err - } - } - if core.InitCodes.Has(act.Current.Code) { + } else if core.InitCodes.Has(act.Current.Code) { methods, handler, err = initdiff.StateDiffFor(version) - if err != nil { - return err - } - } - if core.PowerCodes.Has(act.Current.Code) { + } else if core.PowerCodes.Has(act.Current.Code) { methods, handler, err = powerdiff.StateDiffFor(version) - if err != nil { - return err - } - } - if core.MarketCodes.Has(act.Current.Code) { + } else if core.MarketCodes.Has(act.Current.Code) { methods, handler, err = marketdiff.StateDiffFor(version) - if err != nil { - return err - } - } - if core.VerifregCodes.Has(act.Current.Code) { + } else if core.VerifregCodes.Has(act.Current.Code) { methods, handler, err = verifregdiff.StateDiffFor(version) - if err != nil { - return err - } + } else { + return } - if methods == nil { - return nil + if err != nil { + select { + case <-ctx.Done(): + return + default: + results <- &StateDiffResult{ + ActorDiff: nil, + Address: act.Address, + Error: err, + } + } } actorDiff := &actors.StateDiffer{ - Methods: methods, + Methods: methods, + ActorHandler: handler, ReportHandler: func(reports []actors.DifferReport) error { for _, report := range reports { log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) } return nil }, - ActorHandler: handler, } res, err := actorDiff.ActorDiff(ctx, api, act) - results <- &StateDiffResult{ - ActorDiff: res, - Address: act.Address, - Error: err, + select { + case <-ctx.Done(): + return + default: + results <- &StateDiffResult{ + ActorDiff: res, + Address: act.Address, + Error: err, + } } - return nil } diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index 1ec4eaa21..4e22ae2a6 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -15,7 +15,7 @@ type DifferReport struct { Result ActorStateChange } -func ExecuteStateDiff(ctx context.Context, api tasks.DataSource, act *ActorChange, fns ...ActorDiffMethods) []DifferReport { +func executeStateDiff(ctx context.Context, api tasks.DataSource, act *ActorChange, fns ...ActorDiffMethods) []DifferReport { out := make([]DifferReport, len(fns)) for i, fn := range fns { start := time.Now() @@ -41,7 +41,7 @@ type ReportHandlerFn = func(reports []DifferReport) error type ActorHandlerFn = func(changes []ActorStateChange) (ActorDiffResult, error) func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) { - reports := ExecuteStateDiff(ctx, api, act, s.Methods...) + reports := executeStateDiff(ctx, api, act, s.Methods...) if s.ReportHandler != nil { if err := s.ReportHandler(reports); err != nil { diff --git a/pkg/extract/actors/rawdiff/state.go b/pkg/extract/actors/rawdiff/state.go index d4fc423ee..30b8ff996 100644 --- a/pkg/extract/actors/rawdiff/state.go +++ b/pkg/extract/actors/rawdiff/state.go @@ -14,11 +14,11 @@ import ( func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { var stateDiff = new(StateDiffResult) for _, change := range changes { - switch change.Kind() { - case KindActorChange: - stateDiff.ActorStateChanges = change.(*ActorChange) + switch v := change.(type) { + case *ActorChange: + stateDiff.ActorStateChanges = v default: - return nil, fmt.Errorf("unhandledd change kind: %s", change.Kind()) + return nil, fmt.Errorf("unhandled change type: %T", v) } } return stateDiff, nil diff --git a/pkg/extract/actors/rawdiff/version.go b/pkg/extract/actors/rawdiff/version.go index 831371e95..229befb1d 100644 --- a/pkg/extract/actors/rawdiff/version.go +++ b/pkg/extract/actors/rawdiff/version.go @@ -1,11 +1,13 @@ package rawdiff import ( + "fmt" + actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" ) func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { - return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, nil + return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, fmt.Errorf("you got an error") } diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go index 0e83a6010..34d765970 100644 --- a/pkg/extract/extract.go +++ b/pkg/extract/extract.go @@ -47,15 +47,23 @@ func State(ctx context.Context, api tasks.DataSource, current, executed *types.T start := time.Now() // all blocks, messages, implicit messages, from executed and receipts from current blockmessages, err = FullBlockMessages(grpCtx, api, current, executed) - log.Infow("extracted fullblock", "duration", time.Since(start)) - return err + if err != nil { + log.Errorw("failed to extract full block messages", "error", err) + return err + } + log.Infow("extracted full block messages", "duration", time.Since(start)) + return nil }) grp.Go(func() error { start := time.Now() // all actor changes between current and parent, actor state exists in current. - actorChanges, err = Actors(grpCtx, api, current, executed, actorVersion) - log.Infow("extracted actors", "duration", time.Since(start)) - return err + actorChanges, err = Actors(grpCtx, api, current, executed, actorVersion, 8) + if err != nil { + log.Errorw("failed to extract actor states", "error", err) + return err + } + log.Infow("extracted actor states", "duration", time.Since(start)) + return nil }) if err := grp.Wait(); err != nil { From 827b2aa5b4bdc80de08996fe3c3da8560f6e236d Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 9 Feb 2023 14:58:51 -0800 Subject: [PATCH 44/48] worker pool works --- pkg/extract/actors.go | 63 ++++++++++++++++++++-------------- pkg/extract/actors/executor.go | 6 ++++ 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index a7b100e09..03376f733 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -94,6 +94,19 @@ func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []add return keys } +func doWork(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { + log.Infow("starting worker", "actor", act.Address) + defer log.Infow("stopping worker", "actor", act.Address) + select { + case <-ctx.Done(): + log.Infow("canceling worker", "error", ctx.Err(), "actor", act.Address) + return + default: + doRawActorDiff(ctx, api, act, version, results) + //doActorDiff(ctx, api, act, version, results) + } +} + func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version, workers int) (*ActorStateChanges, error) { actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) if err != nil { @@ -110,6 +123,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. results := make(chan *StateDiffResult) wg := sync.WaitGroup{} workerCtx, cancel := context.WithCancel(ctx) + activeWorkers := 0 defer cancel() // sort actors on actor id in ascending order, causes market actor to be differed early, which is the slowest actor to diff. for _, addr := range sortedActorChangeKeys(actorChanges) { @@ -122,32 +136,33 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. Type: change.ChangeType, } wg.Add(1) + activeWorkers++ pool.Submit(func() { - select { - case <-workerCtx.Done(): - return - default: - - } defer wg.Done() - doRawActorDiff(workerCtx, api, act, actorVersion, results) - doActorDiff(workerCtx, api, act, actorVersion, results) + doWork(workerCtx, api, act, actorVersion, results) + activeWorkers-- }) } go func() { + log.Info("waiting for workers to complete") wg.Wait() + log.Info("worker completed, closing worker channel") close(results) }() for stateDiff := range results { if err := stateDiff.Error; err != nil { - // drain any pensing results - for res := range results { - log.Infow("drain result", "actor", res.Address) - } + log.Infow("activeworkers", "count", activeWorkers) + log.Info("canceling workers") + cancel() + log.Info("canceled workers") // stop the pool - pool.Stop() + log.Info("stopping worker pool") + pool.StopWait() + log.Info("stopped worker pool") + log.Infow("activeworkers", "count", activeWorkers) + <-results return nil, err } switch v := stateDiff.ActorDiff.(type) { @@ -178,12 +193,11 @@ func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Actor select { case <-ctx.Done(): return - default: - results <- &StateDiffResult{ - ActorDiff: nil, - Address: act.Address, - Error: err, - } + case results <- &StateDiffResult{ + ActorDiff: nil, + Address: act.Address, + Error: err, + }: } return } @@ -202,12 +216,11 @@ func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Actor select { case <-ctx.Done(): return - default: - results <- &StateDiffResult{ - ActorDiff: res, - Address: act.Address, - Error: err, - } + case results <- &StateDiffResult{ + ActorDiff: res, + Address: act.Address, + Error: err, + }: } return } diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index 4e22ae2a6..6ffaf7557 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -4,9 +4,13 @@ import ( "context" "time" + logging "github.com/ipfs/go-log/v2" + "github.com/filecoin-project/lily/tasks" ) +var log = logging.Logger("lily/extract/actors") + type DifferReport struct { DiffType string StartTime time.Time @@ -41,6 +45,8 @@ type ReportHandlerFn = func(reports []DifferReport) error type ActorHandlerFn = func(changes []ActorStateChange) (ActorDiffResult, error) func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) { + log.Info("ActorDiff", "actor", act.Address) + defer log.Infow("DiffedActor", "actor", act.Address) reports := executeStateDiff(ctx, api, act, s.Methods...) if s.ReportHandler != nil { From 4814a89f07666b0f7146170a1f6c7ffd2ba34788 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 9 Feb 2023 16:54:02 -0800 Subject: [PATCH 45/48] okay worker pool works for reals this time --- pkg/extract/actors.go | 211 ++++++++++++++------------ pkg/extract/actors/executor.go | 35 ++--- pkg/extract/actors/rawdiff/version.go | 4 +- pkg/extract/cleanup.go | 1 + 4 files changed, 134 insertions(+), 117 deletions(-) create mode 100644 pkg/extract/cleanup.go diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 03376f733..60671b740 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -68,7 +68,6 @@ func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { type StateDiffResult struct { ActorDiff actors.ActorDiffResult Address address.Address - Error error } type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version @@ -95,16 +94,6 @@ func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []add } func doWork(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { - log.Infow("starting worker", "actor", act.Address) - defer log.Infow("stopping worker", "actor", act.Address) - select { - case <-ctx.Done(): - log.Infow("canceling worker", "error", ctx.Err(), "actor", act.Address) - return - default: - doRawActorDiff(ctx, api, act, version, results) - //doActorDiff(ctx, api, act, version, results) - } } func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version, workers int) (*ActorStateChanges, error) { @@ -119,12 +108,14 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries } + resCh := make(chan *StateDiffResult, workers*2) + errCh := make(chan error, workers*2) + done := make(chan struct{}) + cancel := make(chan struct{}) + scheduledWorkers := 0 + pool := workerpool.New(workers) - results := make(chan *StateDiffResult) wg := sync.WaitGroup{} - workerCtx, cancel := context.WithCancel(ctx) - activeWorkers := 0 - defer cancel() // sort actors on actor id in ascending order, causes market actor to be differed early, which is the slowest actor to diff. for _, addr := range sortedActorChangeKeys(actorChanges) { addr := addr @@ -135,71 +126,120 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. Current: change.Current, Type: change.ChangeType, } + wg.Add(1) - activeWorkers++ + scheduledWorkers++ pool.Submit(func() { - defer wg.Done() - doWork(workerCtx, api, act, actorVersion, results) - activeWorkers-- + defer func() { + wg.Done() + scheduledWorkers-- + + }() + res, err := doRawActorDiff(ctx, api, act, actorVersion) + if err != nil { + select { + case <-cancel: + return + case errCh <- err: + return + } + } + select { + case <-cancel: + return + case resCh <- res: + return + } }) + wg.Add(1) + scheduledWorkers++ + pool.Submit(func() { + defer func() { + wg.Done() + scheduledWorkers-- + + }() + res, ok, err := doActorDiff(ctx, api, act, actorVersion) + if err != nil { + select { + case <-cancel: + log.Info("worker canceled") + return + case errCh <- err: + return + } + } + if ok { + select { + case <-cancel: + log.Info("worker canceled") + return + case resCh <- res: + return + } + } + log.Infow("no actor diff for actor", "address", act.Address) + + }) } + go func() { - log.Info("waiting for workers to complete") wg.Wait() - log.Info("worker completed, closing worker channel") - close(results) + done <- struct{}{} + log.Info("killed go routine") }() - for stateDiff := range results { - if err := stateDiff.Error; err != nil { - log.Infow("activeworkers", "count", activeWorkers) - log.Info("canceling workers") - cancel() - log.Info("canceled workers") - // stop the pool - log.Info("stopping worker pool") - pool.StopWait() - log.Info("stopped worker pool") - log.Infow("activeworkers", "count", activeWorkers) - <-results + for { + select { + case <-ctx.Done(): + log.Info("context done") + + case <-done: + log.Info("done processing") + return asc, nil + + case err := <-errCh: + log.Infow("worker received error while processing", "error", err) + cancel <- struct{}{} + log.Info("canceled reaming workers") + pool.Stop() + log.Info("stopped pool") + for i := 0; i < scheduledWorkers; i++ { + wg.Done() + } + log.Info("cleanup go routine") + <-done return nil, err - } - switch v := stateDiff.ActorDiff.(type) { - case *rawdiff.StateDiffResult: - asc.RawActors[stateDiff.Address] = v - case *minerDiffV1.StateDiffResult: - asc.MinerActors[stateDiff.Address] = v - case *initDiffV1.StateDiffResult: - asc.InitActor = v - case *powerDiffV1.StateDiffResult: - asc.PowerActor = v - case *marketDiffV1.StateDiffResult: - asc.MarketActor = v - case *dcapDiffV1.StateDiffResult: - asc.DatacapActor = v - case *verifDiffV1.StateDiffResult, *verifDiffV2.StateDiffResult: - asc.VerifregActor = v - default: - return nil, fmt.Errorf("unknown StateDiffResult type: %T", v) + + case res := <-resCh: + switch v := res.ActorDiff.(type) { + case *rawdiff.StateDiffResult: + asc.RawActors[res.Address] = v + case *minerDiffV1.StateDiffResult: + asc.MinerActors[res.Address] = v + case *initDiffV1.StateDiffResult: + asc.InitActor = v + case *powerDiffV1.StateDiffResult: + asc.PowerActor = v + case *marketDiffV1.StateDiffResult: + asc.MarketActor = v + case *dcapDiffV1.StateDiffResult: + asc.DatacapActor = v + case *verifDiffV1.StateDiffResult, *verifDiffV2.StateDiffResult: + asc.VerifregActor = v + default: + panic(fmt.Errorf("unknown StateDiffResult type: %T", v)) + } } } - return asc, nil + } -func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { +func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version) (*StateDiffResult, error) { methods, handler, err := rawdiff.StateDiffFor(version) if err != nil { - select { - case <-ctx.Done(): - return - case results <- &StateDiffResult{ - ActorDiff: nil, - Address: act.Address, - Error: err, - }: - } - return + return nil, err } actorDiff := &actors.StateDiffer{ @@ -213,19 +253,16 @@ func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Actor }, } res, err := actorDiff.ActorDiff(ctx, api, act) - select { - case <-ctx.Done(): - return - case results <- &StateDiffResult{ + if err != nil { + return nil, err + } + return &StateDiffResult{ ActorDiff: res, Address: act.Address, - Error: err, - }: - } - return + }, nil } -func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { +func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version) (*StateDiffResult, bool, error) { var ( methods []actors.ActorDiffMethods handler actors.ActorHandlerFn @@ -244,20 +281,11 @@ func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorCha } else if core.VerifregCodes.Has(act.Current.Code) { methods, handler, err = verifregdiff.StateDiffFor(version) } else { - return + return nil, false, nil } if err != nil { - select { - case <-ctx.Done(): - return - default: - results <- &StateDiffResult{ - ActorDiff: nil, - Address: act.Address, - Error: err, - } - } + return nil, false, nil } actorDiff := &actors.StateDiffer{ @@ -271,14 +299,11 @@ func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorCha }, } res, err := actorDiff.ActorDiff(ctx, api, act) - select { - case <-ctx.Done(): - return - default: - results <- &StateDiffResult{ - ActorDiff: res, - Address: act.Address, - Error: err, - } + if err != nil { + return nil, false, err } + return &StateDiffResult{ + ActorDiff: res, + Address: act.Address, + }, true, nil } diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index 6ffaf7557..f4baa7b92 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -19,22 +19,6 @@ type DifferReport struct { Result ActorStateChange } -func executeStateDiff(ctx context.Context, api tasks.DataSource, act *ActorChange, fns ...ActorDiffMethods) []DifferReport { - out := make([]DifferReport, len(fns)) - for i, fn := range fns { - start := time.Now() - res, err := fn.Diff(ctx, api, act) - out[i] = DifferReport{ - DiffType: fn.Type(), - StartTime: start, - Duration: time.Since(start), - Error: err, - Result: res, - } - } - return out -} - type StateDiffer struct { Methods []ActorDiffMethods ReportHandler ReportHandlerFn @@ -45,18 +29,27 @@ type ReportHandlerFn = func(reports []DifferReport) error type ActorHandlerFn = func(changes []ActorStateChange) (ActorDiffResult, error) func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) { - log.Info("ActorDiff", "actor", act.Address) - defer log.Infow("DiffedActor", "actor", act.Address) - reports := executeStateDiff(ctx, api, act, s.Methods...) + out := make([]DifferReport, len(s.Methods)) + for i, fn := range s.Methods { + start := time.Now() + res, err := fn.Diff(ctx, api, act) + out[i] = DifferReport{ + DiffType: fn.Type(), + StartTime: start, + Duration: time.Since(start), + Error: err, + Result: res, + } + } if s.ReportHandler != nil { - if err := s.ReportHandler(reports); err != nil { + if err := s.ReportHandler(out); err != nil { return nil, err } } var results []ActorStateChange - for _, report := range reports { + for _, report := range out { results = append(results, report.Result) } return s.ActorHandler(results) diff --git a/pkg/extract/actors/rawdiff/version.go b/pkg/extract/actors/rawdiff/version.go index 229befb1d..831371e95 100644 --- a/pkg/extract/actors/rawdiff/version.go +++ b/pkg/extract/actors/rawdiff/version.go @@ -1,13 +1,11 @@ package rawdiff import ( - "fmt" - actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/lily/pkg/extract/actors" ) func StateDiffFor(av actortypes.Version) ([]actors.ActorDiffMethods, actors.ActorHandlerFn, error) { - return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, fmt.Errorf("you got an error") + return []actors.ActorDiffMethods{Actor{}}, ActorStateChangeHandler, nil } diff --git a/pkg/extract/cleanup.go b/pkg/extract/cleanup.go new file mode 100644 index 000000000..7e3d3b908 --- /dev/null +++ b/pkg/extract/cleanup.go @@ -0,0 +1 @@ +package extract From 15b9107f5dcfa70002acab9b4f669aeb55d97d52 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 9 Feb 2023 17:53:58 -0800 Subject: [PATCH 46/48] polish actor workers --- pkg/extract/actors.go | 221 ++++++++++-------- .../actors/datacapdiff/v1/allowances.go | 6 +- pkg/extract/actors/datacapdiff/v1/balances.go | 4 +- pkg/extract/actors/datacapdiff/v1/state.go | 2 +- pkg/extract/actors/executor.go | 4 +- pkg/extract/actors/generic/array.go | 2 +- pkg/extract/actors/generic/map.go | 2 +- pkg/extract/actors/initdiff/v1/addresses.go | 4 +- pkg/extract/actors/initdiff/v1/state.go | 2 +- pkg/extract/actors/marketdiff/v1/deals.go | 4 +- pkg/extract/actors/marketdiff/v1/proposals.go | 4 +- pkg/extract/actors/marketdiff/v1/state.go | 2 +- pkg/extract/actors/minerdiff/v1/infos.go | 4 +- pkg/extract/actors/minerdiff/v1/precommits.go | 4 +- .../actors/minerdiff/v1/sector_status.go | 2 +- pkg/extract/actors/minerdiff/v1/sectors.go | 4 +- pkg/extract/actors/minerdiff/v1/state.go | 2 +- pkg/extract/actors/powerdiff/v1/claims.go | 4 +- pkg/extract/actors/powerdiff/v1/state.go | 2 +- pkg/extract/actors/rawdiff/actor.go | 4 +- pkg/extract/actors/rawdiff/state.go | 2 +- pkg/extract/actors/types.go | 26 ++- pkg/extract/actors/verifregdiff/v1/clients.go | 4 +- pkg/extract/actors/verifregdiff/v1/state.go | 2 +- .../actors/verifregdiff/v1/verifiers.go | 4 +- .../actors/verifregdiff/v2/allocations.go | 6 +- pkg/extract/actors/verifregdiff/v2/claims.go | 6 +- pkg/extract/actors/verifregdiff/v2/state.go | 2 +- pkg/extract/extract.go | 3 +- pkg/transform/cbor/actors/serialize.go | 4 +- 30 files changed, 189 insertions(+), 153 deletions(-) diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 60671b740..74f92e7cf 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -3,11 +3,9 @@ package extract import ( "context" "fmt" - "sort" "sync" "github.com/filecoin-project/go-address" - "github.com/filecoin-project/go-state-types/abi" actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/chain/types" @@ -41,13 +39,13 @@ var log = logging.Logger("lily/extract/processor") type ActorStateChanges struct { Current *types.TipSet Executed *types.TipSet - RawActors map[address.Address]actors.ActorDiffResult - MinerActors map[address.Address]actors.ActorDiffResult - DatacapActor actors.ActorDiffResult - InitActor actors.ActorDiffResult - MarketActor actors.ActorDiffResult - PowerActor actors.ActorDiffResult - VerifregActor actors.ActorDiffResult + RawActors map[address.Address]actors.DiffResult + MinerActors map[address.Address]actors.DiffResult + DatacapActor actors.DiffResult + InitActor actors.DiffResult + MarketActor actors.DiffResult + PowerActor actors.DiffResult + VerifregActor actors.DiffResult } func (a ActorStateChanges) Attributes() []attribute.KeyValue { @@ -66,67 +64,72 @@ func (a ActorStateChanges) MarshalLogObject(enc zapcore.ObjectEncoder) error { } type StateDiffResult struct { - ActorDiff actors.ActorDiffResult - Address address.Address + ActorDiff actors.DiffResult + Actor *actors.Change } -type NetworkVersionGetter = func(ctx context.Context, epoch abi.ChainEpoch) network.Version - -func sortedActorChangeKeys(actors map[address.Address]statetree.ActorDiff) []address.Address { - keys := make([]address.Address, 0, len(actors)) - - for k := range actors { - keys = append(keys, k) +func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, workers int) (*ActorStateChanges, error) { + actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) + if err != nil { + return nil, err } - sort.Slice(keys, func(i, j int) bool { - iKey, err := address.IDFromAddress(keys[i]) - if err != nil { - panic(err) - } - jKey, err := address.IDFromAddress(keys[j]) - if err != nil { - panic(err) - } - return iKey < jKey - }) - - return keys -} - -func doWork(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version, results chan *StateDiffResult) { -} -func Actors(ctx context.Context, api tasks.DataSource, current, executed *types.TipSet, actorVersion actortypes.Version, workers int) (*ActorStateChanges, error) { - actorChanges, err := statetree.ActorChanges(ctx, api.Store(), current, executed) + curNtwkVersion, err := api.NetworkVersion(ctx, current.Key()) if err != nil { return nil, err } - asc := &ActorStateChanges{ - Current: current, - Executed: executed, - MinerActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries - RawActors: make(map[address.Address]actors.ActorDiffResult, len(actorChanges)), // there are at most actorChanges entries + exeNtwkVersion, err := api.NetworkVersion(ctx, executed.Key()) + if err != nil { + return nil, err } + curActorVersion, err := actortypes.VersionForNetwork(network.Version(curNtwkVersion)) + if err != nil { + return nil, err + } + exeActorVersion, err := actortypes.VersionForNetwork(network.Version(exeNtwkVersion)) + if err != nil { + return nil, err + } + log.Infow("extract actor states", + "current_network_version", curNtwkVersion, + "current_actor_version", curActorVersion, + "current_height", current.Height(), + "executed_network_version", exeNtwkVersion, + "executed_actor_version", exeActorVersion, + "executed_height", executed.Height(), + "workers", workers) - resCh := make(chan *StateDiffResult, workers*2) - errCh := make(chan error, workers*2) - done := make(chan struct{}) - cancel := make(chan struct{}) - scheduledWorkers := 0 + var ( + // pool is the worker pool used to manage workers + pool = workerpool.New(workers) + // wg is the wait-group used to synchronize workers + wg = sync.WaitGroup{} + // resCh is the channel actor diff workers return results on + resCh = make(chan *StateDiffResult, workers*2) + // errCh is the channel actor diff workers return errors on + errCh = make(chan error, workers*2) + // done is the channel used for signaling workers have stopped execution + done = make(chan struct{}) + // cancel is the channel used to cancel any active workers + cancel = make(chan struct{}) + // scheduledWorkers is the number of workers scheduled for execution, its value decreases as workers complete + scheduledWorkers = 0 + // workerCtx is the context used by the workers. + // TODO a method deep in the call stack of this function is not respecting context cancellation + workerCtx = context.TODO() + ) - pool := workerpool.New(workers) - wg := sync.WaitGroup{} - // sort actors on actor id in ascending order, causes market actor to be differed early, which is the slowest actor to diff. - for _, addr := range sortedActorChangeKeys(actorChanges) { - addr := addr - change := actorChanges[addr] - act := &actors.ActorChange{ - Address: addr, - Executed: change.Executed, - Current: change.Current, - Type: change.ChangeType, + for addr, change := range actorChanges { + act := &actors.Change{ + Address: addr, + Executed: change.Executed, + ExeVersion: exeActorVersion, + Current: change.Current, + CurVersion: curActorVersion, + Type: change.ChangeType, } + // submit two workers, one to extract raw actor states and one to diff individual actor states wg.Add(1) scheduledWorkers++ pool.Submit(func() { @@ -135,8 +138,9 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. scheduledWorkers-- }() - res, err := doRawActorDiff(ctx, api, act, actorVersion) + res, err := diffRawActorState(workerCtx, api, act) if err != nil { + // attempt to send the error or bail if canceled select { case <-cancel: return @@ -144,6 +148,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. return } } + // attempt to send the result or bail if canceled select { case <-cancel: return @@ -160,75 +165,96 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. scheduledWorkers-- }() - res, ok, err := doActorDiff(ctx, api, act, actorVersion) + res, ok, err := diffTypedActorState(workerCtx, api, act) if err != nil { + // attempt to send the error or bail if canceled select { case <-cancel: - log.Info("worker canceled") return case errCh <- err: return } } if ok { + // attempt to send the result or bail if canceled select { case <-cancel: - log.Info("worker canceled") return case resCh <- res: return } } - log.Infow("no actor diff for actor", "address", act.Address) + // Not all actors have their state diffed, for example account actors have no state to diff: their state is empty. + log.Debugw("no actor diff for actor", "current_network", curNtwkVersion, "executed_network", exeNtwkVersion, + "address", act.Address, "current_code", act.Current.Code, "current_version", + act.CurVersion, "executed_code", act.Executed.Code, "executed_version", act.ExeVersion) }) } + // wait for workers to complete then signal work is done. go func() { wg.Wait() done <- struct{}{} - log.Info("killed go routine") }() + // stop the worker pool, dropping any scheduled workers, and complete any wait-groups for the case a workers were canceled. + cleanup := func() { + pool.Stop() + for i := 0; i < scheduledWorkers; i++ { + wg.Done() + } + } + + // result of diffing all actor states + out := &ActorStateChanges{ + Current: current, + Executed: executed, + MinerActors: make(map[address.Address]actors.DiffResult, len(actorChanges)), // there are at most actorChanges entries + RawActors: make(map[address.Address]actors.DiffResult, len(actorChanges)), // there are at most actorChanges entries + } for { + log.Infof("diff jobs todo %d", scheduledWorkers) select { - case <-ctx.Done(): - log.Info("context done") - - case <-done: - log.Info("done processing") - return asc, nil - + // canceling the context or receiving an error causes all workers to stop and drops any scheduled workers. + case <-workerCtx.Done(): + log.Errorw("context canceled", "error", workerCtx.Err()) + cancel <- struct{}{} + cleanup() + <-done + return nil, workerCtx.Err() case err := <-errCh: log.Infow("worker received error while processing", "error", err) cancel <- struct{}{} - log.Info("canceled reaming workers") - pool.Stop() - log.Info("stopped pool") - for i := 0; i < scheduledWorkers; i++ { - wg.Done() - } - log.Info("cleanup go routine") + cleanup() <-done return nil, err + // happy path, all workers completed and we can return with a result + case <-done: + log.Info("done processing") + cleanup() + return out, nil + + // result returned from a worker corresponding to the actor it diffed. case res := <-resCh: switch v := res.ActorDiff.(type) { case *rawdiff.StateDiffResult: - asc.RawActors[res.Address] = v + out.RawActors[res.Actor.Address] = v case *minerDiffV1.StateDiffResult: - asc.MinerActors[res.Address] = v + out.MinerActors[res.Actor.Address] = v case *initDiffV1.StateDiffResult: - asc.InitActor = v + out.InitActor = v case *powerDiffV1.StateDiffResult: - asc.PowerActor = v + out.PowerActor = v case *marketDiffV1.StateDiffResult: - asc.MarketActor = v + out.MarketActor = v case *dcapDiffV1.StateDiffResult: - asc.DatacapActor = v + out.DatacapActor = v case *verifDiffV1.StateDiffResult, *verifDiffV2.StateDiffResult: - asc.VerifregActor = v + out.VerifregActor = v default: + // this indicates a developer error so it's okay to panic, the code is wrong and must be fixed. panic(fmt.Errorf("unknown StateDiffResult type: %T", v)) } } @@ -236,8 +262,9 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. } -func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version) (*StateDiffResult, error) { - methods, handler, err := rawdiff.StateDiffFor(version) +// diffRawActorState is called on every actor with a state change and returns their state. +func diffRawActorState(ctx context.Context, api tasks.DataSource, act *actors.Change) (*StateDiffResult, error) { + methods, handler, err := rawdiff.StateDiffFor(act.CurVersion) if err != nil { return nil, err } @@ -258,28 +285,30 @@ func doRawActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Actor } return &StateDiffResult{ ActorDiff: res, - Address: act.Address, + Actor: act, }, nil } -func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, version actortypes.Version) (*StateDiffResult, bool, error) { +// diffTypedActorState is called on every actor with a state change and returns artifacts from diffing their internal state +// such as miner hamts, market amts, etc. It returns false if the actor `act` doesn't have a corresponding state diff implementation. +func diffTypedActorState(ctx context.Context, api tasks.DataSource, act *actors.Change) (*StateDiffResult, bool, error) { var ( methods []actors.ActorDiffMethods handler actors.ActorHandlerFn err error ) if core.DataCapCodes.Has(act.Current.Code) { - methods, handler, err = datacapdiff.StateDiffFor(version) + methods, handler, err = datacapdiff.StateDiffFor(act.CurVersion) } else if core.MinerCodes.Has(act.Current.Code) { - methods, handler, err = minerdiff.StateDiffFor(version) + methods, handler, err = minerdiff.StateDiffFor(act.CurVersion) } else if core.InitCodes.Has(act.Current.Code) { - methods, handler, err = initdiff.StateDiffFor(version) + methods, handler, err = initdiff.StateDiffFor(act.CurVersion) } else if core.PowerCodes.Has(act.Current.Code) { - methods, handler, err = powerdiff.StateDiffFor(version) + methods, handler, err = powerdiff.StateDiffFor(act.CurVersion) } else if core.MarketCodes.Has(act.Current.Code) { - methods, handler, err = marketdiff.StateDiffFor(version) + methods, handler, err = marketdiff.StateDiffFor(act.CurVersion) } else if core.VerifregCodes.Has(act.Current.Code) { - methods, handler, err = verifregdiff.StateDiffFor(version) + methods, handler, err = verifregdiff.StateDiffFor(act.CurVersion) } else { return nil, false, nil } @@ -304,6 +333,6 @@ func doActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorCha } return &StateDiffResult{ ActorDiff: res, - Address: act.Address, + Actor: act, }, true, nil } diff --git a/pkg/extract/actors/datacapdiff/v1/allowances.go b/pkg/extract/actors/datacapdiff/v1/allowances.go index 524b89ef4..36edf6fb6 100644 --- a/pkg/extract/actors/datacapdiff/v1/allowances.go +++ b/pkg/extract/actors/datacapdiff/v1/allowances.go @@ -68,7 +68,7 @@ func (a Allowance) Type() string { return KindDataCapAllowance } -func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindDataCapAllowance, zap.Inline(act), "duration", time.Since(start)) @@ -76,7 +76,7 @@ func (Allowance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act return DiffAllowances(ctx, api, act) } -func DiffAllowances(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (AllowanceChangeMap, error) { +func DiffAllowances(ctx context.Context, api tasks.DataSource, act *actors.Change) (AllowanceChangeMap, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapAllowancesMapLoader) if err != nil { return nil, err @@ -100,7 +100,7 @@ func DiffAllowances(ctx context.Context, api tasks.DataSource, act *actors.Actor return out, nil } -func diffOwnerMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, ownerAddress address.Address, ownerKey []byte) ([]*AllowanceChange, error) { +func diffOwnerMap(ctx context.Context, api tasks.DataSource, act *actors.Change, ownerAddress address.Address, ownerKey []byte) ([]*AllowanceChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { datacapState := i.(datacap.State) clientAllocationMap, err := datacapState.AllowanceMapForOwner(ownerAddress) diff --git a/pkg/extract/actors/datacapdiff/v1/balances.go b/pkg/extract/actors/datacapdiff/v1/balances.go index 7ad45a94f..eca1116a6 100644 --- a/pkg/extract/actors/datacapdiff/v1/balances.go +++ b/pkg/extract/actors/datacapdiff/v1/balances.go @@ -53,7 +53,7 @@ func (b Balance) Type() string { return KindDataCapBalance } -func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindDataCapBalance, zap.Inline(act), "duration", time.Since(start)) @@ -61,7 +61,7 @@ func (Balance) Diff(ctx context.Context, api tasks.DataSource, act *actors.Actor return DiffBalances(ctx, api, act) } -func DiffBalances(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (BalanceChangeList, error) { +func DiffBalances(ctx context.Context, api tasks.DataSource, act *actors.Change) (BalanceChangeList, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, DatacapStateLoader, DatacapBalancesMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/datacapdiff/v1/state.go b/pkg/extract/actors/datacapdiff/v1/state.go index de48293cf..52d0f8423 100644 --- a/pkg/extract/actors/datacapdiff/v1/state.go +++ b/pkg/extract/actors/datacapdiff/v1/state.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index f4baa7b92..154013982 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -26,9 +26,9 @@ type StateDiffer struct { } type ReportHandlerFn = func(reports []DifferReport) error -type ActorHandlerFn = func(changes []ActorStateChange) (ActorDiffResult, error) +type ActorHandlerFn = func(changes []ActorStateChange) (DiffResult, error) -func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) { +func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act *Change) (DiffResult, error) { out := make([]DifferReport, len(s.Methods)) for i, fn := range s.Methods { start := time.Now() diff --git a/pkg/extract/actors/generic/array.go b/pkg/extract/actors/generic/array.go index 75b706b6c..ed22e70fe 100644 --- a/pkg/extract/actors/generic/array.go +++ b/pkg/extract/actors/generic/array.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/tasks" ) -func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (core.ArrayModifications, error) { +func DiffActorArray(ctx context.Context, api tasks.DataSource, act *actors.Change, actorStateLoader ActorStateLoader, actorArrayLoader ActorStateArrayLoader) (core.ArrayModifications, error) { if act.Type == core.ChangeTypeRemove { executedActor, err := actorStateLoader(api.Store(), act.Executed) if err != nil { diff --git a/pkg/extract/actors/generic/map.go b/pkg/extract/actors/generic/map.go index 96c0d1537..14d32f96b 100644 --- a/pkg/extract/actors/generic/map.go +++ b/pkg/extract/actors/generic/map.go @@ -10,7 +10,7 @@ import ( "github.com/filecoin-project/lily/tasks" ) -func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (core.MapModifications, error) { +func DiffActorMap(ctx context.Context, api tasks.DataSource, act *actors.Change, actorStateLoader ActorStateLoader, actorMapLoader ActorStateMapLoader) (core.MapModifications, error) { if act.Type == core.ChangeTypeRemove { executedActor, err := actorStateLoader(api.Store(), act.Executed) if err != nil { diff --git a/pkg/extract/actors/initdiff/v1/addresses.go b/pkg/extract/actors/initdiff/v1/addresses.go index 68824c8d7..b99f3c7ed 100644 --- a/pkg/extract/actors/initdiff/v1/addresses.go +++ b/pkg/extract/actors/initdiff/v1/addresses.go @@ -55,7 +55,7 @@ func (a Addresses) Type() string { return KindInitAddresses } -func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindInitAddresses, zap.Inline(act), "duration", time.Since(start)) @@ -63,7 +63,7 @@ func (Addresses) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act return AddressesDiff(ctx, api, act) } -func AddressesDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func AddressesDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, InitStateLoader, InitAddressesMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/initdiff/v1/state.go b/pkg/extract/actors/initdiff/v1/state.go index 51996010c..72796789b 100644 --- a/pkg/extract/actors/initdiff/v1/state.go +++ b/pkg/extract/actors/initdiff/v1/state.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/marketdiff/v1/deals.go b/pkg/extract/actors/marketdiff/v1/deals.go index daef1e813..933646aa8 100644 --- a/pkg/extract/actors/marketdiff/v1/deals.go +++ b/pkg/extract/actors/marketdiff/v1/deals.go @@ -51,7 +51,7 @@ func (d Deals) Type() string { return KindMarketDeal } -func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMarketDeal, zap.Inline(act), "duration", time.Since(start)) @@ -59,7 +59,7 @@ func (Deals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorCh return DiffDeals(ctx, api, act) } -func DiffDeals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffDeals(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketDealsArrayLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/marketdiff/v1/proposals.go b/pkg/extract/actors/marketdiff/v1/proposals.go index 07ccbf154..c4d4ba52d 100644 --- a/pkg/extract/actors/marketdiff/v1/proposals.go +++ b/pkg/extract/actors/marketdiff/v1/proposals.go @@ -58,7 +58,7 @@ func (p Proposals) Type() string { return KindMarketProposal } -func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMarketProposal, zap.Inline(act), "duration", time.Since(start)) @@ -66,7 +66,7 @@ func (Proposals) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act return DiffProposals(ctx, api, act) } -func DiffProposals(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffProposals(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { arrayChanges, err := generic.DiffActorArray(ctx, api, act, MarketStateLoader, MarketProposlasArrayLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/marketdiff/v1/state.go b/pkg/extract/actors/marketdiff/v1/state.go index 73aa88d87..9f6899d83 100644 --- a/pkg/extract/actors/marketdiff/v1/state.go +++ b/pkg/extract/actors/marketdiff/v1/state.go @@ -13,7 +13,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/minerdiff/v1/infos.go b/pkg/extract/actors/minerdiff/v1/infos.go index fa0909544..b83364526 100644 --- a/pkg/extract/actors/minerdiff/v1/infos.go +++ b/pkg/extract/actors/minerdiff/v1/infos.go @@ -38,7 +38,7 @@ func (i Info) Type() string { return KindMinerInfo } -func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Info) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMinerInfo, zap.Inline(act), "duration", time.Since(start)) @@ -55,7 +55,7 @@ type DiffInfoAPI interface { // separate method for testing purposes -func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.ActorChange) (*InfoChange, error) { +func InfoDiff(ctx context.Context, api DiffInfoAPI, act *actors.Change) (*InfoChange, error) { // was removed, no new info if act.Type == core.ChangeTypeRemove { return nil, nil diff --git a/pkg/extract/actors/minerdiff/v1/precommits.go b/pkg/extract/actors/minerdiff/v1/precommits.go index 27abe7765..694bcffb2 100644 --- a/pkg/extract/actors/minerdiff/v1/precommits.go +++ b/pkg/extract/actors/minerdiff/v1/precommits.go @@ -59,7 +59,7 @@ func (c PreCommit) Type() string { return KindMinerPreCommit } -func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMinerPreCommit, zap.Inline(act), "duration", time.Since(start)) @@ -67,7 +67,7 @@ func (PreCommit) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act return PreCommitDiff(ctx, api, act) } -func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func PreCommitDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, MinerStateLoader, MinerPreCommitMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/minerdiff/v1/sector_status.go b/pkg/extract/actors/minerdiff/v1/sector_status.go index 50590860e..49ee1b81e 100644 --- a/pkg/extract/actors/minerdiff/v1/sector_status.go +++ b/pkg/extract/actors/minerdiff/v1/sector_status.go @@ -40,7 +40,7 @@ func (s SectorStatus) Type() string { return KindMinerSectorStatus } -func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (SectorStatus) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMinerSectorStatus, zap.Inline(act), "duration", time.Since(start)) diff --git a/pkg/extract/actors/minerdiff/v1/sectors.go b/pkg/extract/actors/minerdiff/v1/sectors.go index fca4f1eeb..1a755e6ec 100644 --- a/pkg/extract/actors/minerdiff/v1/sectors.go +++ b/pkg/extract/actors/minerdiff/v1/sectors.go @@ -52,7 +52,7 @@ func (s Sectors) Type() string { return KindMinerSector } -func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindMinerSector, zap.Inline(act), "duration", time.Since(start)) @@ -60,7 +60,7 @@ func (Sectors) Diff(ctx context.Context, api tasks.DataSource, act *actors.Actor return DiffSectors(ctx, api, act) } -func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffSectors(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { arrayChange, err := generic.DiffActorArray(ctx, api, act, MinerStateLoader, MinerSectorArrayLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/minerdiff/v1/state.go b/pkg/extract/actors/minerdiff/v1/state.go index 565eeb27d..ff69bbfb8 100644 --- a/pkg/extract/actors/minerdiff/v1/state.go +++ b/pkg/extract/actors/minerdiff/v1/state.go @@ -15,7 +15,7 @@ import ( var log = logging.Logger("extract/actors/miner") -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/powerdiff/v1/claims.go b/pkg/extract/actors/powerdiff/v1/claims.go index e3f662f05..124eaca89 100644 --- a/pkg/extract/actors/powerdiff/v1/claims.go +++ b/pkg/extract/actors/powerdiff/v1/claims.go @@ -52,7 +52,7 @@ func (c Claims) Type() string { return KindPowerClaims } -func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindPowerClaims, zap.Inline(act), "duration", time.Since(start)) @@ -60,7 +60,7 @@ func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorC return DiffClaims(ctx, api, act) } -func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, PowerStateLoader, PowerClaimsMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/powerdiff/v1/state.go b/pkg/extract/actors/powerdiff/v1/state.go index ed1857535..90a71a887 100644 --- a/pkg/extract/actors/powerdiff/v1/state.go +++ b/pkg/extract/actors/powerdiff/v1/state.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/rawdiff/actor.go b/pkg/extract/actors/rawdiff/actor.go index 47c4aacb6..55bd19847 100644 --- a/pkg/extract/actors/rawdiff/actor.go +++ b/pkg/extract/actors/rawdiff/actor.go @@ -31,7 +31,7 @@ func (a *ActorChange) Kind() actors.ActorStateKind { type Actor struct{} -func (Actor) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Actor) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindActorChange, zap.Inline(act), "duration", time.Since(start)) @@ -43,7 +43,7 @@ func (Actor) Type() string { return KindActorChange } -func ActorDiff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func ActorDiff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { switch act.Type { case core.ChangeTypeAdd: currentState, err := api.ChainReadObj(ctx, act.Current.Head) diff --git a/pkg/extract/actors/rawdiff/state.go b/pkg/extract/actors/rawdiff/state.go index 30b8ff996..7d5f0998e 100644 --- a/pkg/extract/actors/rawdiff/state.go +++ b/pkg/extract/actors/rawdiff/state.go @@ -11,7 +11,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, change := range changes { switch v := change.(type) { diff --git a/pkg/extract/actors/types.go b/pkg/extract/actors/types.go index 4d3518f78..441547669 100644 --- a/pkg/extract/actors/types.go +++ b/pkg/extract/actors/types.go @@ -5,6 +5,7 @@ import ( "time" "github.com/filecoin-project/go-address" + actortypes "github.com/filecoin-project/go-state-types/actors" "github.com/filecoin-project/go-state-types/store" "github.com/filecoin-project/lotus/chain/types" typegen "github.com/whyrusleeping/cbor-gen" @@ -29,16 +30,16 @@ type ExtractionReport struct { Duration time.Duration // how long the differ took to run } -type ActorDiffResult interface { +type DiffResult interface { MarshalStateChange(ctx context.Context, s store.Store) (typegen.CBORMarshaler, error) } type ActorDiff interface { - State(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorDiffResult, error) + State(ctx context.Context, api tasks.DataSource, act *Change) (DiffResult, error) } type ActorDiffMethods interface { - Diff(ctx context.Context, api tasks.DataSource, act *ActorChange) (ActorStateChange, error) + Diff(ctx context.Context, api tasks.DataSource, act *Change) (ActorStateChange, error) Type() string } @@ -48,14 +49,19 @@ type ActorStateChange interface { Kind() ActorStateKind } -type ActorChange struct { - Address address.Address - Executed *types.Actor - Current *types.Actor - Type core.ChangeType +type Change struct { + Address address.Address + + Executed *types.Actor + ExeVersion actortypes.Version + + Current *types.Actor + CurVersion actortypes.Version + + Type core.ChangeType } -func (a ActorChange) Attributes() []attribute.KeyValue { +func (a Change) Attributes() []attribute.KeyValue { return []attribute.KeyValue{ attribute.String("address", a.Address.String()), attribute.String("type", builtin.ActorNameByCode(a.Current.Code)), @@ -63,7 +69,7 @@ func (a ActorChange) Attributes() []attribute.KeyValue { } } -func (a ActorChange) MarshalLogObject(enc zapcore.ObjectEncoder) error { +func (a Change) MarshalLogObject(enc zapcore.ObjectEncoder) error { for _, a := range a.Attributes() { enc.AddString(string(a.Key), a.Value.Emit()) } diff --git a/pkg/extract/actors/verifregdiff/v1/clients.go b/pkg/extract/actors/verifregdiff/v1/clients.go index f10a8c2b4..50cf1bc9e 100644 --- a/pkg/extract/actors/verifregdiff/v1/clients.go +++ b/pkg/extract/actors/verifregdiff/v1/clients.go @@ -59,7 +59,7 @@ func (c Clients) Type() string { return KindVerifregClients } -func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindVerifregClients, zap.Inline(act), "duration", time.Since(start)) @@ -67,7 +67,7 @@ func (Clients) Diff(ctx context.Context, api tasks.DataSource, act *actors.Actor return DiffClients(ctx, api, act) } -func DiffClients(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffClients(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregClientsMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/verifregdiff/v1/state.go b/pkg/extract/actors/verifregdiff/v1/state.go index e708a8b01..6712f2c43 100644 --- a/pkg/extract/actors/verifregdiff/v1/state.go +++ b/pkg/extract/actors/verifregdiff/v1/state.go @@ -12,7 +12,7 @@ import ( "github.com/filecoin-project/lily/pkg/extract/actors" ) -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/actors/verifregdiff/v1/verifiers.go b/pkg/extract/actors/verifregdiff/v1/verifiers.go index 5d794c59a..a930cb985 100644 --- a/pkg/extract/actors/verifregdiff/v1/verifiers.go +++ b/pkg/extract/actors/verifregdiff/v1/verifiers.go @@ -55,7 +55,7 @@ func (v Verifiers) Type() string { return KindVerifregVerifiers } -func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindVerifregVerifiers, zap.Inline(act), "duration", time.Since(start)) @@ -63,7 +63,7 @@ func (Verifiers) Diff(ctx context.Context, api tasks.DataSource, act *actors.Act return DiffVerifiers(ctx, api, act) } -func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffVerifiers(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, VerifregStateLoader, VerifiregVerifiersMapLoader) if err != nil { return nil, err diff --git a/pkg/extract/actors/verifregdiff/v2/allocations.go b/pkg/extract/actors/verifregdiff/v2/allocations.go index 10c0c6898..7760c8077 100644 --- a/pkg/extract/actors/verifregdiff/v2/allocations.go +++ b/pkg/extract/actors/verifregdiff/v2/allocations.go @@ -70,7 +70,7 @@ func (a Allocations) Type() string { return KindVerifregAllocations } -func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindVerifregAllocations, zap.Inline(act), "duration", time.Since(start)) @@ -78,7 +78,7 @@ func (Allocations) Diff(ctx context.Context, api tasks.DataSource, act *actors.A return DiffAllocations(ctx, api, act) } -func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregAllocationMapLoader) if err != nil { return nil, err @@ -102,7 +102,7 @@ func DiffAllocations(ctx context.Context, api tasks.DataSource, act *actors.Acto return out, nil } -func diffClientMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, clientAddress address.Address, clientKey []byte) ([]*AllocationsChange, error) { +func diffClientMap(ctx context.Context, api tasks.DataSource, act *actors.Change, clientAddress address.Address, clientKey []byte) ([]*AllocationsChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { verifregState := i.(verifreg.State) clientAllocationMap, err := verifregState.AllocationMapForClient(clientAddress) diff --git a/pkg/extract/actors/verifregdiff/v2/claims.go b/pkg/extract/actors/verifregdiff/v2/claims.go index ae8858cc6..7c06ef5eb 100644 --- a/pkg/extract/actors/verifregdiff/v2/claims.go +++ b/pkg/extract/actors/verifregdiff/v2/claims.go @@ -72,7 +72,7 @@ func (c Claims) Type() string { return KindVerifregClaims } -func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { start := time.Now() defer func() { log.Debugw("Diff", "kind", KindVerifregClaims, zap.Inline(act), "duration", time.Since(start)) @@ -80,7 +80,7 @@ func (Claims) Diff(ctx context.Context, api tasks.DataSource, act *actors.ActorC return DiffClaims(ctx, api, act) } -func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChange) (actors.ActorStateChange, error) { +func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.Change) (actors.ActorStateChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, VerifregClaimsMapLoader) if err != nil { return nil, err @@ -105,7 +105,7 @@ func DiffClaims(ctx context.Context, api tasks.DataSource, act *actors.ActorChan return out, nil } -func diffProviderMap(ctx context.Context, api tasks.DataSource, act *actors.ActorChange, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { +func diffProviderMap(ctx context.Context, api tasks.DataSource, act *actors.Change, providerAddress address.Address, providerKey []byte) ([]*ClaimsChange, error) { mapChange, err := generic.DiffActorMap(ctx, api, act, v0.VerifregStateLoader, func(i interface{}) (adt.Map, *adt.MapOpts, error) { verifregState := i.(verifreg.State) providerClaimMap, err := verifregState.ClaimMapForProvider(providerAddress) diff --git a/pkg/extract/actors/verifregdiff/v2/state.go b/pkg/extract/actors/verifregdiff/v2/state.go index 1043ea40c..7bf2059e2 100644 --- a/pkg/extract/actors/verifregdiff/v2/state.go +++ b/pkg/extract/actors/verifregdiff/v2/state.go @@ -15,7 +15,7 @@ import ( var log = logging.Logger("extract/actors/verifreg") -func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.ActorDiffResult, error) { +func ActorStateChangeHandler(changes []actors.ActorStateChange) (actors.DiffResult, error) { var stateDiff = new(StateDiffResult) for _, stateChange := range changes { switch v := stateChange.(type) { diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go index 34d765970..27a62d591 100644 --- a/pkg/extract/extract.go +++ b/pkg/extract/extract.go @@ -2,6 +2,7 @@ package extract import ( "context" + "runtime" "time" actortypes "github.com/filecoin-project/go-state-types/actors" @@ -57,7 +58,7 @@ func State(ctx context.Context, api tasks.DataSource, current, executed *types.T grp.Go(func() error { start := time.Now() // all actor changes between current and parent, actor state exists in current. - actorChanges, err = Actors(grpCtx, api, current, executed, actorVersion, 8) + actorChanges, err = Actors(ctx, api, current, executed, runtime.NumCPU()) if err != nil { log.Errorw("failed to extract actor states", "error", err) return err diff --git a/pkg/transform/cbor/actors/serialize.go b/pkg/transform/cbor/actors/serialize.go index 944438ef5..26b39e2b9 100644 --- a/pkg/transform/cbor/actors/serialize.go +++ b/pkg/transform/cbor/actors/serialize.go @@ -160,7 +160,7 @@ func ProcessActorsStates(ctx context.Context, s store.Store, changes *extract.Ac return out, nil } -func PutActorDiffResult(ctx context.Context, s store.Store, result actors.ActorDiffResult) (cid.Cid, error) { +func PutActorDiffResult(ctx context.Context, s store.Store, result actors.DiffResult) (cid.Cid, error) { changes, err := result.MarshalStateChange(ctx, s) if err != nil { return cid.Undef, err @@ -168,7 +168,7 @@ func PutActorDiffResult(ctx context.Context, s store.Store, result actors.ActorD return s.Put(ctx, changes) } -func PutActorDiffResultMap(ctx context.Context, s store.Store, results map[address.Address]actors.ActorDiffResult) (cid.Cid, error) { +func PutActorDiffResultMap(ctx context.Context, s store.Store, results map[address.Address]actors.DiffResult) (cid.Cid, error) { actorHamt, err := adt.MakeEmptyMap(s, 5 /*TODO*/) if err != nil { return cid.Undef, err From 70815f898039478fdb2df012a2fdfaa44c343580 Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 10 Feb 2023 12:55:15 -0800 Subject: [PATCH 47/48] fix race in worker pool --- pkg/extract/actors.go | 63 ++++++++++++++++++++-------------- pkg/extract/actors/executor.go | 5 +-- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 74f92e7cf..25794ee9d 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "sync" + "sync/atomic" "github.com/filecoin-project/go-address" actortypes "github.com/filecoin-project/go-state-types/actors" @@ -113,7 +114,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. // cancel is the channel used to cancel any active workers cancel = make(chan struct{}) // scheduledWorkers is the number of workers scheduled for execution, its value decreases as workers complete - scheduledWorkers = 0 + scheduledWorkers = int64(0) // workerCtx is the context used by the workers. // TODO a method deep in the call stack of this function is not respecting context cancellation workerCtx = context.TODO() @@ -135,26 +136,30 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. pool.Submit(func() { defer func() { wg.Done() - scheduledWorkers-- + atomic.AddInt64(&scheduledWorkers, -1) }() res, err := diffRawActorState(workerCtx, api, act) if err != nil { // attempt to send the error or bail if canceled + for { + select { + case <-cancel: + return + case errCh <- err: + return + } + } + } + // attempt to send the result or bail if canceled + for { select { case <-cancel: return - case errCh <- err: + case resCh <- res: return } } - // attempt to send the result or bail if canceled - select { - case <-cancel: - return - case resCh <- res: - return - } }) wg.Add(1) @@ -162,26 +167,30 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. pool.Submit(func() { defer func() { wg.Done() - scheduledWorkers-- + atomic.AddInt64(&scheduledWorkers, -1) }() res, ok, err := diffTypedActorState(workerCtx, api, act) if err != nil { // attempt to send the error or bail if canceled - select { - case <-cancel: - return - case errCh <- err: - return + for { + select { + case <-cancel: + return + case errCh <- err: + return + } } } if ok { // attempt to send the result or bail if canceled - select { - case <-cancel: - return - case resCh <- res: - return + for { + select { + case <-cancel: + return + case resCh <- res: + return + } } } // Not all actors have their state diffed, for example account actors have no state to diff: their state is empty. @@ -196,14 +205,20 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. go func() { wg.Wait() done <- struct{}{} + log.Info("cleaned up happy path go routine") }() // stop the worker pool, dropping any scheduled workers, and complete any wait-groups for the case a workers were canceled. cleanup := func() { + log.Info("cancel any remaining workers") + close(cancel) pool.Stop() - for i := 0; i < scheduledWorkers; i++ { + log.Info("worker pool stopped") + for i := scheduledWorkers; i > 0; i-- { wg.Done() } + close(resCh) + close(errCh) } // result of diffing all actor states @@ -219,13 +234,11 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. // canceling the context or receiving an error causes all workers to stop and drops any scheduled workers. case <-workerCtx.Done(): log.Errorw("context canceled", "error", workerCtx.Err()) - cancel <- struct{}{} cleanup() <-done return nil, workerCtx.Err() case err := <-errCh: - log.Infow("worker received error while processing", "error", err) - cancel <- struct{}{} + log.Errorw("worker received error while processing", "error", err) cleanup() <-done return nil, err diff --git a/pkg/extract/actors/executor.go b/pkg/extract/actors/executor.go index 154013982..9ceb58c53 100644 --- a/pkg/extract/actors/executor.go +++ b/pkg/extract/actors/executor.go @@ -15,7 +15,6 @@ type DifferReport struct { DiffType string StartTime time.Time Duration time.Duration - Error error Result ActorStateChange } @@ -33,11 +32,13 @@ func (s *StateDiffer) ActorDiff(ctx context.Context, api tasks.DataSource, act * for i, fn := range s.Methods { start := time.Now() res, err := fn.Diff(ctx, api, act) + if err != nil { + return nil, err + } out[i] = DifferReport{ DiffType: fn.Type(), StartTime: start, Duration: time.Since(start), - Error: err, Result: res, } } From e952206944f43859b9651803bd402fbfaf02a95c Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 10 Feb 2023 18:05:27 -0800 Subject: [PATCH 48/48] testing: wire up walker --- lens/lily/impl.go | 152 +++++++++++++++++++++-------------- pkg/extract/actors.go | 15 ++-- pkg/extract/chain.go | 6 +- pkg/extract/extract.go | 4 + pkg/indexer/inderxer.go | 44 ++++++++++ pkg/transform/cbor/router.go | 68 ++++++++++++---- 6 files changed, 198 insertions(+), 91 deletions(-) create mode 100644 pkg/indexer/inderxer.go diff --git a/lens/lily/impl.go b/lens/lily/impl.go index e2eb889cb..8160de0ac 100644 --- a/lens/lily/impl.go +++ b/lens/lily/impl.go @@ -11,7 +11,6 @@ import ( "github.com/filecoin-project/go-state-types/exitcode" network2 "github.com/filecoin-project/go-state-types/network" "github.com/filecoin-project/lotus/api" - "github.com/filecoin-project/lotus/blockstore" "github.com/filecoin-project/lotus/chain/consensus/filcns" "github.com/filecoin-project/lotus/chain/events" "github.com/filecoin-project/lotus/chain/state" @@ -43,7 +42,7 @@ import ( "github.com/filecoin-project/lily/lens/lily/modules" "github.com/filecoin-project/lily/lens/util" "github.com/filecoin-project/lily/network" - "github.com/filecoin-project/lily/pkg/extract" + indexer2 "github.com/filecoin-project/lily/pkg/indexer" "github.com/filecoin-project/lily/pkg/transform/cbor" "github.com/filecoin-project/lily/pkg/transform/timescale" "github.com/filecoin-project/lily/schedule" @@ -138,62 +137,64 @@ func (m *LilyNodeAPI) StartTipSetWorker(_ context.Context, cfg *LilyTipSetWorker } func (m *LilyNodeAPI) LilyIndexIPLD(_ context.Context, cfg *LilyIndexIPLDConfig) (bool, error) { - // the context's passed to these methods live for the duration of the clients request, so make a new one. - ctx := context.Background() - - // TODO add file buffering - f, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) - defer f.Close() - if err != nil { - return false, err - } - - taskAPI, err := datasource.NewDataSource(m) - if err != nil { - return false, err - } - + return false, nil /* - // connect to the database, were gonna write stuff to this. - db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ - NamingStrategy: schema.NamingStrategy{ - TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list - // TODO figure out how to make a new schema with gorm... - }, - }) + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() + + // TODO add file buffering + f, err := os.OpenFile(cfg.Path, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) + defer f.Close() if err != nil { return false, err } - if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { + + taskAPI, err := datasource.NewDataSource(m) + if err != nil { return false, err } + // connect to the database, were gonna write stuff to this. + db, err := gorm.Open(postgres.Open("host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable"), &gorm.Config{ + NamingStrategy: schema.NamingStrategy{ + TablePrefix: "z_", // all tables created will be prefixed with `z_` because I am lazy and want them all in the same spot at the bottom of my table list + // TODO figure out how to make a new schema with gorm... + }, + }) + if err != nil { + return false, err + } + if err := lambda.ParseParams(ctx, taskAPI, db); err != nil { + return false, err + } - */ - currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) - if err != nil { - return false, err - } + currentTs, err := m.ChainGetTipSet(ctx, cfg.TipSet) + if err != nil { + return false, err + } - executedTs, err := m.ChainGetTipSet(ctx, currentTs.Parents()) - if err != nil { - return false, err - } + executedTs, err := m.ChainGetTipSet(ctx, currentTs.Parents()) + if err != nil { + return false, err + } - chainState, err := extract.State(ctx, taskAPI, currentTs, executedTs) - if err != nil { - return false, err - } + chainState, err := extract.State(ctx, taskAPI, currentTs, executedTs) + if err != nil { + return false, err + } - bs := blockstore.NewMemorySync() - root, err := cbor.PersistToStore(ctx, bs, currentTs, executedTs, chainState) - if err != nil { - return false, err - } - if err := cbor.WriteCarV1(ctx, root, bs, f); err != nil { - return false, err - } - return true, nil + bs := blockstore.NewMemorySync() + if err := cbor.Persist(ctx, chainState); err != nil; { + return false, err + } + if err != nil { + return false, err + } + if err := cbor.WriteCarV1(ctx, root, bs, f); err != nil { + return false, err + } + return true, nil + */ } func (m *LilyNodeAPI) LilyIndexIPLDFile(_ context.Context, cfg *LilyIndexIPLDFileConfig) (bool, error) { @@ -375,29 +376,58 @@ func (m *LilyNodeAPI) LilyWatchNotify(_ context.Context, cfg *LilyWatchNotifyCon } func (m *LilyNodeAPI) LilyWalk(_ context.Context, cfg *LilyWalkConfig) (*schedule.JobSubmitResult, error) { - // the context's passed to these methods live for the duration of the clients request, so make a new one. - ctx := context.Background() + /* + // the context's passed to these methods live for the duration of the clients request, so make a new one. + ctx := context.Background() - md := storage.Metadata{ - JobName: cfg.JobConfig.Name, - } + md := storage.Metadata{ + JobName: cfg.JobConfig.Name, + } - // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. - strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) - if err != nil { - return nil, err - } + // create a database connection for this watch, ensure its pingable, and run migrations if needed/configured to. + strg, err := m.StorageCatalog.Connect(ctx, cfg.JobConfig.Storage, md) + if err != nil { + return nil, err + } + + taskAPI, err := datasource.NewDataSource(m) + if err != nil { + return nil, err + } + + // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. + idx, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + if err != nil { + return nil, err + } + + res := m.Scheduler.Submit(&schedule.JobConfig{ + Name: cfg.JobConfig.Name, + Type: "walk", + Params: map[string]string{ + "window": cfg.JobConfig.Window.String(), + "minHeight": fmt.Sprintf("%d", cfg.From), + "maxHeight": fmt.Sprintf("%d", cfg.To), + "storage": cfg.JobConfig.Storage, + }, + Tasks: cfg.JobConfig.Tasks, + Job: walk.NewWalker(idx, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), + RestartOnFailure: cfg.JobConfig.RestartOnFailure, + RestartOnCompletion: cfg.JobConfig.RestartOnCompletion, + RestartDelay: cfg.JobConfig.RestartDelay, + }) + + */ taskAPI, err := datasource.NewDataSource(m) if err != nil { return nil, err } - - // instantiate an indexer to extract block, message, and actor state data from observed tipsets and persists it to the storage. - idx, err := integrated.NewManager(strg, tipset.NewBuilder(taskAPI, cfg.JobConfig.Name), integrated.WithWindow(cfg.JobConfig.Window)) + transformer, err := cbor.NewTransformer("/Users/frrist/Workspace/src/github.com/filecoin-project/lily/garage", cfg.JobConfig.Name) if err != nil { return nil, err } + stateIndexer := indexer2.NewStateIndexer(taskAPI, transformer) res := m.Scheduler.Submit(&schedule.JobConfig{ Name: cfg.JobConfig.Name, @@ -409,7 +439,7 @@ func (m *LilyNodeAPI) LilyWalk(_ context.Context, cfg *LilyWalkConfig) (*schedul "storage": cfg.JobConfig.Storage, }, Tasks: cfg.JobConfig.Tasks, - Job: walk.NewWalker(idx, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), + Job: walk.NewWalker(stateIndexer, m, cfg.JobConfig.Name, cfg.JobConfig.Tasks, cfg.From, cfg.To), RestartOnFailure: cfg.JobConfig.RestartOnFailure, RestartOnCompletion: cfg.JobConfig.RestartOnCompletion, RestartDelay: cfg.JobConfig.RestartDelay, diff --git a/pkg/extract/actors.go b/pkg/extract/actors.go index 25794ee9d..feed962d8 100644 --- a/pkg/extract/actors.go +++ b/pkg/extract/actors.go @@ -197,7 +197,6 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. log.Debugw("no actor diff for actor", "current_network", curNtwkVersion, "executed_network", exeNtwkVersion, "address", act.Address, "current_code", act.Current.Code, "current_version", act.CurVersion, "executed_code", act.Executed.Code, "executed_version", act.ExeVersion) - }) } @@ -205,15 +204,15 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. go func() { wg.Wait() done <- struct{}{} - log.Info("cleaned up happy path go routine") + log.Debugw("cleaned up happy path go routine") }() // stop the worker pool, dropping any scheduled workers, and complete any wait-groups for the case a workers were canceled. cleanup := func() { - log.Info("cancel any remaining workers") + log.Debug("cancel any remaining workers") close(cancel) pool.Stop() - log.Info("worker pool stopped") + log.Debug("worker pool stopped") for i := scheduledWorkers; i > 0; i-- { wg.Done() } @@ -229,7 +228,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. RawActors: make(map[address.Address]actors.DiffResult, len(actorChanges)), // there are at most actorChanges entries } for { - log.Infof("diff jobs todo %d", scheduledWorkers) + log.Debugw("diff jobs todo %d", scheduledWorkers) select { // canceling the context or receiving an error causes all workers to stop and drops any scheduled workers. case <-workerCtx.Done(): @@ -245,7 +244,7 @@ func Actors(ctx context.Context, api tasks.DataSource, current, executed *types. // happy path, all workers completed and we can return with a result case <-done: - log.Info("done processing") + log.Debug("done processing") cleanup() return out, nil @@ -287,7 +286,7 @@ func diffRawActorState(ctx context.Context, api tasks.DataSource, act *actors.Ch ActorHandler: handler, ReportHandler: func(reports []actors.DifferReport) error { for _, report := range reports { - log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) + log.Debugw("reporting", "type", report.DiffType, "duration", report.Duration) } return nil }, @@ -335,7 +334,7 @@ func diffTypedActorState(ctx context.Context, api tasks.DataSource, act *actors. ActorHandler: handler, ReportHandler: func(reports []actors.DifferReport) error { for _, report := range reports { - log.Infow("reporting", "type", report.DiffType, "duration", report.Duration) + log.Debugw("reporting", "type", report.DiffType, "duration", report.Duration) } return nil }, diff --git a/pkg/extract/chain.go b/pkg/extract/chain.go index f5a5513ef..d44b9b329 100644 --- a/pkg/extract/chain.go +++ b/pkg/extract/chain.go @@ -12,8 +12,6 @@ import ( ) type MessageStateChanges struct { - Current *types.TipSet - Executed *types.TipSet BaseFee abi.TokenAmount CirculatingSupply CirculatingSupply FullBlocks map[cid.Cid]*chain.FullBlock @@ -44,9 +42,7 @@ func FullBlockMessages(ctx context.Context, api tasks.DataSource, current, execu } return &MessageStateChanges{ - Current: current, - Executed: executed, - BaseFee: baseFee, + BaseFee: baseFee, CirculatingSupply: CirculatingSupply{ FilVested: cs.FilVested, FilMined: cs.FilMined, diff --git a/pkg/extract/extract.go b/pkg/extract/extract.go index 27a62d591..e0a2fc0db 100644 --- a/pkg/extract/extract.go +++ b/pkg/extract/extract.go @@ -17,6 +17,8 @@ type ChainState struct { NetworkName string NetworkVersion uint64 ActorVersion uint64 + Current *types.TipSet + Parent *types.TipSet Message *MessageStateChanges Actors *ActorStateChanges } @@ -75,6 +77,8 @@ func State(ctx context.Context, api tasks.DataSource, current, executed *types.T NetworkName: networkName, NetworkVersion: uint64(networkVersion), ActorVersion: uint64(actorVersion), + Current: current, + Parent: executed, Message: blockmessages, Actors: actorChanges, }, nil diff --git a/pkg/indexer/inderxer.go b/pkg/indexer/inderxer.go new file mode 100644 index 000000000..e46379b5d --- /dev/null +++ b/pkg/indexer/inderxer.go @@ -0,0 +1,44 @@ +package indexer + +import ( + "context" + + "github.com/filecoin-project/lotus/chain/types" + + "github.com/filecoin-project/lily/chain/indexer" + "github.com/filecoin-project/lily/pkg/extract" + "github.com/filecoin-project/lily/tasks" +) + +func NewStateIndexer(api tasks.DataSource, handler StateHandler) *StateIndexer { + return &StateIndexer{ + api: api, + handler: handler, + } +} + +type StateIndexer struct { + api tasks.DataSource + handler StateHandler +} + +type StateHandler interface { + Persist(ctx context.Context, chainState *extract.ChainState) error +} + +func (s *StateIndexer) TipSet(ctx context.Context, ts *types.TipSet, opts ...indexer.Option) (bool, error) { + parent, err := s.api.TipSet(ctx, ts.Parents()) + if err != nil { + return false, err + } + + chainState, err := extract.State(ctx, s.api, ts, parent) + if err != nil { + return false, err + } + + if err := s.handler.Persist(ctx, chainState); err != nil { + return false, err + } + return true, nil +} diff --git a/pkg/transform/cbor/router.go b/pkg/transform/cbor/router.go index 0a75b2c53..15818a080 100644 --- a/pkg/transform/cbor/router.go +++ b/pkg/transform/cbor/router.go @@ -1,9 +1,13 @@ package cbor import ( + "bufio" "context" "fmt" "io" + "os" + "path/filepath" + "time" "github.com/filecoin-project/go-state-types/abi" adtStore "github.com/filecoin-project/go-state-types/store" @@ -108,6 +112,44 @@ func (s *StateExtractionIPLD) MarshalLogObject(enc zapcore.ObjectEncoder) error return nil } +func NewTransformer(dir, prefix string) (*Transformer, error) { + if dirInfo, err := os.Stat(dir); err != nil { + if os.IsNotExist(err) { + return nil, fmt.Errorf("path (%s) does not exist: %w", dir, err) + } + if !dirInfo.IsDir() { + return nil, fmt.Errorf("path (%s) is not a directory", dir) + } + } + return &Transformer{carDirectory: dir, prefix: prefix}, nil +} + +type Transformer struct { + carDirectory string + prefix string +} + +func (t *Transformer) Persist(ctx context.Context, chainState *extract.ChainState) error { + start := time.Now() + carPath := filepath.Join(t.carDirectory, fmt.Sprintf("%s_%d_%d.car", t.prefix, chainState.Parent.Height(), chainState.Current.Height())) + f, err := os.OpenFile(carPath, os.O_CREATE|os.O_TRUNC|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer f.Close() + bs := blockstore.NewMemorySync() + root, err := PersistToStore(ctx, bs, chainState) + if err != nil { + return err + } + bw := bufio.NewWriterSize(f, 1<<20) + if err := WriteCarV1(ctx, root, bs, bw); err != nil { + return err + } + log.Infow("created chain delta", "path", carPath, "duration", time.Since(start)) + return nil +} + func WriteCarV1(ctx context.Context, root cid.Cid, bs blockstore.Blockstore, w io.Writer) error { if err := v1car.WriteHeader(&v1car.CarHeader{ Roots: []cid.Cid{root}, @@ -133,17 +175,9 @@ func WriteCarV1(ctx context.Context, root cid.Cid, bs blockstore.Blockstore, w i return nil } -func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, executed *types.TipSet, chainState *extract.ChainState) (cid.Cid, error) { +func PersistToStore(ctx context.Context, bs blockstore.Blockstore, chainState *extract.ChainState) (cid.Cid, error) { store := adtStore.WrapBlockStore(ctx, bs) - // sanity check - if !chainState.Message.Current.Equals(chainState.Actors.Current) { - return cid.Undef, fmt.Errorf("actor and message current tipset does not match") - } - if !chainState.Message.Executed.Equals(chainState.Actors.Executed) { - return cid.Undef, fmt.Errorf("actor and message executed tipset does not match") - } - implicitMsgsAMT, err := cbormessages.MakeImplicitMessagesHAMT(ctx, store, chainState.Message.ImplicitMessages) if err != nil { return cid.Undef, err @@ -165,8 +199,8 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec } extractedState := &StateExtractionIPLD{ - Current: *current, - Parent: *executed, + Current: *chainState.Current, + Parent: *chainState.Parent, BaseFee: chainState.Message.BaseFee, FilVested: chainState.Message.CirculatingSupply.FilVested, FilMined: chainState.Message.CirculatingSupply.FilMined, @@ -198,7 +232,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec dsn := "host=localhost user=postgres password=password dbname=postgres port=5432 sslmode=disable" db, err := gorm.Open(postgres.Open(dsn), &gorm.Config{ NamingStrategy: schema.NamingStrategy{ - TablePrefix: "lily_cbor", + TablePrefix: "lily_cbor_", SingularTable: false, }, }) @@ -209,7 +243,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec return cid.Undef, err } rootModel := RootStateModel{ - Height: uint64(current.Height()), + Height: uint64(chainState.Current.Height()), Cid: root.String(), StateVersion: rootState.StateVersion, NetworkName: rootState.NetworkName, @@ -217,9 +251,9 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec StateExtraction: rootState.State.String(), } stateModel := StateExtractionModel{ - Height: uint64(current.Height()), - CurrentTipSet: current.String(), - ParentTipSet: executed.String(), + Height: uint64(chainState.Current.Height()), + CurrentTipSet: chainState.Current.String(), + ParentTipSet: chainState.Parent.String(), BaseFee: chainState.Message.BaseFee.String(), FullBlocks: fullBlkHAMT.String(), ImplicitMessages: implicitMsgsAMT.String(), @@ -233,7 +267,7 @@ func PersistToStore(ctx context.Context, bs blockstore.Blockstore, current, exec return err } as := actorStateContainer.AsModel() - as.Height = uint64(current.Height()) + as.Height = uint64(chainState.Current.Height()) if err := tx.Create(as).Error; err != nil { return err }