Skip to content

Commit 344e9a8

Browse files
birdychangbirdychang
authored
fix: batch insert for locked_funds, sector_events and sector_info_v7 (#1216)
* fix: batch insert into database * added transformers and minor fixes * fix tests * fix more tests
1 parent 5a34226 commit 344e9a8

File tree

9 files changed

+118
-42
lines changed

9 files changed

+118
-42
lines changed

‎chain/indexer/integrated/processor/state.go‎

Lines changed: 27 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -441,9 +441,13 @@ func MakeProcessors(api tasks.DataSource, indexerTasks []string) (*IndexerProces
441441
mineractors.AllCodes(), minertask.InfoExtractor{},
442442
))
443443
case tasktype.MinerLockedFund:
444-
out.ActorProcessors[t] = actorstate.NewTask(api, actorstate.NewTypedActorExtractorMap(
445-
mineractors.AllCodes(), minertask.LockedFundsExtractor{},
446-
))
444+
out.ActorProcessors[t] = actorstate.NewTaskWithTransformer(
445+
api,
446+
actorstate.NewTypedActorExtractorMap(
447+
mineractors.AllCodes(), minertask.LockedFundsExtractor{},
448+
),
449+
minertask.LockedFundsExtractor{},
450+
)
447451
case tasktype.MinerPreCommitInfo:
448452
out.ActorProcessors[t] = actorstate.NewTask(api, actorstate.NewCustomTypedActorExtractorMap(
449453
map[cid.Cid][]actorstate.ActorStateExtractor{
@@ -465,9 +469,13 @@ func MakeProcessors(api tasks.DataSource, indexerTasks []string) (*IndexerProces
465469
mineractors.AllCodes(), minertask.SectorDealsExtractor{},
466470
))
467471
case tasktype.MinerSectorEvent:
468-
out.ActorProcessors[t] = actorstate.NewTask(api, actorstate.NewTypedActorExtractorMap(
469-
mineractors.AllCodes(), minertask.SectorEventsExtractor{},
470-
))
472+
out.ActorProcessors[t] = actorstate.NewTaskWithTransformer(
473+
api,
474+
actorstate.NewTypedActorExtractorMap(
475+
mineractors.AllCodes(), minertask.SectorEventsExtractor{},
476+
),
477+
minertask.SectorEventsExtractor{},
478+
)
471479
case tasktype.MinerSectorPost:
472480
out.ActorProcessors[t] = actorstate.NewTask(api, actorstate.NewTypedActorExtractorMap(
473481
mineractors.AllCodes(), minertask.PoStExtractor{},
@@ -484,15 +492,19 @@ func MakeProcessors(api tasks.DataSource, indexerTasks []string) (*IndexerProces
484492
},
485493
))
486494
case tasktype.MinerSectorInfoV7:
487-
out.ActorProcessors[t] = actorstate.NewTask(api, actorstate.NewCustomTypedActorExtractorMap(
488-
map[cid.Cid][]actorstate.ActorStateExtractor{
489-
mineractors.VersionCodes()[actorstypes.Version7]: {minertask.V7SectorInfoExtractor{}},
490-
mineractors.VersionCodes()[actorstypes.Version8]: {minertask.V7SectorInfoExtractor{}},
491-
mineractors.VersionCodes()[actorstypes.Version9]: {minertask.V7SectorInfoExtractor{}},
492-
mineractors.VersionCodes()[actorstypes.Version10]: {minertask.V7SectorInfoExtractor{}},
493-
mineractors.VersionCodes()[actorstypes.Version11]: {minertask.V7SectorInfoExtractor{}},
494-
},
495-
))
495+
out.ActorProcessors[t] = actorstate.NewTaskWithTransformer(
496+
api,
497+
actorstate.NewCustomTypedActorExtractorMap(
498+
map[cid.Cid][]actorstate.ActorStateExtractor{
499+
mineractors.VersionCodes()[actorstypes.Version7]: {minertask.V7SectorInfoExtractor{}},
500+
mineractors.VersionCodes()[actorstypes.Version8]: {minertask.V7SectorInfoExtractor{}},
501+
mineractors.VersionCodes()[actorstypes.Version9]: {minertask.V7SectorInfoExtractor{}},
502+
mineractors.VersionCodes()[actorstypes.Version10]: {minertask.V7SectorInfoExtractor{}},
503+
mineractors.VersionCodes()[actorstypes.Version11]: {minertask.V7SectorInfoExtractor{}},
504+
},
505+
),
506+
minertask.V7SectorInfoExtractor{},
507+
)
496508

497509
//
498510
// Power

‎chain/indexer/integrated/processor/state_internal_test.go‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,9 @@ func TestNewProcessor(t *testing.T) {
8080
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.DeadlineInfoExtractor{})), proc.actorProcessors[tasktype.MinerCurrentDeadlineInfo])
8181
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.FeeDebtExtractor{})), proc.actorProcessors[tasktype.MinerFeeDebt])
8282
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.InfoExtractor{})), proc.actorProcessors[tasktype.MinerInfo])
83-
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.LockedFundsExtractor{})), proc.actorProcessors[tasktype.MinerLockedFund])
83+
require.Equal(t, actorstate.NewTaskWithTransformer(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.LockedFundsExtractor{}), minertask.LockedFundsExtractor{}), proc.actorProcessors[tasktype.MinerLockedFund])
8484
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorDealsExtractor{})), proc.actorProcessors[tasktype.MinerSectorDeal])
85-
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorEventsExtractor{})), proc.actorProcessors[tasktype.MinerSectorEvent])
85+
require.Equal(t, actorstate.NewTaskWithTransformer(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorEventsExtractor{}), minertask.SectorEventsExtractor{}), proc.actorProcessors[tasktype.MinerSectorEvent])
8686
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.PoStExtractor{})), proc.actorProcessors[tasktype.MinerSectorPost])
8787
require.Equal(t, actorstate.NewTask(nil, actorstate.NewCustomTypedActorExtractorMap(
8888
map[cid.Cid][]actorstate.ActorStateExtractor{
@@ -94,15 +94,21 @@ func TestNewProcessor(t *testing.T) {
9494
miner.VersionCodes()[actorstypes.Version6]: {minertask.SectorInfoExtractor{}},
9595
},
9696
)), proc.actorProcessors[tasktype.MinerSectorInfoV1_6])
97-
require.Equal(t, actorstate.NewTask(nil, actorstate.NewCustomTypedActorExtractorMap(
98-
map[cid.Cid][]actorstate.ActorStateExtractor{
99-
miner.VersionCodes()[actorstypes.Version7]: {minertask.V7SectorInfoExtractor{}},
100-
miner.VersionCodes()[actorstypes.Version8]: {minertask.V7SectorInfoExtractor{}},
101-
miner.VersionCodes()[actorstypes.Version9]: {minertask.V7SectorInfoExtractor{}},
102-
miner.VersionCodes()[actorstypes.Version10]: {minertask.V7SectorInfoExtractor{}},
103-
miner.VersionCodes()[actorstypes.Version11]: {minertask.V7SectorInfoExtractor{}},
104-
},
105-
)), proc.actorProcessors[tasktype.MinerSectorInfoV7])
97+
require.Equal(
98+
t,
99+
actorstate.NewTaskWithTransformer(
100+
nil,
101+
actorstate.NewCustomTypedActorExtractorMap(
102+
map[cid.Cid][]actorstate.ActorStateExtractor{
103+
miner.VersionCodes()[actorstypes.Version7]: {minertask.V7SectorInfoExtractor{}},
104+
miner.VersionCodes()[actorstypes.Version8]: {minertask.V7SectorInfoExtractor{}},
105+
miner.VersionCodes()[actorstypes.Version9]: {minertask.V7SectorInfoExtractor{}},
106+
miner.VersionCodes()[actorstypes.Version10]: {minertask.V7SectorInfoExtractor{}},
107+
miner.VersionCodes()[actorstypes.Version11]: {minertask.V7SectorInfoExtractor{}},
108+
}),
109+
minertask.V7SectorInfoExtractor{},
110+
),
111+
proc.actorProcessors[tasktype.MinerSectorInfoV7])
106112
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(power.AllCodes(), powertask.ClaimedPowerExtractor{})), proc.actorProcessors[tasktype.PowerActorClaim])
107113
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(power.AllCodes(), powertask.ChainPowerExtractor{})), proc.actorProcessors[tasktype.ChainPower])
108114
require.Equal(t, actorstate.NewTask(nil, actorstate.NewTypedActorExtractorMap(reward.AllCodes(), rewardtask.RewardExtractor{})), proc.actorProcessors[tasktype.ChainReward])

‎chain/indexer/integrated/processor/state_test.go‎

Lines changed: 27 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -62,10 +62,6 @@ func TestMakeProcessorsActors(t *testing.T) {
6262
taskName: tasktype.MinerInfo,
6363
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.InfoExtractor{}),
6464
},
65-
{
66-
taskName: tasktype.MinerLockedFund,
67-
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.LockedFundsExtractor{}),
68-
},
6965
{
7066
taskName: tasktype.MinerPreCommitInfo,
7167
extractor: actorstate.NewCustomTypedActorExtractorMap(
@@ -88,10 +84,6 @@ func TestMakeProcessorsActors(t *testing.T) {
8884
taskName: tasktype.MinerSectorDeal,
8985
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorDealsExtractor{}),
9086
},
91-
{
92-
taskName: tasktype.MinerSectorEvent,
93-
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorEventsExtractor{}),
94-
},
9587
{
9688
taskName: tasktype.MinerSectorPost,
9789
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.PoStExtractor{}),
@@ -109,6 +101,30 @@ func TestMakeProcessorsActors(t *testing.T) {
109101
},
110102
),
111103
},
104+
}
105+
for _, tc := range testCases {
106+
t.Run(tc.taskName, func(t *testing.T) {
107+
proc, err := processor.MakeProcessors(nil, []string{tc.taskName})
108+
require.NoError(t, err)
109+
require.Len(t, proc.ActorProcessors, 1)
110+
require.Equal(t, actorstate.NewTask(nil, tc.extractor), proc.ActorProcessors[tc.taskName])
111+
})
112+
}
113+
testCases2 := []struct {
114+
taskName string
115+
extractor actorstate.ActorExtractorMap
116+
transformer actorstate.ActorDataTransformer
117+
}{
118+
{
119+
taskName: tasktype.MinerLockedFund,
120+
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.LockedFundsExtractor{}),
121+
transformer: minertask.LockedFundsExtractor{},
122+
},
123+
{
124+
taskName: tasktype.MinerSectorEvent,
125+
extractor: actorstate.NewTypedActorExtractorMap(miner.AllCodes(), minertask.SectorEventsExtractor{}),
126+
transformer: minertask.SectorEventsExtractor{},
127+
},
112128
{
113129
taskName: tasktype.MinerSectorInfoV7,
114130
extractor: actorstate.NewCustomTypedActorExtractorMap(
@@ -120,14 +136,15 @@ func TestMakeProcessorsActors(t *testing.T) {
120136
miner.VersionCodes()[actorstypes.Version11]: {minertask.V7SectorInfoExtractor{}},
121137
},
122138
),
139+
transformer: minertask.V7SectorInfoExtractor{},
123140
},
124141
}
125-
for _, tc := range testCases {
142+
for _, tc := range testCases2 {
126143
t.Run(tc.taskName, func(t *testing.T) {
127144
proc, err := processor.MakeProcessors(nil, []string{tc.taskName})
128145
require.NoError(t, err)
129146
require.Len(t, proc.ActorProcessors, 1)
130-
require.Equal(t, actorstate.NewTask(nil, tc.extractor), proc.ActorProcessors[tc.taskName])
147+
require.Equal(t, actorstate.NewTaskWithTransformer(nil, tc.extractor, tc.transformer), proc.ActorProcessors[tc.taskName])
131148
})
132149
}
133150
})

‎model/actors/init/idaddress.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ func (ia *IDAddress) Persist(ctx context.Context, s model.StorageBatch, version
6666
type IDAddressList []*IDAddress
6767

6868
func (ias IDAddressList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error {
69-
ctx, span := otel.Tracer("").Start(ctx, "IDAddressList.PersistWithTx")
69+
ctx, span := otel.Tracer("").Start(ctx, "IDAddressList.Persist")
7070
if span.IsRecording() {
7171
span.SetAttributes(attribute.Int("count", len(ias)))
7272
}

‎model/actors/market/dealstate.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ func (ds *MarketDealState) Persist(ctx context.Context, s model.StorageBatch, ve
2929
type MarketDealStates []*MarketDealState
3030

3131
func (dss MarketDealStates) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error {
32-
ctx, span := otel.Tracer("").Start(ctx, "MarketDealStates.PersistWithTx")
32+
ctx, span := otel.Tracer("").Start(ctx, "MarketDealStates.Persist")
3333
if span.IsRecording() {
3434
span.SetAttributes(attribute.Int("count", len(dss)))
3535
}

‎model/actors/power/chainpower.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func (cp *ChainPower) AsVersion(version model.Version) (interface{}, bool) {
7979
}
8080

8181
func (cp *ChainPower) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error {
82-
ctx, span := otel.Tracer("").Start(ctx, "ChainPower.PersistWithTx")
82+
ctx, span := otel.Tracer("").Start(ctx, "ChainPower.Persist")
8383
defer span.End()
8484

8585
ctx, _ = tag.New(ctx, tag.Upsert(metrics.Table, "chain_powers"))
@@ -96,10 +96,8 @@ func (cp *ChainPower) Persist(ctx context.Context, s model.StorageBatch, version
9696
// ChainPowerList is a slice of ChainPowers for batch insertion.
9797
type ChainPowerList []*ChainPower
9898

99-
// PersistWithTx makes a batch insertion of the list using the given
100-
// transaction.
10199
func (cpl ChainPowerList) Persist(ctx context.Context, s model.StorageBatch, version model.Version) error {
102-
ctx, span := otel.Tracer("").Start(ctx, "ChainPowerList.PersistWithTx")
100+
ctx, span := otel.Tracer("").Start(ctx, "ChainPowerList.Persist")
103101
if span.IsRecording() {
104102
span.SetAttributes(attribute.Int("count", len(cpl)))
105103
}

‎tasks/actorstate/miner/locked_funds.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,3 +55,18 @@ func (LockedFundsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo,
5555
PreCommitDeposits: currLocked.PreCommitDeposits.String(),
5656
}, nil
5757
}
58+
59+
func (LockedFundsExtractor) Transform(ctx context.Context, data model.PersistableList) (model.PersistableList, error) {
60+
persistableList := make(minermodel.MinerLockedFundsList, 0, len(data))
61+
for _, d := range data {
62+
if d == nil {
63+
continue
64+
}
65+
a, ok := d.(*minermodel.MinerLockedFund)
66+
if !ok {
67+
return nil, fmt.Errorf("expected MinerLockedFund type but got: %T", d)
68+
}
69+
persistableList = append(persistableList, a)
70+
}
71+
return model.PersistableList{persistableList}, nil
72+
}

‎tasks/actorstate/miner/sector_events.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,20 @@ func (SectorEventsExtractor) Extract(ctx context.Context, a actorstate.ActorInfo
133133
return sectorEventModel, nil
134134
}
135135

136+
func (SectorEventsExtractor) Transform(ctx context.Context, data model.PersistableList) (model.PersistableList, error) {
137+
persistableList := make(minermodel.MinerSectorEventList, 0, len(data))
138+
for _, d := range data {
139+
ml, ok := d.(minermodel.MinerSectorEventList)
140+
if !ok {
141+
return nil, fmt.Errorf("expected MinerSectorEventList type but got: %T", d)
142+
}
143+
for _, m := range ml {
144+
persistableList = append(persistableList, m)
145+
}
146+
}
147+
return model.PersistableList{persistableList}, nil
148+
}
149+
136150
// ExtractSectorEvents transforms sectorChanges, preCommitChanges, and sectorStateChanges to a MinerSectorEventList.
137151
func ExtractSectorEvents(extState extraction.State, sectorChanges *miner.SectorChanges, preCommitChanges *miner.PreCommitChanges, sectorStateChanges *SectorStateEvents) (minermodel.MinerSectorEventList, error) {
138152
sectorStateEvents, err := ExtractMinerSectorStateEvents(extState, sectorStateChanges)

‎tasks/actorstate/miner/sectorv7.go‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,17 @@ func (V7SectorInfoExtractor) Extract(ctx context.Context, a actorstate.ActorInfo
7676

7777
return sectorModel, nil
7878
}
79+
80+
func (V7SectorInfoExtractor) Transform(ctx context.Context, data model.PersistableList) (model.PersistableList, error) {
81+
persistableList := make(minermodel.MinerSectorInfoV7List, 0, len(data))
82+
for _, d := range data {
83+
ml, ok := d.(minermodel.MinerSectorInfoV7List)
84+
if !ok {
85+
return nil, fmt.Errorf("expected MinerSectorInfoV7 type but got: %T", d)
86+
}
87+
for _, m := range ml {
88+
persistableList = append(persistableList, m)
89+
}
90+
}
91+
return model.PersistableList{persistableList}, nil
92+
}

0 commit comments

Comments
 (0)