Skip to content

Commit 315d896

Browse files
magik6krvagg
magik6k
authored and
rvagg
committed
fix wallet exporter, supra allocs, config doc (#581)
1 parent e465f42 commit 315d896

File tree

12 files changed

+178
-93
lines changed

12 files changed

+178
-93
lines changed

‎cmd/curio/tasks/tasks.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ func addSealingTasks(
389389
cfg.Seal.BatchSealPipelines,
390390
!cfg.Seal.SingleHasherPerThread,
391391
cfg.Seal.LayerNVMEDevices,
392-
machineHostPort, db, full, stor, si)
392+
machineHostPort, db, full, stor, si, slr)
393393
if err != nil {
394394
return nil, xerrors.Errorf("setting up batch sealer: %w", err)
395395
}

‎deps/config/doc_gen.go‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎deps/config/types.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@ type CurioSubsystemsConfig struct {
410410
// via Client Settings on the Proofshare webui page. Buy delay can also be set in the Client Settings page. (Default: false)
411411
EnableRemoteProofs bool
412412

413+
// The maximum number of remote proofs that can be uploaded simultaneously by each node. (Default: 15)
413414
RemoteProofMaxUploads int
414415

415416
// EnableWalletExporter enables the wallet exporter on the node. This will export wallet stats to prometheus.

‎deps/stats/wallet_exporter.go‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func init() {
6868
&view.View{
6969
Measure: WalletExporterMeasures.BalanceNFil,
7070
Aggregation: view.LastValue(),
71-
TagKeys: []tag.Key{tagAddressKey, tagTypeKey},
71+
TagKeys: []tag.Key{tagAddressKey, tagTypeKey, tagName},
7272
},
7373
&view.View{
7474
Measure: WalletExporterMeasures.Power,
@@ -87,22 +87,22 @@ func init() {
8787
},
8888
&view.View{
8989
Measure: WalletExporterMeasures.GasUnitsRequested,
90-
Aggregation: view.Count(),
90+
Aggregation: view.Sum(),
9191
TagKeys: []tag.Key{tagFromKey, tagFromName, tagToKey, tagToName, tagMethodKey, tagSendReasonKey},
9292
},
9393
&view.View{
9494
Measure: WalletExporterMeasures.GasUnitsUsed,
95-
Aggregation: view.Count(),
95+
Aggregation: view.Sum(),
9696
TagKeys: []tag.Key{tagFromKey, tagFromName, tagToKey, tagToName, tagMethodKey, tagSendReasonKey, tagExitCodeKey},
9797
},
9898
&view.View{
9999
Measure: WalletExporterMeasures.SentNFil,
100-
Aggregation: view.Count(),
100+
Aggregation: view.Sum(),
101101
TagKeys: []tag.Key{tagFromKey, tagFromName, tagToKey, tagToName, tagMethodKey, tagSendReasonKey, tagExitCodeKey},
102102
},
103103
&view.View{
104104
Measure: WalletExporterMeasures.GasPaidNFil,
105-
Aggregation: view.Count(),
105+
Aggregation: view.Sum(),
106106
TagKeys: []tag.Key{tagFromKey, tagFromName, tagToKey, tagToName, tagMethodKey, tagSendReasonKey, tagExitCodeKey},
107107
},
108108
)
@@ -121,6 +121,7 @@ func init() {
121121
var (
122122
tagAddressKey, _ = tag.NewKey("address")
123123
tagTypeKey, _ = tag.NewKey("type")
124+
tagName, _ = tag.NewKey("name")
124125

125126
tagFromKey, _ = tag.NewKey("from")
126127
tagFromName, _ = tag.NewKey("from_name")
@@ -186,10 +187,10 @@ func walletExporterBalances(ctx context.Context, db *harmonydb.DB, api api.FullN
186187
}
187188

188189
resolved := make(map[address.Address]walletAddrStr)
189-
for _, wallet := range names {
190+
for wallet := range names {
190191
addr, err := address.NewFromString(wallet)
191192
if err != nil {
192-
log.Errorf("failed to resolve wallet: %v", err)
193+
log.Errorf("failed to resolve wallet '%s': %v", wallet, err)
193194
continue
194195
}
195196
kaddr, err := api.StateAccountKey(ctx, addr, types.EmptyTSK)
@@ -207,9 +208,11 @@ func walletExporterBalances(ctx context.Context, db *harmonydb.DB, api api.FullN
207208
log.Errorf("failed to get balance for wallet: %v", err)
208209
continue
209210
}
211+
210212
_ = stats.RecordWithTags(ctx, []tag.Mutator{
211213
tag.Upsert(tagAddressKey, addr.String()),
212214
tag.Upsert(tagTypeKey, "wallet"),
215+
tag.Upsert(tagName, names[resolved[addr]]),
213216
}, WalletExporterMeasures.BalanceNFil.M(attoToNano(act.Balance)))
214217
}
215218

@@ -257,7 +260,7 @@ func walletExporterNewWatchedMsgs(ctx context.Context, db *harmonydb.DB, api api
257260
_, err := db.BeginTransaction(ctx, func(tx *harmonydb.Tx) (commit bool, err error) {
258261
unobservedMsgs = nil
259262

260-
rows, err := tx.Query(`SELECT created_at FROM wallet_exporter_processing`)
263+
rows, err := tx.Query(`SELECT processed_until FROM wallet_exporter_processing`)
261264
if err != nil {
262265
return false, err
263266
}

‎documentation/en/configuration/default-curio-configuration.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,8 @@ description: The default curio configuration
309309
# type: bool
310310
#EnableRemoteProofs = false
311311

312+
# The maximum number of remote proofs that can be uploaded simultaneously by each node. (Default: 15)
313+
#
312314
# type: int
313315
#RemoteProofMaxUploads = 15
314316

‎lib/ffi/piece_funcs.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ import (
1414

1515
func (sb *SealCalls) WritePiece(ctx context.Context, taskID *harmonytask.TaskID, pieceID storiface.PieceNumber, size int64, data io.Reader, storageType storiface.PathType) error {
1616
// Use storageType in AcquireSector
17-
paths, _, done, err := sb.sectors.AcquireSector(ctx, taskID, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType)
17+
paths, _, done, err := sb.Sectors.AcquireSector(ctx, taskID, pieceID.Ref(), storiface.FTNone, storiface.FTPiece, storageType)
1818
if err != nil {
1919
return err
2020
}
@@ -67,9 +67,9 @@ func (sb *SealCalls) WritePiece(ctx context.Context, taskID *harmonytask.TaskID,
6767
}
6868

6969
func (sb *SealCalls) PieceReader(ctx context.Context, id storiface.PieceNumber) (io.ReadCloser, error) {
70-
return sb.sectors.storage.ReaderSeq(ctx, id.Ref(), storiface.FTPiece)
70+
return sb.Sectors.storage.ReaderSeq(ctx, id.Ref(), storiface.FTPiece)
7171
}
7272

7373
func (sb *SealCalls) RemovePiece(ctx context.Context, id storiface.PieceNumber) error {
74-
return sb.sectors.storage.Remove(ctx, id.Ref().ID, storiface.FTPiece, true, nil)
74+
return sb.Sectors.storage.Remove(ctx, id.Ref().ID, storiface.FTPiece, true, nil)
7575
}

‎lib/ffi/scrub_funcs.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
)
1919

2020
func (sb *SealCalls) CheckUnsealedCID(ctx context.Context, s storiface.SectorRef) (cid.Cid, error) {
21-
reader, err := sb.sectors.storage.ReaderSeq(ctx, s, storiface.FTUnsealed)
21+
reader, err := sb.Sectors.storage.ReaderSeq(ctx, s, storiface.FTUnsealed)
2222
if err != nil {
2323
return cid.Undef, xerrors.Errorf("getting unsealed sector reader: %w", err)
2424
}

‎lib/ffi/sdr_funcs.go‎

Lines changed: 29 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
"github.com/ipfs/go-cid"
1515
logging "github.com/ipfs/go-log/v2"
1616
"github.com/puzpuzpuz/xsync/v2"
17+
"github.com/samber/lo"
1718
"golang.org/x/xerrors"
1819

1920
"github.com/filecoin-project/curio/harmony/harmonytask"
@@ -43,19 +44,19 @@ type ExternPrecommit2 func(ctx context.Context, sector storiface.SectorRef, cach
4344
}
4445
*/
4546
type SealCalls struct {
46-
sectors *storageProvider
47+
Sectors *storageProvider
4748

4849
/*// externCalls cointain overrides for calling alternative sealing logic
4950
externCalls ExternalSealer*/
5051
}
5152

5253
func NewSealCalls(st *paths.Remote, ls *paths.Local, si paths.SectorIndex) *SealCalls {
5354
return &SealCalls{
54-
sectors: &storageProvider{
55+
Sectors: &storageProvider{
5556
storage: st,
5657
localStore: ls,
5758
sindex: si,
58-
storageReservations: xsync.NewIntegerMapOf[harmonytask.TaskID, *StorageReservation](),
59+
storageReservations: xsync.NewIntegerMapOf[harmonytask.TaskID, []*StorageReservation](),
5960
},
6061
}
6162
}
@@ -64,7 +65,7 @@ type storageProvider struct {
6465
storage *paths.Remote
6566
localStore *paths.Local
6667
sindex paths.SectorIndex
67-
storageReservations *xsync.MapOf[harmonytask.TaskID, *StorageReservation]
68+
storageReservations *xsync.MapOf[harmonytask.TaskID, []*StorageReservation]
6869
}
6970

7071
func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask.TaskID, sector storiface.SectorRef, existing, allocate storiface.SectorFileType, sealing storiface.PathType) (fspaths, ids storiface.SectorPaths, release func(dontDeclare ...storiface.SectorFileType), err error) {
@@ -74,7 +75,12 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
7475
var ok bool
7576
var resv *StorageReservation
7677
if taskID != nil {
77-
resv, ok = l.storageReservations.Load(*taskID)
78+
resvs, rok := l.storageReservations.Load(*taskID)
79+
if rok {
80+
resv, ok = lo.Find(resvs, func(res *StorageReservation) bool {
81+
return res.SectorRef.ID() == sector.ID
82+
})
83+
}
7884
}
7985
if ok && resv != nil {
8086
if resv.Alloc != allocate || resv.Existing != existing {
@@ -144,7 +150,7 @@ func (l *storageProvider) AcquireSector(ctx context.Context, taskID *harmonytask
144150
}
145151

146152
func (sb *SealCalls) GenerateSDR(ctx context.Context, taskID harmonytask.TaskID, into storiface.SectorFileType, sector storiface.SectorRef, ticket abi.SealRandomness, commDcid cid.Cid) error {
147-
paths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, &taskID, sector, storiface.FTNone, into, storiface.PathSealing)
153+
paths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, &taskID, sector, storiface.FTNone, into, storiface.PathSealing)
148154
if err != nil {
149155
return xerrors.Errorf("acquiring sector paths: %w", err)
150156
}
@@ -223,7 +229,7 @@ func (sb *SealCalls) ensureOneCopy(ctx context.Context, sid abi.SectorID, pathID
223229

224230
log.Debugw("ensureOneCopy", "sector", sid, "type", fileType, "keep", keepIn)
225231

226-
if err := sb.sectors.storage.Remove(ctx, sid, fileType, true, keepIn); err != nil {
232+
if err := sb.Sectors.storage.Remove(ctx, sid, fileType, true, keepIn); err != nil {
227233
return err
228234
}
229235
}
@@ -237,7 +243,7 @@ func (sb *SealCalls) TreeRC(ctx context.Context, task *harmonytask.TaskID, secto
237243
return cid.Undef, cid.Undef, xerrors.Errorf("make phase1 output: %w", err)
238244
}
239245

240-
fspaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTSealed, storiface.PathSealing)
246+
fspaths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTSealed, storiface.PathSealing)
241247
if err != nil {
242248
return cid.Undef, cid.Undef, xerrors.Errorf("acquiring sector paths: %w", err)
243249
}
@@ -352,7 +358,7 @@ func (sb *SealCalls) GenerateSynthPoRep() {
352358
}
353359

354360
func (sb *SealCalls) PoRepSnark(ctx context.Context, sn storiface.SectorRef, sealed, unsealed cid.Cid, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness) ([]byte, error) {
355-
vproof, err := sb.sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
361+
vproof, err := sb.Sectors.storage.GeneratePoRepVanillaProof(ctx, sn, sealed, unsealed, ticket, seed)
356362
if err != nil {
357363
return nil, xerrors.Errorf("failed to generate vanilla proof: %w", err)
358364
}
@@ -498,7 +504,7 @@ func (sb *SealCalls) makePhase1Out(unsCid cid.Cid, spt abi.RegisteredSealProof)
498504
}
499505

500506
func (sb *SealCalls) LocalStorage(ctx context.Context) ([]storiface.StoragePath, error) {
501-
return sb.sectors.localStore.Local(ctx)
507+
return sb.Sectors.localStore.Local(ctx)
502508
}
503509

504510
func changePathType(path string, newType storiface.SectorFileType) (string, error) {
@@ -549,7 +555,7 @@ func (sb *SealCalls) GenerateUnsealedSector(ctx context.Context, sector storifac
549555

550556
defer func() {
551557
// We don't pass FTUnsealed to Acquire, so releaseSector won't declare it. Do it here.
552-
if err := sb.sectors.sindex.StorageDeclareSector(ctx, storiface.ID(pathIDs.Unsealed), sector.ID, storiface.FTUnsealed, true); err != nil {
558+
if err := sb.Sectors.sindex.StorageDeclareSector(ctx, storiface.ID(pathIDs.Unsealed), sector.ID, storiface.FTUnsealed, true); err != nil {
553559
log.Errorf("declare unsealed sector error: %s", err)
554560
}
555561
}()
@@ -635,7 +641,7 @@ func TruncateAndMoveUnsealed(tempUnsealed, unsealed string, ssize abi.SectorSize
635641
}
636642

637643
func (sb *SealCalls) FinalizeSector(ctx context.Context, sector storiface.SectorRef, keepUnsealed bool) error {
638-
sectorPaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, nil, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
644+
sectorPaths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, nil, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
639645
if err != nil {
640646
return xerrors.Errorf("acquiring sector paths: %w", err)
641647
}
@@ -694,11 +700,16 @@ func (sb *SealCalls) MoveStorage(ctx context.Context, sector storiface.SectorRef
694700

695701
var opts []storiface.AcquireOption
696702
if taskID != nil {
697-
resv, ok := sb.sectors.storageReservations.Load(*taskID)
703+
resvs, ok := sb.Sectors.storageReservations.Load(*taskID)
698704
// if the reservation is missing MoveStorage will simply create one internally. This is fine as the reservation
699705
// will only be missing when the node is restarting, which means that the missing reservations will get recreated
700706
// anyways, and before we start claiming other tasks.
701707
if ok {
708+
if len(resvs) != 1 {
709+
return xerrors.Errorf("task %d has %d reservations, expected 1", taskID, len(resvs))
710+
}
711+
resv := resvs[0]
712+
702713
defer resv.Release()
703714

704715
if resv.Alloc != storiface.FTNone {
@@ -712,13 +723,13 @@ func (sb *SealCalls) MoveStorage(ctx context.Context, sector storiface.SectorRef
712723
}
713724
}
714725

715-
err := sb.sectors.storage.MoveStorage(ctx, sector, toMove, opts...)
726+
err := sb.Sectors.storage.MoveStorage(ctx, sector, toMove, opts...)
716727
if err != nil {
717728
return xerrors.Errorf("moving storage: %w", err)
718729
}
719730

720731
for _, fileType := range toMove.AllSet() {
721-
if err := sb.sectors.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
732+
if err := sb.Sectors.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
722733
return xerrors.Errorf("rm copies (t:%s, s:%v): %w", fileType, sector, err)
723734
}
724735
}
@@ -727,7 +738,7 @@ func (sb *SealCalls) MoveStorage(ctx context.Context, sector storiface.SectorRef
727738
}
728739

729740
func (sb *SealCalls) sectorStorageType(ctx context.Context, sector storiface.SectorRef, ft storiface.SectorFileType) (sectorFound bool, ptype storiface.PathType, err error) {
730-
stores, err := sb.sectors.sindex.StorageFindSector(ctx, sector.ID, ft, 0, false)
741+
stores, err := sb.Sectors.sindex.StorageFindSector(ctx, sector.ID, ft, 0, false)
731742
if err != nil {
732743
return false, "", xerrors.Errorf("finding sector: %w", err)
733744
}
@@ -746,7 +757,7 @@ func (sb *SealCalls) sectorStorageType(ctx context.Context, sector storiface.Sec
746757

747758
// PreFetch fetches the sector file to local storage before SDR and TreeRC Tasks
748759
func (sb *SealCalls) PreFetch(ctx context.Context, sector storiface.SectorRef, task *harmonytask.TaskID) (fsPath, pathID storiface.SectorPaths, releaseSector func(...storiface.SectorFileType), err error) {
749-
fsPath, pathID, releaseSector, err = sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
760+
fsPath, pathID, releaseSector, err = sb.Sectors.AcquireSector(ctx, task, sector, storiface.FTCache, storiface.FTNone, storiface.PathSealing)
750761
if err != nil {
751762
return storiface.SectorPaths{}, storiface.SectorPaths{}, nil, xerrors.Errorf("acquiring sector paths: %w", err)
752763
}
@@ -782,7 +793,7 @@ func (sb *SealCalls) TreeD(ctx context.Context, sector storiface.SectorRef, unse
782793
}
783794

784795
func (sb *SealCalls) SyntheticProofs(ctx context.Context, task *harmonytask.TaskID, sector storiface.SectorRef, sealed cid.Cid, unsealed cid.Cid, randomness abi.SealRandomness, pieces []abi.PieceInfo, keepUnsealed bool) error {
785-
fspaths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, task, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathSealing)
796+
fspaths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, task, sector, storiface.FTCache|storiface.FTSealed, storiface.FTNone, storiface.PathSealing)
786797
if err != nil {
787798
return xerrors.Errorf("acquiring sector paths: %w", err)
788799
}

‎lib/ffi/snap_funcs.go‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (sb *SealCalls) EncodeUpdate(
4141
noDecl = storiface.FTUnsealed
4242
}
4343

44-
paths, pathIDs, releaseSector, err := sb.sectors.AcquireSector(ctx, &taskID, sector, storiface.FTNone, storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTUnsealed, storiface.PathSealing)
44+
paths, pathIDs, releaseSector, err := sb.Sectors.AcquireSector(ctx, &taskID, sector, storiface.FTNone, storiface.FTUpdate|storiface.FTUpdateCache|storiface.FTUnsealed, storiface.PathSealing)
4545
if err != nil {
4646
return cid.Undef, cid.Undef, xerrors.Errorf("acquiring sector paths: %w", err)
4747
}
@@ -133,7 +133,7 @@ func (sb *SealCalls) EncodeUpdate(
133133

134134
log.Debugw("get key data", "keyPath", keyPath, "keyCachePath", keyCachePath, "sectorID", sector.ID, "taskID", taskID)
135135

136-
r, err := sb.sectors.storage.ReaderSeq(ctx, sector, storiface.FTSealed)
136+
r, err := sb.Sectors.storage.ReaderSeq(ctx, sector, storiface.FTSealed)
137137
if err != nil {
138138
return cid.Undef, cid.Undef, xerrors.Errorf("getting sealed sector reader: %w", err)
139139
}
@@ -177,7 +177,7 @@ func (sb *SealCalls) EncodeUpdate(
177177

178178
// fetch cache
179179
var buf bytes.Buffer // usually 73.2 MiB
180-
err = sb.sectors.storage.ReadMinCacheInto(ctx, sector, storiface.FTCache, &buf)
180+
err = sb.Sectors.storage.ReadMinCacheInto(ctx, sector, storiface.FTCache, &buf)
181181
if err != nil {
182182
return cid.Undef, cid.Undef, xerrors.Errorf("reading cache: %w", err)
183183
}
@@ -269,7 +269,7 @@ func (sb *SealCalls) EncodeUpdate(
269269
}
270270

271271
func (sb *SealCalls) ProveUpdate(ctx context.Context, proofType abi.RegisteredUpdateProof, sector storiface.SectorRef, key, sealed, unsealed cid.Cid) ([]byte, error) {
272-
jsonb, err := sb.sectors.storage.ReadSnapVanillaProof(ctx, sector)
272+
jsonb, err := sb.Sectors.storage.ReadSnapVanillaProof(ctx, sector)
273273
if err != nil {
274274
return nil, xerrors.Errorf("read snap vanilla proof: %w", err)
275275
}
@@ -301,11 +301,16 @@ func (sb *SealCalls) MoveStorageSnap(ctx context.Context, sector storiface.Secto
301301

302302
var opts []storiface.AcquireOption
303303
if taskID != nil {
304-
resv, ok := sb.sectors.storageReservations.Load(*taskID)
304+
resvs, ok := sb.Sectors.storageReservations.Load(*taskID)
305305
// if the reservation is missing MoveStorage will simply create one internally. This is fine as the reservation
306306
// will only be missing when the node is restarting, which means that the missing reservations will get recreated
307307
// anyways, and before we start claiming other tasks.
308308
if ok {
309+
if len(resvs) != 1 {
310+
return xerrors.Errorf("task %d has %d reservations, expected 1", taskID, len(resvs))
311+
}
312+
resv := resvs[0]
313+
309314
defer resv.Release()
310315

311316
if resv.Alloc != storiface.FTNone {
@@ -319,13 +324,13 @@ func (sb *SealCalls) MoveStorageSnap(ctx context.Context, sector storiface.Secto
319324
}
320325
}
321326

322-
err := sb.sectors.storage.MoveStorage(ctx, sector, toMove, opts...)
327+
err := sb.Sectors.storage.MoveStorage(ctx, sector, toMove, opts...)
323328
if err != nil {
324329
return xerrors.Errorf("moving storage: %w", err)
325330
}
326331

327332
for _, fileType := range toMove.AllSet() {
328-
if err := sb.sectors.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
333+
if err := sb.Sectors.storage.RemoveCopies(ctx, sector.ID, fileType); err != nil {
329334
return xerrors.Errorf("rm copies (t:%s, s:%v): %w", fileType, sector, err)
330335
}
331336
}

0 commit comments

Comments
 (0)