Skip to content

Commit 82b7727

Browse files
ognotsognots
committed
use ChainExportRangeInternal to write snapshots directly to disk
we expect this to be perform much better, than using the ChainExport method and streaming it through the API. this commit only guarantees that filecoin-chain-archiver binary builds. writing some tests and doing manual testing is still required.
1 parent 1e25ee8 commit 82b7727

File tree

5 files changed

+254
-462
lines changed

5 files changed

+254
-462
lines changed

‎cmd/filecoin-chain-archiver/cmds/create.go

Lines changed: 37 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"io"
99
"math/rand"
1010
"net/url"
11+
"os"
12+
"path/filepath"
1113
"strings"
1214
"syscall"
1315
"time"
@@ -62,16 +64,6 @@ type multi struct {
6264
cs []io.Closer
6365
}
6466

65-
func MultiWriteCloser(ws ...io.Writer) io.WriteCloser {
66-
m := &multi{Writer: io.MultiWriter(ws...)}
67-
for _, w := range ws {
68-
if c, ok := w.(io.Closer); ok {
69-
m.cs = append(m.cs, c)
70-
}
71-
}
72-
return m
73-
}
74-
7567
func (m *multi) Close() error {
7668
var first error
7769
for _, c := range m.cs {
@@ -187,12 +179,6 @@ var cmdCreate = &cli.Command{
187179
EnvVars: []string{"FCA_CREATE_HEIGHT"},
188180
Value: 0,
189181
},
190-
&cli.IntFlag{
191-
Name: "stateroot-count",
192-
Usage: "number of stateroots to included in snapshot",
193-
EnvVars: []string{"FCA_CREATE_STATEROOT_COUNT"},
194-
Value: 2000,
195-
},
196182
&cli.DurationFlag{
197183
Name: "progress-update",
198184
Usage: "how frequenty to provide provide update logs",
@@ -217,7 +203,6 @@ var cmdCreate = &cli.Command{
217203
flagConfidence := cctx.Int("confidence")
218204
flagHeight := cctx.Int("height")
219205
flagAfter := cctx.Int("after")
220-
flagStaterootCount := cctx.Int("stateroot-count")
221206

222207
u, err := url.Parse(flagBucketEndpoint)
223208
if err != nil {
@@ -299,7 +284,13 @@ var cmdCreate = &cli.Command{
299284
time.Sleep(time.Until(t))
300285
bt := time.Now()
301286

302-
tsk, err := cm.GetTipset(ctx, height)
287+
headTs, err := cm.GetTipset(ctx, height)
288+
if err != nil {
289+
return err
290+
}
291+
292+
tailHeight := height - 2880
293+
tailTs, err := cm.GetTipset(ctx, tailHeight)
303294
if err != nil {
304295
return err
305296
}
@@ -324,7 +315,7 @@ var cmdCreate = &cli.Command{
324315
logger.Infow("iteration", "value", iteration)
325316
cm.ShiftStartNode(iteration)
326317

327-
node, peerID, err := cm.GetNodeWithTipSet(ctx, tsk, filterList)
318+
node, peerID, err := cm.GetNodeWithTipSet(ctx, headTs, filterList)
328319
if err != nil {
329320
return err
330321
}
@@ -340,12 +331,7 @@ var cmdCreate = &cli.Command{
340331
return xerrors.Errorf("failed to aquire lock")
341332
}
342333

343-
rr, wr := io.Pipe()
344-
rc, wc := io.Pipe()
345-
346-
mw := MultiWriteCloser(wr, wc)
347-
348-
e := export.NewExport(node, tsk, abi.ChainEpoch(flagStaterootCount), true, mw)
334+
e := export.NewExport(node, headTs, tailTs)
349335
errCh := make(chan error)
350336
go func() {
351337
errCh <- e.Export(ctx)
@@ -372,21 +358,36 @@ var cmdCreate = &cli.Command{
372358
}
373359
}()
374360

361+
name, err := filepath.Glob("snapshot_" + string(tailHeight) + "_" + string(height) + "*.car")
362+
if err != nil {
363+
return err
364+
}
365+
rrfn := filepath.Join(cctx.String("repo"), name[0])
366+
rr, err := os.OpenFile(rrfn, os.O_RDONLY, 444)
367+
if err != nil {
368+
return err
369+
}
370+
defer rr.Close()
371+
372+
cname := name[0] + ".zstd"
373+
rcfn := filepath.Join(cctx.String("repo"), string(cname[0]))
374+
rc, err := os.OpenFile(rcfn, os.O_RDONLY, 444)
375+
if err != nil {
376+
return err
377+
}
378+
defer rc.Close()
379+
375380
go func() {
376-
var lastSize int
381+
var lastSize int64
377382
for {
378383
select {
379384
case <-time.After(flagProgressUpdate):
380-
size, done := e.Progress()
385+
size := e.Progress(rrfn)
381386
if size == 0 {
382387
continue
383388
}
384389

385-
if done {
386-
return
387-
}
388-
389-
logger.Infow("update", "total", size, "speed", (size-lastSize)/int(flagProgressUpdate/time.Second))
390+
logger.Infow("update", "total", size, "speed", (size-lastSize)/int64(flagProgressUpdate/time.Second))
390391
lastSize = size
391392
}
392393
}
@@ -432,23 +433,21 @@ var cmdCreate = &cli.Command{
432433
return err
433434
}
434435

435-
t := export.TimeAtHeight(gtp, height, 30*time.Second)
436-
437-
name := fmt.Sprintf("%d_%s", height, t.Format("2006_01_02T15_04_05Z"))
436+
//t := export.TimeAtHeight(gtp, height, 30*time.Second)
438437

439-
logger.Infow("object", "name", name)
438+
logger.Infow("object", "name", name[0])
440439

441440
g, ctxGroup := errgroup.WithContext(ctx)
442441
var siRaw *snapshotInfo
443442
var siCompressed *snapshotInfo
444443
g.Go(func() error {
445444
var err error
446-
siRaw, err = runUploadRaw(ctxGroup, minioClient, flagBucket, flagNamePrefix, flagRetrievalEndpointPrefix, name, peerID, bt, rr)
445+
siRaw, err = runUploadRaw(ctxGroup, minioClient, flagBucket, flagNamePrefix, flagRetrievalEndpointPrefix, name[0], peerID, bt, rr)
447446
return err
448447
})
449448
g.Go(func() error {
450449
var err error
451-
siCompressed, err = runUploadCompressed(ctxGroup, minioClient, flagBucket, flagNamePrefix, flagRetrievalEndpointPrefix, name, peerID, bt, rc)
450+
siCompressed, err = runUploadCompressed(ctxGroup, minioClient, flagBucket, flagNamePrefix, flagRetrievalEndpointPrefix, name[0], peerID, bt, rc)
452451
return err
453452
})
454453
if err := g.Wait(); err != nil {

‎go.mod

Lines changed: 49 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@ go 1.19
44

55
require (
66
github.com/BurntSushi/toml v1.1.0
7-
github.com/filecoin-project/go-jsonrpc v0.2.1
7+
github.com/filecoin-project/go-jsonrpc v0.2.3
88
github.com/filecoin-project/go-state-types v0.10.0
9-
github.com/filecoin-project/lotus v1.20.3
9+
github.com/filecoin-project/lotus v1.21.0-rc1
1010
github.com/gorilla/mux v1.8.0
1111
github.com/ipfs/go-log/v2 v2.5.1
12-
github.com/klauspost/compress v1.15.10
12+
github.com/klauspost/compress v1.15.12
1313
github.com/minio/minio-go/v7 v7.0.24
14-
github.com/prometheus/client_golang v1.13.0
14+
github.com/prometheus/client_golang v1.14.0
1515
github.com/slok/go-http-metrics v0.10.0
1616
github.com/stretchr/testify v1.8.1
1717
github.com/urfave/cli/v2 v2.16.3
18-
golang.org/x/sync v0.0.0-20220907140024-f12130a52804
18+
golang.org/x/sync v0.1.0
1919
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2
2020
)
2121

@@ -28,29 +28,31 @@ require (
2828
github.com/benbjohnson/clock v1.3.0 // indirect
2929
github.com/beorn7/perks v1.0.1 // indirect
3030
github.com/cespare/xxhash v1.1.0 // indirect
31-
github.com/cespare/xxhash/v2 v2.1.2 // indirect
31+
github.com/cespare/xxhash/v2 v2.2.0 // indirect
3232
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
3333
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
3434
github.com/daaku/go.zipexe v1.0.2 // indirect
3535
github.com/davecgh/go-spew v1.1.1 // indirect
3636
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
37-
github.com/dgraph-io/badger/v2 v2.2007.3 // indirect
37+
github.com/dgraph-io/badger/v2 v2.2007.4 // indirect
3838
github.com/dgraph-io/ristretto v0.1.0 // indirect
39-
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
39+
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
4040
github.com/dustin/go-humanize v1.0.0 // indirect
41+
github.com/elastic/gosigar v0.14.2 // indirect
4142
github.com/filecoin-project/go-address v1.1.0 // indirect
4243
github.com/filecoin-project/go-amt-ipld/v2 v2.1.0 // indirect
4344
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0 // indirect
4445
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect
4546
github.com/filecoin-project/go-bitfield v0.2.4 // indirect
4647
github.com/filecoin-project/go-cbor-util v0.0.1 // indirect
4748
github.com/filecoin-project/go-crypto v0.0.1 // indirect
48-
github.com/filecoin-project/go-data-transfer v1.15.2 // indirect
49-
github.com/filecoin-project/go-fil-markets v1.25.2 // indirect
49+
github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc4 // indirect
50+
github.com/filecoin-project/go-fil-markets v1.27.0-rc1 // indirect
5051
github.com/filecoin-project/go-hamt-ipld v0.1.5 // indirect
5152
github.com/filecoin-project/go-hamt-ipld/v2 v2.0.0 // indirect
5253
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.0 // indirect
5354
github.com/filecoin-project/go-padreader v0.0.1 // indirect
55+
github.com/filecoin-project/go-statemachine v1.0.3 // indirect
5456
github.com/filecoin-project/go-statestore v0.2.0 // indirect
5557
github.com/filecoin-project/specs-actors v0.9.15 // indirect
5658
github.com/filecoin-project/specs-actors/v2 v2.3.6 // indirect
@@ -72,39 +74,39 @@ require (
7274
github.com/google/uuid v1.3.0 // indirect
7375
github.com/gorilla/websocket v1.5.0 // indirect
7476
github.com/hako/durafmt v0.0.0-20200710122514-c0fb7b4da026 // indirect
75-
github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 // indirect
77+
github.com/hannahhoward/cbor-gen-for v0.0.0-20230214144701-5d17c9d5243c // indirect
7678
github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e // indirect
77-
github.com/hashicorp/golang-lru v0.5.4 // indirect
79+
github.com/hashicorp/golang-lru v0.6.0 // indirect
80+
github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
7881
github.com/icza/backscanner v0.0.0-20210726202459-ac2ffc679f94 // indirect
7982
github.com/ipfs/bbloom v0.0.4 // indirect
8083
github.com/ipfs/go-block-format v0.1.1 // indirect
81-
github.com/ipfs/go-blockservice v0.4.0 // indirect
84+
github.com/ipfs/go-blockservice v0.5.0 // indirect
8285
github.com/ipfs/go-cid v0.3.2 // indirect
8386
github.com/ipfs/go-datastore v0.6.0 // indirect
84-
github.com/ipfs/go-ds-badger2 v0.1.2 // indirect
87+
github.com/ipfs/go-ds-badger2 v0.1.3 // indirect
8588
github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
8689
github.com/ipfs/go-ds-measure v0.2.0 // indirect
8790
github.com/ipfs/go-fs-lock v0.0.7 // indirect
88-
github.com/ipfs/go-graphsync v0.13.2 // indirect
91+
github.com/ipfs/go-graphsync v0.14.3 // indirect
8992
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
90-
github.com/ipfs/go-ipfs-cmds v0.7.0 // indirect
93+
github.com/ipfs/go-ipfs-cmds v0.8.2 // indirect
9194
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
9295
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
93-
github.com/ipfs/go-ipfs-files v0.1.1 // indirect
94-
github.com/ipfs/go-ipfs-http-client v0.4.0 // indirect
96+
github.com/ipfs/go-ipfs-http-client v0.5.0 // indirect
9597
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
9698
github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
9799
github.com/ipfs/go-ipld-format v0.4.0 // indirect
98100
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
99-
github.com/ipfs/go-libipfs v0.4.1 // indirect
101+
github.com/ipfs/go-libipfs v0.5.0 // indirect
100102
github.com/ipfs/go-log v1.0.5 // indirect
101-
github.com/ipfs/go-merkledag v0.8.1 // indirect
103+
github.com/ipfs/go-merkledag v0.9.0 // indirect
102104
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
103105
github.com/ipfs/go-path v0.3.0 // indirect
104-
github.com/ipfs/go-unixfs v0.4.0 // indirect
106+
github.com/ipfs/go-unixfs v0.4.3 // indirect
105107
github.com/ipfs/go-verifcid v0.0.2 // indirect
106-
github.com/ipfs/interface-go-ipfs-core v0.7.0 // indirect
107-
github.com/ipld/go-car v0.4.0 // indirect
108+
github.com/ipfs/interface-go-ipfs-core v0.11.1 // indirect
109+
github.com/ipld/go-car v0.5.0 // indirect
108110
github.com/ipld/go-codec-dagpb v1.5.0 // indirect
109111
github.com/ipld/go-ipld-prime v0.20.0 // indirect
110112
github.com/ipld/go-ipld-selector-text-lite v0.0.1 // indirect
@@ -115,18 +117,15 @@ require (
115117
github.com/json-iterator/go v1.1.12 // indirect
116118
github.com/kelseyhightower/envconfig v1.4.0 // indirect
117119
github.com/klauspost/cpuid v1.3.1 // indirect
118-
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
120+
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
119121
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
120122
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
121-
github.com/libp2p/go-libp2p v0.23.4 // indirect
122-
github.com/libp2p/go-libp2p-core v0.20.1 // indirect
123-
github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect
124-
github.com/libp2p/go-msgio v0.2.0 // indirect
125-
github.com/libp2p/go-openssl v0.1.0 // indirect
123+
github.com/libp2p/go-libp2p v0.26.2 // indirect
124+
github.com/libp2p/go-libp2p-pubsub v0.9.3 // indirect
125+
github.com/libp2p/go-msgio v0.3.0 // indirect
126126
github.com/magefile/mage v1.9.0 // indirect
127-
github.com/mattn/go-isatty v0.0.16 // indirect
128-
github.com/mattn/go-pointer v0.0.1 // indirect
129-
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
127+
github.com/mattn/go-isatty v0.0.17 // indirect
128+
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
130129
github.com/miekg/dns v1.1.50 // indirect
131130
github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect
132131
github.com/minio/md5-simd v1.1.0 // indirect
@@ -136,49 +135,48 @@ require (
136135
github.com/modern-go/reflect2 v1.0.2 // indirect
137136
github.com/mr-tron/base58 v1.2.0 // indirect
138137
github.com/multiformats/go-base32 v0.1.0 // indirect
139-
github.com/multiformats/go-base36 v0.1.0 // indirect
138+
github.com/multiformats/go-base36 v0.2.0 // indirect
140139
github.com/multiformats/go-multiaddr v0.8.0 // indirect
141140
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
142141
github.com/multiformats/go-multibase v0.1.1 // indirect
143142
github.com/multiformats/go-multicodec v0.8.0 // indirect
144143
github.com/multiformats/go-multihash v0.2.1 // indirect
145-
github.com/multiformats/go-varint v0.0.6 // indirect
144+
github.com/multiformats/go-multistream v0.4.1 // indirect
145+
github.com/multiformats/go-varint v0.0.7 // indirect
146146
github.com/nkovacs/streamquote v1.0.0 // indirect
147147
github.com/opentracing/opentracing-go v1.2.0 // indirect
148148
github.com/pkg/errors v0.9.1 // indirect
149149
github.com/pmezard/go-difflib v1.0.0 // indirect
150-
github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e // indirect
151-
github.com/prometheus/client_model v0.2.0 // indirect
150+
github.com/polydawn/refmt v0.89.0 // indirect
151+
github.com/prometheus/client_model v0.3.0 // indirect
152152
github.com/prometheus/common v0.37.0 // indirect
153153
github.com/prometheus/procfs v0.8.0 // indirect
154154
github.com/raulk/clock v1.1.0 // indirect
155155
github.com/rs/cors v1.7.0 // indirect
156156
github.com/rs/xid v1.2.1 // indirect
157157
github.com/russross/blackfriday/v2 v2.1.0 // indirect
158158
github.com/shirou/gopsutil v2.18.12+incompatible // indirect
159-
github.com/sirupsen/logrus v1.8.1 // indirect
160-
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
159+
github.com/sirupsen/logrus v1.9.0 // indirect
161160
github.com/spaolacci/murmur3 v1.1.0 // indirect
162161
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
163162
github.com/valyala/bytebufferpool v1.0.0 // indirect
164163
github.com/valyala/fasttemplate v1.2.1 // indirect
165164
github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect
166-
github.com/whyrusleeping/cbor-gen v0.0.0-20221021053955-c138aae13722 // indirect
167-
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
165+
github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
168166
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
169-
go.opencensus.io v0.23.0 // indirect
170-
go.opentelemetry.io/otel v1.11.1 // indirect
171-
go.opentelemetry.io/otel/trace v1.11.1 // indirect
167+
go.opencensus.io v0.24.0 // indirect
168+
go.opentelemetry.io/otel v1.12.0 // indirect
169+
go.opentelemetry.io/otel/trace v1.12.0 // indirect
172170
go.uber.org/atomic v1.10.0 // indirect
173-
go.uber.org/multierr v1.8.0 // indirect
174-
go.uber.org/zap v1.23.0 // indirect
171+
go.uber.org/multierr v1.9.0 // indirect
172+
go.uber.org/zap v1.24.0 // indirect
175173
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
176-
golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
177-
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
178-
golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5 // indirect
179-
golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8 // indirect
180-
golang.org/x/text v0.3.7 // indirect
181-
golang.org/x/tools v0.1.12 // indirect
174+
golang.org/x/crypto v0.5.0 // indirect
175+
golang.org/x/mod v0.7.0 // indirect
176+
golang.org/x/net v0.7.0 // indirect
177+
golang.org/x/sys v0.5.0 // indirect
178+
golang.org/x/text v0.7.0 // indirect
179+
golang.org/x/tools v0.3.0 // indirect
182180
google.golang.org/protobuf v1.28.1 // indirect
183181
gopkg.in/ini.v1 v1.57.0 // indirect
184182
gopkg.in/yaml.v3 v3.0.1 // indirect

0 commit comments

Comments
 (0)