Skip to content

Commit ad44475

Browse files
authored
Merge pull request #14785 from Arachnid/downloaddb
cmd: Added support for downloading to another DB instance
2 parents cefeb58 + 35767df commit ad44475

File tree

4 files changed

+243
-8
lines changed

4 files changed

+243
-8
lines changed

cmd/geth/chaincmd.go

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,9 @@ import (
3131
"github.com/ethereum/go-ethereum/core"
3232
"github.com/ethereum/go-ethereum/core/state"
3333
"github.com/ethereum/go-ethereum/core/types"
34+
"github.com/ethereum/go-ethereum/eth/downloader"
3435
"github.com/ethereum/go-ethereum/ethdb"
36+
"github.com/ethereum/go-ethereum/event"
3537
"github.com/ethereum/go-ethereum/log"
3638
"github.com/ethereum/go-ethereum/trie"
3739
"github.com/syndtr/goleveldb/leveldb/util"
@@ -71,7 +73,7 @@ It expects the genesis file as argument.`,
7173
The import command imports blocks from an RLP-encoded form. The form can be one file
7274
with several RLP-encoded blocks, or several files can be used.
7375
74-
If only one file is used, import error will result in failure. If several files are used,
76+
If only one file is used, import error will result in failure. If several files are used,
7577
processing will proceed even if an individual RLP-file import failure occurs.`,
7678
}
7779
exportCommand = cli.Command{
@@ -90,6 +92,23 @@ Requires a first argument of the file to write to.
9092
Optional second and third arguments control the first and
9193
last block to write. In this mode, the file will be appended
9294
if already existing.`,
95+
}
96+
copydbCommand = cli.Command{
97+
Action: utils.MigrateFlags(copyDb),
98+
Name: "copydb",
99+
Usage: "Create a local chain from a target chaindata folder",
100+
ArgsUsage: "<sourceChaindataDir>",
101+
Flags: []cli.Flag{
102+
utils.DataDirFlag,
103+
utils.CacheFlag,
104+
utils.SyncModeFlag,
105+
utils.FakePoWFlag,
106+
utils.TestnetFlag,
107+
utils.RinkebyFlag,
108+
},
109+
Category: "BLOCKCHAIN COMMANDS",
110+
Description: `
111+
The first argument must be the directory containing the blockchain to download from`,
93112
}
94113
removedbCommand = cli.Command{
95114
Action: utils.MigrateFlags(removeDB),
@@ -268,6 +287,54 @@ func exportChain(ctx *cli.Context) error {
268287
return nil
269288
}
270289

290+
func copyDb(ctx *cli.Context) error {
291+
// Ensure we have a source chain directory to copy
292+
if len(ctx.Args()) != 1 {
293+
utils.Fatalf("Source chaindata directory path argument missing")
294+
}
295+
// Initialize a new chain for the running node to sync into
296+
stack := makeFullNode(ctx)
297+
chain, chainDb := utils.MakeChain(ctx, stack)
298+
299+
syncmode := *utils.GlobalTextMarshaler(ctx, utils.SyncModeFlag.Name).(*downloader.SyncMode)
300+
dl := downloader.New(syncmode, chainDb, new(event.TypeMux), chain, nil, nil)
301+
302+
// Create a source peer to satisfy downloader requests from
303+
db, err := ethdb.NewLDBDatabase(ctx.Args().First(), ctx.GlobalInt(utils.CacheFlag.Name), 256)
304+
if err != nil {
305+
return err
306+
}
307+
hc, err := core.NewHeaderChain(db, chain.Config(), chain.Engine(), func() bool { return false })
308+
if err != nil {
309+
return err
310+
}
311+
peer := downloader.NewFakePeer("local", db, hc, dl)
312+
if err = dl.RegisterPeer("local", 63, peer); err != nil {
313+
return err
314+
}
315+
// Synchronise with the simulated peer
316+
start := time.Now()
317+
318+
currentHeader := hc.CurrentHeader()
319+
if err = dl.Synchronise("local", currentHeader.Hash(), hc.GetTd(currentHeader.Hash(), currentHeader.Number.Uint64()), syncmode); err != nil {
320+
return err
321+
}
322+
for dl.Synchronising() {
323+
time.Sleep(10 * time.Millisecond)
324+
}
325+
fmt.Printf("Database copy done in %v\n", time.Since(start))
326+
327+
// Compact the entire database to remove any sync overhead
328+
start = time.Now()
329+
fmt.Println("Compacting entire database...")
330+
if err = chainDb.(*ethdb.LDBDatabase).LDB().CompactRange(util.Range{}); err != nil {
331+
utils.Fatalf("Compaction failed: %v", err)
332+
}
333+
fmt.Printf("Compaction done in %v.\n\n", time.Since(start))
334+
335+
return nil
336+
}
337+
271338
func removeDB(ctx *cli.Context) error {
272339
stack, _ := makeConfigNode(ctx)
273340

cmd/geth/main.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ func init() {
146146
initCommand,
147147
importCommand,
148148
exportCommand,
149+
copydbCommand,
149150
removedbCommand,
150151
dumpCommand,
151152
// See monitorcmd.go:

cmd/utils/flags.go

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ import (
3131
"github.com/ethereum/go-ethereum/accounts"
3232
"github.com/ethereum/go-ethereum/accounts/keystore"
3333
"github.com/ethereum/go-ethereum/common"
34+
"github.com/ethereum/go-ethereum/consensus"
35+
"github.com/ethereum/go-ethereum/consensus/clique"
3436
"github.com/ethereum/go-ethereum/consensus/ethash"
3537
"github.com/ethereum/go-ethereum/core"
3638
"github.com/ethereum/go-ethereum/core/state"
@@ -1086,17 +1088,22 @@ func MakeChain(ctx *cli.Context, stack *node.Node) (chain *core.BlockChain, chai
10861088
var err error
10871089
chainDb = MakeChainDatabase(ctx, stack)
10881090

1089-
engine := ethash.NewFaker()
1090-
if !ctx.GlobalBool(FakePoWFlag.Name) {
1091-
engine = ethash.New(
1092-
stack.ResolvePath(eth.DefaultConfig.EthashCacheDir), eth.DefaultConfig.EthashCachesInMem, eth.DefaultConfig.EthashCachesOnDisk,
1093-
stack.ResolvePath(eth.DefaultConfig.EthashDatasetDir), eth.DefaultConfig.EthashDatasetsInMem, eth.DefaultConfig.EthashDatasetsOnDisk,
1094-
)
1095-
}
10961091
config, _, err := core.SetupGenesisBlock(chainDb, MakeGenesis(ctx))
10971092
if err != nil {
10981093
Fatalf("%v", err)
10991094
}
1095+
var engine consensus.Engine
1096+
if config.Clique != nil {
1097+
engine = clique.New(config.Clique, chainDb)
1098+
} else {
1099+
engine = ethash.NewFaker()
1100+
if !ctx.GlobalBool(FakePoWFlag.Name) {
1101+
engine = ethash.New(
1102+
stack.ResolvePath(eth.DefaultConfig.EthashCacheDir), eth.DefaultConfig.EthashCachesInMem, eth.DefaultConfig.EthashCachesOnDisk,
1103+
stack.ResolvePath(eth.DefaultConfig.EthashDatasetDir), eth.DefaultConfig.EthashDatasetsInMem, eth.DefaultConfig.EthashDatasetsOnDisk,
1104+
)
1105+
}
1106+
}
11001107
vmcfg := vm.Config{EnablePreimageRecording: ctx.GlobalBool(VMEnableDebugFlag.Name)}
11011108
chain, err = core.NewBlockChain(chainDb, config, engine, vmcfg)
11021109
if err != nil {

eth/downloader/fakepeer.go

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
// Copyright 2017 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package downloader
18+
19+
import (
20+
"math/big"
21+
22+
"github.com/ethereum/go-ethereum/common"
23+
"github.com/ethereum/go-ethereum/core"
24+
"github.com/ethereum/go-ethereum/core/types"
25+
"github.com/ethereum/go-ethereum/ethdb"
26+
)
27+
28+
// FakePeer is a mock downloader peer that operates on a local database instance
29+
// instead of being an actual live node. It's useful for testing and to implement
30+
// sync commands from an xisting local database.
31+
type FakePeer struct {
32+
id string
33+
db ethdb.Database
34+
hc *core.HeaderChain
35+
dl *Downloader
36+
}
37+
38+
// NewFakePeer creates a new mock downloader peer with the given data sources.
39+
func NewFakePeer(id string, db ethdb.Database, hc *core.HeaderChain, dl *Downloader) *FakePeer {
40+
return &FakePeer{id: id, db: db, hc: hc, dl: dl}
41+
}
42+
43+
// Head implements downloader.Peer, returning the current head hash and number
44+
// of the best known header.
45+
func (p *FakePeer) Head() (common.Hash, *big.Int) {
46+
header := p.hc.CurrentHeader()
47+
return header.Hash(), header.Number
48+
}
49+
50+
// RequestHeadersByHash implements downloader.Peer, returning a batch of headers
51+
// defined by the origin hash and the associaed query parameters.
52+
func (p *FakePeer) RequestHeadersByHash(hash common.Hash, amount int, skip int, reverse bool) error {
53+
var (
54+
headers []*types.Header
55+
unknown bool
56+
)
57+
for !unknown && len(headers) < amount {
58+
origin := p.hc.GetHeaderByHash(hash)
59+
if origin == nil {
60+
break
61+
}
62+
number := origin.Number.Uint64()
63+
headers = append(headers, origin)
64+
if reverse {
65+
for i := 0; i < int(skip)+1; i++ {
66+
if header := p.hc.GetHeader(hash, number); header != nil {
67+
hash = header.ParentHash
68+
number--
69+
} else {
70+
unknown = true
71+
break
72+
}
73+
}
74+
} else {
75+
var (
76+
current = origin.Number.Uint64()
77+
next = current + uint64(skip) + 1
78+
)
79+
if header := p.hc.GetHeaderByNumber(next); header != nil {
80+
if p.hc.GetBlockHashesFromHash(header.Hash(), uint64(skip+1))[skip] == hash {
81+
hash = header.Hash()
82+
} else {
83+
unknown = true
84+
}
85+
} else {
86+
unknown = true
87+
}
88+
}
89+
}
90+
p.dl.DeliverHeaders(p.id, headers)
91+
return nil
92+
}
93+
94+
// RequestHeadersByNumber implements downloader.Peer, returning a batch of headers
95+
// defined by the origin number and the associaed query parameters.
96+
func (p *FakePeer) RequestHeadersByNumber(number uint64, amount int, skip int, reverse bool) error {
97+
var (
98+
headers []*types.Header
99+
unknown bool
100+
)
101+
for !unknown && len(headers) < amount {
102+
origin := p.hc.GetHeaderByNumber(number)
103+
if origin == nil {
104+
break
105+
}
106+
if reverse {
107+
if number >= uint64(skip+1) {
108+
number -= uint64(skip + 1)
109+
} else {
110+
unknown = true
111+
}
112+
} else {
113+
number += uint64(skip + 1)
114+
}
115+
headers = append(headers, origin)
116+
}
117+
p.dl.DeliverHeaders(p.id, headers)
118+
return nil
119+
}
120+
121+
// RequestBodies implements downloader.Peer, returning a batch of block bodies
122+
// corresponding to the specified block hashes.
123+
func (p *FakePeer) RequestBodies(hashes []common.Hash) error {
124+
var (
125+
txs [][]*types.Transaction
126+
uncles [][]*types.Header
127+
)
128+
for _, hash := range hashes {
129+
block := core.GetBlock(p.db, hash, p.hc.GetBlockNumber(hash))
130+
131+
txs = append(txs, block.Transactions())
132+
uncles = append(uncles, block.Uncles())
133+
}
134+
p.dl.DeliverBodies(p.id, txs, uncles)
135+
return nil
136+
}
137+
138+
// RequestReceipts implements downloader.Peer, returning a batch of transaction
139+
// receipts corresponding to the specified block hashes.
140+
func (p *FakePeer) RequestReceipts(hashes []common.Hash) error {
141+
var receipts [][]*types.Receipt
142+
for _, hash := range hashes {
143+
receipts = append(receipts, core.GetBlockReceipts(p.db, hash, p.hc.GetBlockNumber(hash)))
144+
}
145+
p.dl.DeliverReceipts(p.id, receipts)
146+
return nil
147+
}
148+
149+
// RequestNodeData implements downloader.Peer, returning a batch of state trie
150+
// nodes corresponding to the specified trie hashes.
151+
func (p *FakePeer) RequestNodeData(hashes []common.Hash) error {
152+
var data [][]byte
153+
for _, hash := range hashes {
154+
if entry, err := p.db.Get(hash.Bytes()); err == nil {
155+
data = append(data, entry)
156+
}
157+
}
158+
p.dl.DeliverNodeData(p.id, data)
159+
return nil
160+
}

0 commit comments

Comments
 (0)