cfx sync

Author: Kang

cmd/startCfxSync.go

@@ -0,0 +1,51 @@
+package cmd
+
+import (
+	"context"
+	"sync"
+
+	"github.com/Conflux-Chain/data-house/common"
+	"github.com/Conflux-Chain/data-house/model"
+	"github.com/Conflux-Chain/data-house/sync/cfx"
+	"github.com/Conflux-Chain/go-conflux-util/cmd"
+	"github.com/Conflux-Chain/go-conflux-util/store"
+	"github.com/Conflux-Chain/go-conflux-util/viper"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+// startCfxSyncCmd represents the startCfxSync command
+var startCfxSyncCmd = &cobra.Command{
+	Use:   "startCfxSync",
+	Short: "start Cfx sync",
+	Run:   startCfx,
+}
+
+func init() {
+	rootCmd.AddCommand(startCfxSyncCmd)
+}
+
+func startCfx(*cobra.Command, []string) {
+	ctx, cancel := context.WithCancel(context.Background())
+	var wg sync.WaitGroup
+
+	storeConfig := store.MustNewConfigFromViper()
+	db := storeConfig.MustOpenOrCreate(model.CfxTables...)
+
+	var config common.CfxConfig
+	viper.MustUnmarshalKey("cfx", &config)
+
+	logrus.WithField("batch", config.Batch).Info("Starting cfx sync ...")
+	if config.Batch {
+		if err := cfx.StartSyncBatch(ctx, &wg, &config, db); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		if err := cfx.StartSync(ctx, &wg, &config, db); err != nil {
+			logrus.WithError(err).Fatal("failed to start cfx sync")
+		}
+	}
+	logrus.Info("cfx sync started")
+
+	cmd.GracefulShutdown(&wg, cancel)
+}

cmd/startEvmSync.go

@@ -26,8 +26,6 @@ func init() {
 }
 
 func start(*cobra.Command, []string) {
-	logrus.Info("Starting evm sync ...")
-
 	ctx, cancel := context.WithCancel(context.Background())
 	var wg sync.WaitGroup
 
@@ -37,8 +35,15 @@ func start(*cobra.Command, []string) {
 	var config common.EvmConfig
 	viper.MustUnmarshalKey("evm", &config)
 
-	if err := evm.StartSync(ctx, &wg, &config, db); err != nil {
-		logrus.WithError(err).Fatal("failed to start evm sync")
+	logrus.WithField("batch", config.Batch).Info("Starting evm sync ...")
+	if config.Batch {
+		if err := evm.StartSyncBatch(ctx, &wg, &config, db); err != nil {
+			logrus.Fatal(err)
+		}
+	} else {
+		if err := evm.StartSync(ctx, &wg, &config, db); err != nil {
+			logrus.WithError(err).Fatal("failed to start evm sync")
+		}
 	}
 	logrus.Info("Evm sync started")
 

common/config.go

@@ -1,8 +1,26 @@
 package common
 
-import "github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm"
+import (
+	"github.com/Conflux-Chain/go-conflux-util/blockchain/sync"
+	"github.com/Conflux-Chain/go-conflux-util/blockchain/sync/core"
+	"github.com/Conflux-Chain/go-conflux-util/blockchain/sync/evm"
+)
 
-type EvmConfig struct {
+//blockchain/sync/sync_db.go
+
+type Config[T evm.BlockData | core.EpochData] struct {
+	sync.CatchupParamsDB[T]
+	sync.ParamsDB[T]
 	FirstBlock uint64
-	evm.Config
+	Batch      bool
+}
+
+type EvmConfig struct {
+	Config[evm.BlockData]
+	evm.AdapterConfig
+}
+
+type CfxConfig struct {
+	Config[core.EpochData]
+	core.AdapterConfig
 }

common/recover.go

@@ -0,0 +1,15 @@
+package common
+
+import (
+	"github.com/sirupsen/logrus"
+	"os"
+	"runtime/debug"
+)
+
+func Recover() {
+	if r := recover(); r != nil {
+		logrus.Error("recover panic, ", r)
+		logrus.Info(string(debug.Stack()))
+		os.Exit(-1)
+	}
+}

common/stack.go

@@ -0,0 +1,39 @@
+package common
+
+type Stack[T any] struct {
+	items []T
+}
+
+func NewStack[T any]() *Stack[T] {
+	return &Stack[T]{items: make([]T, 0)}
+}
+
+func (s *Stack[T]) Push(item T) {
+	s.items = append(s.items, item)
+}
+
+func (s *Stack[T]) Pop() (T, bool) {
+	if len(s.items) == 0 {
+		var zero T
+		return zero, false
+	}
+	item := s.items[len(s.items)-1]
+	s.items = s.items[:len(s.items)-1]
+	return item, true
+}
+
+func (s *Stack[T]) Peek() (T, bool) {
+	if len(s.items) == 0 {
+		var zero T
+		return zero, false
+	}
+	return s.items[len(s.items)-1], true
+}
+
+func (s *Stack[T]) IsEmpty() bool {
+	return len(s.items) == 0
+}
+
+func (s *Stack[T]) Size() int {
+	return len(s.items)
+}

go.mod

@@ -3,8 +3,10 @@ module github.com/Conflux-Chain/data-house
 go 1.23.10
 
 require (
-	github.com/Conflux-Chain/go-conflux-util v0.6.9-0.20251226095241-429733610f4c
+	github.com/Conflux-Chain/go-conflux-sdk v1.5.11
+	github.com/Conflux-Chain/go-conflux-util v0.6.9-0.20260108031542-576cfb1db8a1
 	github.com/openweb3/web3go v0.3.0
+	github.com/pkg/errors v0.9.1
 	github.com/shopspring/decimal v1.3.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.10.2
@@ -19,33 +21,33 @@ require (
 	github.com/StackExchange/wmi v1.2.1 // indirect
 	github.com/VictoriaMetrics/fastcache v1.12.2 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
-	github.com/bits-and-blooms/bitset v1.20.0 // indirect
+	github.com/bits-and-blooms/bitset v1.22.0 // indirect
 	github.com/btcsuite/btcd v0.24.0 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
 	github.com/btcsuite/btcd/btcutil v1.1.5 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
-	github.com/consensys/bavard v0.1.27 // indirect
-	github.com/consensys/gnark-crypto v0.16.0 // indirect
+	github.com/consensys/gnark-crypto v0.18.0 // indirect
 	github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect
 	github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/deckarep/golang-set v1.8.0 // indirect
 	github.com/deckarep/golang-set/v2 v2.6.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
-	github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+	github.com/ethereum/c-kzg-4844/v2 v2.1.1 // indirect
 	github.com/ethereum/go-ethereum v1.15.11 // indirect
 	github.com/ethereum/go-verkle v0.2.2 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
 	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/go-telegram/bot v1.2.2 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
 	github.com/gofrs/flock v0.8.1 // indirect
 	github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-cmp v0.7.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
-	github.com/google/uuid v1.5.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
@@ -56,6 +58,7 @@ require (
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/kilic/bls12-381 v0.1.0 // indirect
 	github.com/klauspost/compress v1.16.0 // indirect
 	github.com/magiconair/properties v1.8.5 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
@@ -64,7 +67,6 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/mcuadros/go-defaults v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
-	github.com/mmcloughlin/addchain v0.4.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/openweb3/go-ethereum-hdwallet v0.1.0 // indirect
 	github.com/openweb3/go-rpc-provider v0.3.5 // indirect
@@ -75,7 +77,7 @@ require (
 	github.com/pion/stun/v2 v2.0.0 // indirect
 	github.com/pion/transport/v2 v2.2.1 // indirect
 	github.com/pion/transport/v3 v3.0.1 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
@@ -84,24 +86,25 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.9 // indirect
 	github.com/spf13/viper v1.10.0 // indirect
+	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	github.com/supranational/blst v0.3.14 // indirect
+	github.com/supranational/blst v0.3.15 // indirect
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tklauser/go-sysconf v0.3.12 // indirect
 	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/tyler-smith/go-bip39 v1.1.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.40.0 // indirect
-	golang.org/x/crypto v0.35.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/sync v0.15.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
+	golang.org/x/text v0.26.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.6.0 // indirect
 	gorm.io/driver/sqlite v1.6.0 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
-	rsc.io/tmplfunc v0.0.3 // indirect
 )