From 7f6a1dc9d048e79110c9117c356868c56e44d0bd Mon Sep 17 00:00:00 2001 From: iuwqyir Date: Fri, 16 May 2025 10:27:12 +0300 Subject: [PATCH] chain validation and fix command --- cmd/root.go | 2 + cmd/validate.go | 76 +++++++ cmd/validate_and_fix.go | 151 +++++++++++++ go.mod | 27 ++- go.sum | 118 ++++------ internal/common/abi.go | 2 + internal/common/trace.go | 52 ++--- internal/orchestrator/validator.go | 164 ++++++++++++++ internal/rpc/rpc.go | 6 +- internal/rpc/serializer.go | 12 +- internal/storage/clickhouse.go | 176 ++++++++++++++- internal/storage/connector.go | 9 + internal/validation/cursor.go | 72 ++++++ internal/validation/duplicates.go | 246 ++++++++++++++++++++ internal/validation/root_calculator.go | 297 +++++++++++++++++++++++++ test/mocks/MockIMainStorage.go | 120 ++++++++++ 16 files changed, 1409 insertions(+), 121 deletions(-) create mode 100644 cmd/validate.go create mode 100644 cmd/validate_and_fix.go create mode 100644 internal/orchestrator/validator.go create mode 100644 internal/validation/cursor.go create mode 100644 internal/validation/duplicates.go create mode 100644 internal/validation/root_calculator.go diff --git a/cmd/root.go b/cmd/root.go index c398746..d9627ee 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -220,6 +220,8 @@ func init() { viper.BindPFlag("workMode.liveModeThreshold", rootCmd.PersistentFlags().Lookup("workMode-liveModeThreshold")) rootCmd.AddCommand(orchestratorCmd) rootCmd.AddCommand(apiCmd) + rootCmd.AddCommand(validateAndFixCmd) + rootCmd.AddCommand(validateCmd) } func initConfig() { diff --git a/cmd/validate.go b/cmd/validate.go new file mode 100644 index 0000000..f5ee173 --- /dev/null +++ b/cmd/validate.go @@ -0,0 +1,76 @@ +package cmd + +import ( + "math/big" + + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + config "github.com/thirdweb-dev/indexer/configs" + "github.com/thirdweb-dev/indexer/internal/orchestrator" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" +) + +var ( + validateCmd = &cobra.Command{ + Use: "validate", + Short: "Validate blockchain data integrity", + Long: "Validate a range of blocks for data integrity issues including transaction roots and logs bloom verification", + Run: func(cmd *cobra.Command, args []string) { + RunValidate(cmd, args) + }, + } +) + +/** + * Validates a range of blocks (end and start are inclusive) for a given chain + * First argument is the start block number + * Second argument (optional) is the end block number + */ +func RunValidate(cmd *cobra.Command, args []string) { + if len(args) < 1 { + log.Fatal().Msg("Start block number is required") + } + startBlock, success := new(big.Int).SetString(args[0], 10) + if !success { + log.Fatal().Msg("Failed to parse start block number") + } + + var endBlock *big.Int + if len(args) > 1 { + endBlock, success = new(big.Int).SetString(args[1], 10) + if !success { + log.Fatal().Msg("Failed to parse end block number") + } + } + if endBlock == nil { + endBlock = startBlock + } + + rpcClient, err := rpc.Initialize() + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize RPC") + } + log.Info().Msgf("Running validation for chain %d", rpcClient.GetChainID()) + + s, err := storage.NewStorageConnector(&config.Cfg.Storage) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize storage") + } + + validator := orchestrator.NewValidator(rpcClient, s) + + _, invalidBlocks, err := validator.ValidateBlockRange(startBlock, endBlock) + if err != nil { + log.Fatal().Err(err).Msg("Failed to validate blocks") + } + + if len(invalidBlocks) > 0 { + log.Info().Msgf("Found %d invalid blocks", len(invalidBlocks)) + for _, block := range invalidBlocks { + log.Info().Msgf("Invalid block: %s", block.Block.Number) + } + } else { + log.Info().Msg("No invalid blocks found") + } +} diff --git a/cmd/validate_and_fix.go b/cmd/validate_and_fix.go new file mode 100644 index 0000000..99ed67d --- /dev/null +++ b/cmd/validate_and_fix.go @@ -0,0 +1,151 @@ +package cmd + +import ( + "crypto/tls" + "fmt" + "math/big" + "strconv" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/rs/zerolog/log" + "github.com/spf13/cobra" + config "github.com/thirdweb-dev/indexer/configs" + "github.com/thirdweb-dev/indexer/internal/orchestrator" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" + "github.com/thirdweb-dev/indexer/internal/validation" +) + +var ( + validateAndFixCmd = &cobra.Command{ + Use: "validateAndFix", + Short: "Validate and fix blockchain data", + Long: "Validate blockchain data in batches and automatically fix any issues found including duplicates, gaps, and invalid blocks", + Run: func(cmd *cobra.Command, args []string) { + RunValidateAndFix(cmd, args) + }, + } +) + +func RunValidateAndFix(cmd *cobra.Command, args []string) { + batchSize := big.NewInt(1000) + fixBatchSize := 0 // default is no batch size + if len(args) > 0 { + batchSizeFromArgs, err := strconv.Atoi(args[0]) + if err != nil { + log.Fatal().Err(err).Msg("Failed to parse batch size") + } + if batchSizeFromArgs < 1 { + batchSizeFromArgs = 1 + } + batchSize = big.NewInt(int64(batchSizeFromArgs)) + log.Info().Msgf("Using batch size %d from args", batchSize) + } + if len(args) > 1 { + fixBatchSizeFromArgs, err := strconv.Atoi(args[1]) + if err != nil { + log.Fatal().Err(err).Msg("Failed to parse fix batch size") + } + fixBatchSize = fixBatchSizeFromArgs + } + log.Debug().Msgf("Batch size: %d, fix batch size: %d", batchSize, fixBatchSize) + batchSize = new(big.Int).Sub(batchSize, big.NewInt(1)) // -1 because range ends are inclusive + + rpcClient, err := rpc.Initialize() + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize RPC") + } + log.Info().Msgf("Running validationAndFix for chain %d", rpcClient.GetChainID()) + + s, err := storage.NewStorageConnector(&config.Cfg.Storage) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize storage") + } + cursor, err := validation.InitCursor(rpcClient.GetChainID(), s) + if err != nil { + log.Fatal().Err(err).Msg("Failed to initialize cursor") + } + log.Debug().Msgf("Cursor initialized for chain %d, starting from block %d", rpcClient.GetChainID(), cursor.LastScannedBlockNumber) + + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{fmt.Sprintf("%s:%d", config.Cfg.Storage.Main.Clickhouse.Host, config.Cfg.Storage.Main.Clickhouse.Port)}, + Protocol: clickhouse.Native, + TLS: &tls.Config{ + MinVersion: tls.VersionTLS12, + }, + Auth: clickhouse.Auth{ + Username: config.Cfg.Storage.Main.Clickhouse.Username, + Password: config.Cfg.Storage.Main.Clickhouse.Password, + }, + Settings: func() clickhouse.Settings { + settings := clickhouse.Settings{ + "do_not_merge_across_partitions_select_final": "1", + "use_skip_indexes_if_final": "1", + "optimize_move_to_prewhere_if_final": "1", + "async_insert": "1", + "wait_for_async_insert": "1", + } + return settings + }(), + }) + if err != nil { + log.Fatal().Err(err).Msg("Failed to connect to ClickHouse") + } + defer conn.Close() + + startBlock := new(big.Int).Add(cursor.LastScannedBlockNumber, big.NewInt(1)) + + for startBlock.Cmp(cursor.MaxBlockNumber) <= 0 { + batchEndBlock := new(big.Int).Add(startBlock, batchSize) + if batchEndBlock.Cmp(cursor.MaxBlockNumber) > 0 { + batchEndBlock = new(big.Int).Set(cursor.MaxBlockNumber) + } + + log.Info().Msgf("Validating batch of blocks from %s to %s", startBlock.String(), batchEndBlock.String()) + err := validateAndFixRange(rpcClient, s, conn, startBlock, batchEndBlock, fixBatchSize) + if err != nil { + log.Fatal().Err(err).Msgf("failed to validate and fix range %v-%v", startBlock, batchEndBlock) + } + + startBlock = new(big.Int).Add(batchEndBlock, big.NewInt(1)) + cursor.Update(batchEndBlock) + } +} + +/** + * Validates a range of blocks (end and start are inclusive) for a given chain and fixes any problems it finds + */ +func validateAndFixRange(rpcClient rpc.IRPCClient, s storage.IStorage, conn clickhouse.Conn, startBlock *big.Int, endBlock *big.Int, fixBatchSize int) error { + validator := orchestrator.NewValidator(rpcClient, s) + + chainId := rpcClient.GetChainID() + err := validation.FindAndRemoveDuplicates(conn, chainId, startBlock, endBlock) + if err != nil { + return fmt.Errorf("failed to find and fix duplicates: %w", err) + } + + err = validator.FindAndFixGaps(startBlock, endBlock) + if err != nil { + return fmt.Errorf("failed to find and fix gaps: %w", err) + } + + _, invalidBlocks, err := validator.ValidateBlockRange(startBlock, endBlock) + if err != nil { + return fmt.Errorf("failed to validate and fix blocks: %w", err) + } + + invalidBlockNumbers := make([]*big.Int, 0) + for _, blockData := range invalidBlocks { + invalidBlockNumbers = append(invalidBlockNumbers, blockData.Block.Number) + } + + if len(invalidBlocks) > 0 { + err = validator.FixBlocks(invalidBlockNumbers, fixBatchSize) + if err != nil { + return fmt.Errorf("failed to fix blocks: %w", err) + } + } + + log.Debug().Msgf("ValidationAndFix complete for range %v-%v", startBlock, endBlock) + return nil +} diff --git a/go.mod b/go.mod index 58d9878..c28edea 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.23.0 require ( github.com/ClickHouse/clickhouse-go/v2 v2.36.0 - github.com/ethereum/go-ethereum v1.14.8 + github.com/ethereum/go-ethereum v1.15.11 github.com/gin-gonic/gin v1.10.0 github.com/gorilla/schema v1.4.1 + github.com/holiman/uint256 v1.3.2 github.com/prometheus/client_golang v1.20.4 github.com/rs/zerolog v1.33.0 github.com/spf13/cobra v1.8.1 @@ -24,21 +25,21 @@ require ( github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.10.0 // indirect - github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect + github.com/bits-and-blooms/bitset v1.20.0 // indirect github.com/bytedance/sonic v1.12.6 // indirect github.com/bytedance/sonic/loader v0.2.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect - github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect - github.com/crate-crypto/go-kzg-4844 v1.0.0 // indirect + github.com/consensys/bavard v0.1.27 // indirect + github.com/consensys/gnark-crypto v0.16.0 // indirect + github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/deckarep/golang-set/v2 v2.6.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect - github.com/ethereum/c-kzg-4844 v1.0.0 // indirect + github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect + github.com/ethereum/go-verkle v0.2.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect github.com/gin-contrib/sse v0.1.0 // indirect @@ -53,11 +54,11 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.23.0 // indirect github.com/goccy/go-json v0.10.4 // indirect - github.com/golang-jwt/jwt/v4 v4.5.1 // indirect + github.com/gofrs/flock v0.8.1 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/holiman/uint256 v1.3.1 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -68,11 +69,13 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect @@ -81,6 +84,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rivo/uniseg v0.2.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/asm v1.2.0 // indirect @@ -92,13 +96,12 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/supranational/blst v0.3.11 // indirect + github.com/supranational/blst v0.3.14 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twmb/franz-go/pkg/kmsg v1.9.0 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - github.com/urfave/cli/v2 v2.27.4 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/otel v1.36.0 // indirect go.opentelemetry.io/otel/trace v1.36.0 // indirect diff --git a/go.sum b/go.sum index 78832ef..4f81063 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,5 @@ -github.com/ClickHouse/ch-go v0.63.1 h1:s2JyZvWLTCSAGdtjMBBmAgQQHMco6pawLJMOXi0FODM= -github.com/ClickHouse/ch-go v0.63.1/go.mod h1:I1kJJCL3WJcBMGe1m+HVK0+nREaG+JOYYBWjrDrF3R0= github.com/ClickHouse/ch-go v0.66.0 h1:hLslxxAVb2PHpbHr4n0d6aP8CEIpUYGMVT1Yj/Q5Img= github.com/ClickHouse/ch-go v0.66.0/go.mod h1:noiHWyLMJAZ5wYuq3R/K0TcRhrNA8h7o1AqHX0klEhM= -github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0= -github.com/ClickHouse/clickhouse-go/v2 v2.30.1 h1:Dy0n0l+cMbPXs8hFkeeWGaPKrB+MDByUNQBSmRO3W6k= -github.com/ClickHouse/clickhouse-go/v2 v2.30.1/go.mod h1:szk8BMoQV/NgHXZ20ZbwDyvPWmpfhRKjFkc6wzASGxM= github.com/ClickHouse/clickhouse-go/v2 v2.36.0 h1:FJ03h8VdmBUhvR9nQEu5jRLdfG0c/HSxUjiNdOxRQww= github.com/ClickHouse/clickhouse-go/v2 v2.36.0/go.mod h1:aijX64fKD1hAWu/zqWEmiGk7wRE8ZnpN0M3UvjsZG3I= github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= @@ -19,12 +14,8 @@ github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7X github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= -github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= -github.com/btcsuite/btcd/btcec/v2 v2.3.4 h1:3EJjcN70HCu/mwqlUsGK8GcNVyLVxFDlWurTXGPFfiQ= -github.com/btcsuite/btcd/btcec/v2 v2.3.4/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= -github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= +github.com/bits-and-blooms/bitset v1.20.0 h1:2F+rfL86jE2d/bmw7OhqUg2Sj/1rURkBn3MdfoPyRVU= +github.com/bits-and-blooms/bitset v1.20.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -42,24 +33,26 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= -github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= -github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= -github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= -github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/bavard v0.1.27 h1:j6hKUrGAy/H+gpNrpLU3I26n1yc+VMGmd6ID5+gAhOs= +github.com/consensys/bavard v0.1.27/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs= +github.com/consensys/gnark-crypto v0.16.0 h1:8Dl4eYmUWK9WmlP1Bj6je688gBRJCJbT8Mw4KoTAawo= +github.com/consensys/gnark-crypto v0.16.0/go.mod h1:Ke3j06ndtPTVvo++PhGNgvm+lgpLvzbcE2MqljY7diU= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c h1:uQYC5Z1mdLRPrZhHjHxufI8+2UG/i25QG92j0Er9p6I= -github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= -github.com/crate-crypto/go-kzg-4844 v1.0.0 h1:TsSgHwrkTKecKJ4kadtHi4b3xHW5dCFUDFnUp1TsawI= -github.com/crate-crypto/go-kzg-4844 v1.0.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= +github.com/crate-crypto/go-eth-kzg v1.3.0 h1:05GrhASN9kDAidaFJOda6A4BEvgvuXbazXg/0E3OOdI= +github.com/crate-crypto/go-eth-kzg v1.3.0/go.mod h1:J9/u5sWfznSObptgfa92Jq8rTswn6ahQWEuiLHOjCUI= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a h1:W8mUrRp6NOVl3J+MYp5kPMoUZPp7aOYHtaua31lwRHg= +github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a/go.mod h1:sTwzHBvIzm2RfVCGNEBZgRyjwK40bVoun3ZnGOCafNM= +github.com/crate-crypto/go-kzg-4844 v1.1.0 h1:EN/u9k2TF6OWSHrCCDBBU6GLNMq88OspHHlMnHfoyU4= +github.com/crate-crypto/go-kzg-4844 v1.1.0/go.mod h1:JolLjpSff1tCCJKaJx4psrlEdlXuJEC996PL3tTAFks= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -70,20 +63,18 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/ethereum/c-kzg-4844 v1.0.0 h1:0X1LBXxaEtYD9xsyj9B9ctQEZIpnvVDeoBx8aHEwTNA= -github.com/ethereum/c-kzg-4844 v1.0.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= -github.com/ethereum/go-ethereum v1.14.8 h1:NgOWvXS+lauK+zFukEvi85UmmsS/OkV0N23UZ1VTIig= -github.com/ethereum/go-ethereum v1.14.8/go.mod h1:TJhyuDq0JDppAkFXgqjwpdlQApywnu/m10kFPxh8vvs= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0 h1:KrE8I4reeVvf7C1tm8elRjj4BdscTYzz/WAbYyf/JI4= -github.com/ethereum/go-verkle v0.1.1-0.20240306133620-7d920df305f0/go.mod h1:D9AJLVXSyZQXJQVk8oh1EwjISE+sJTn2duYIZC0dy3w= +github.com/ethereum/c-kzg-4844/v2 v2.1.0 h1:gQropX9YFBhl3g4HYhwE70zq3IHFRgbbNPw0Shwzf5w= +github.com/ethereum/c-kzg-4844/v2 v2.1.0/go.mod h1:TC48kOKjJKPbN7C++qIgt0TJzZ70QznYR7Ob+WXl57E= +github.com/ethereum/go-ethereum v1.15.11 h1:JK73WKeu0WC0O1eyX+mdQAVHUV+UR1a9VB/domDngBU= +github.com/ethereum/go-ethereum v1.15.11/go.mod h1:mf8YiHIb0GR4x4TipcvBUPxJLw1mFdmxzoDi11sDRoI= +github.com/ethereum/go-verkle v0.2.2 h1:I2W0WjnrFUIzzVPwm8ykY+7pL2d4VhlsePn4j7cnFk8= +github.com/ethereum/go-verkle v0.2.2/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= -github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= @@ -130,9 +121,8 @@ github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXi github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -151,8 +141,8 @@ github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 h1:X4egAf/gcS1zATw6w github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.3.1 h1:JfTzmih28bittyHM8z360dCjIA9dbPIBlcTI6lmctQs= -github.com/holiman/uint256 v1.3.1/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= @@ -166,8 +156,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= -github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -183,12 +171,10 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= +github.com/leanovate/gopter v0.2.11 h1:vRjThO1EKPb/1NsDXuDrzldR28RLkBflWYcU9CvzWu4= +github.com/leanovate/gopter v0.2.11/go.mod h1:aK3tzZP/C+p1m3SPRE4SYZFGP7jjkuSI4f7Xvpt0S9c= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -199,6 +185,7 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/ github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -225,6 +212,16 @@ github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNH github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8= +github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s= +github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY= +github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms= +github.com/pion/stun/v2 v2.0.0 h1:A5+wXKLAypxQri59+tmQKVs7+l6mMM+3d+eER9ifRU0= +github.com/pion/stun/v2 v2.0.0/go.mod h1:22qRSh08fSEttYUmJZGlriq9+03jtVmXNODgLccj8GQ= +github.com/pion/transport/v2 v2.2.1 h1:7qYnCBlpgSJNYMbLCKuSY9KbQdBFoETvPNETv0y4N7c= +github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g= +github.com/pion/transport/v3 v3.0.1 h1:gDTlPJwROfSfz6QfSi0ZmeCSkFcnWWiiR9ES0ouANiM= +github.com/pion/transport/v3 v3.0.1/go.mod h1:UY7kiITrlMv7/IKgd5eTUcaahZx5oUN3l9SzK5f5xE0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -240,8 +237,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= @@ -271,8 +268,6 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.0 h1:pN6W1ub/G4OfnM+NR9p7xP9R6TltLUzp5JG9yZD3Qg0= github.com/spf13/viper v1.18.0/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= -github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= -github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -288,8 +283,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= -github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/supranational/blst v0.3.14 h1:xNMoHRJOTwMn63ip6qoWJ2Ymgvj7E2b9jY2FAwY+qRo= +github.com/supranational/blst v0.3.14/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= @@ -309,12 +304,10 @@ github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc= github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M= github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M= github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg= -github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= -github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/urfave/cli/v2 v2.27.4 h1:o1owoI+02Eb+K107p27wEX9Bb8eqIoZCfLXloLUSWJ8= -github.com/urfave/cli/v2 v2.27.4/go.mod h1:m4QzxcD2qpra4z7WhzEGn74WZLViBnMpb1ToCAKdGRQ= +github.com/urfave/cli/v2 v2.27.5 h1:WoHEJLdsXr6dDWoJgMq/CboDmyY/8HMMH1fTECbih+w= +github.com/urfave/cli/v2 v2.27.5/go.mod h1:3Sevf16NykTbInEnD0yKkjDAeZDS0A6bzhBH5hrMvTQ= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= @@ -326,16 +319,11 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= -go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= -go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -347,8 +335,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 h1:aAcj0Da7eBAtrTp03QXWvm88pSyOt+UgdZw2BFZ+lEw= @@ -356,8 +342,8 @@ golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8/go.mod h1:CQ1k9gNrJ50XIzaKCR golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= +golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -366,8 +352,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -375,8 +359,6 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ= golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -395,8 +377,6 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -407,19 +387,15 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= +golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/common/abi.go b/internal/common/abi.go index aca0df7..f8b9e5f 100644 --- a/internal/common/abi.go +++ b/internal/common/abi.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/rs/zerolog/log" config "github.com/thirdweb-dev/indexer/configs" "github.com/ethereum/go-ethereum/accounts/abi" @@ -65,6 +66,7 @@ func GetABIForContract(chainId string, contract string) (*abi.ABI, error) { abi, err := abi.JSON(resp.Body) if err != nil { + log.Warn().Err(err).Str("contract", contract).Str("chainId", chainId).Msg("Failed to parse contract ABI") return nil, fmt.Errorf("failed to load contract abi: %v", err) } return &abi, nil diff --git a/internal/common/trace.go b/internal/common/trace.go index b12b076..86791c9 100644 --- a/internal/common/trace.go +++ b/internal/common/trace.go @@ -13,14 +13,14 @@ type Trace struct { TransactionHash string `json:"transaction_hash" ch:"transaction_hash"` TransactionIndex uint64 `json:"transaction_index" ch:"transaction_index"` Subtraces int64 `json:"subtraces" ch:"subtraces"` - TraceAddress []uint64 `json:"trace_address" ch:"trace_address"` + TraceAddress []int64 `json:"trace_address" ch:"trace_address"` TraceType string `json:"trace_type" ch:"type"` CallType string `json:"call_type" ch:"call_type"` Error string `json:"error" ch:"error"` FromAddress string `json:"from_address" ch:"from_address"` ToAddress string `json:"to_address" ch:"to_address"` - Gas *big.Int `json:"gas" ch:"gas"` - GasUsed *big.Int `json:"gas_used" ch:"gas_used"` + Gas uint64 `json:"gas" ch:"gas"` + GasUsed uint64 `json:"gas_used" ch:"gas_used"` Input string `json:"input" ch:"input"` Output string `json:"output" ch:"output"` Value *big.Int `json:"value" ch:"value"` @@ -34,27 +34,27 @@ type Trace struct { type RawTraces = []map[string]interface{} type TraceModel struct { - ChainId string `json:"chain_id"` - BlockNumber uint64 `json:"block_number"` - BlockHash string `json:"block_hash"` - BlockTimestamp uint64 `json:"block_timestamp"` - TransactionHash string `json:"transaction_hash"` - TransactionIndex uint64 `json:"transaction_index"` - Subtraces int64 `json:"subtraces"` - TraceAddress []uint64 `json:"trace_address"` - TraceType string `json:"trace_type"` - CallType string `json:"call_type"` - Error string `json:"error"` - FromAddress string `json:"from_address"` - ToAddress string `json:"to_address"` - Gas uint64 `json:"gas"` - GasUsed uint64 `json:"gas_used"` - Input string `json:"input"` - Output string `json:"output"` - Value uint64 `json:"value"` - Author string `json:"author"` - RewardType string `json:"reward_type"` - RefundAddress string `json:"refund_address"` + ChainId string `json:"chain_id"` + BlockNumber uint64 `json:"block_number"` + BlockHash string `json:"block_hash"` + BlockTimestamp uint64 `json:"block_timestamp"` + TransactionHash string `json:"transaction_hash"` + TransactionIndex uint64 `json:"transaction_index"` + Subtraces int64 `json:"subtraces"` + TraceAddress []int64 `json:"trace_address"` + TraceType string `json:"trace_type"` + CallType string `json:"call_type"` + Error string `json:"error"` + FromAddress string `json:"from_address"` + ToAddress string `json:"to_address"` + Gas uint64 `json:"gas"` + GasUsed uint64 `json:"gas_used"` + Input string `json:"input"` + Output string `json:"output"` + Value uint64 `json:"value"` + Author string `json:"author"` + RewardType string `json:"reward_type"` + RefundAddress string `json:"refund_address"` } func (t *Trace) Serialize() TraceModel { @@ -71,8 +71,8 @@ func (t *Trace) Serialize() TraceModel { Error: t.Error, FromAddress: t.FromAddress, ToAddress: t.ToAddress, - Gas: t.Gas.Uint64(), - GasUsed: t.GasUsed.Uint64(), + Gas: t.Gas, + GasUsed: t.GasUsed, Input: t.Input, Output: t.Output, Value: t.Value.Uint64(), diff --git a/internal/orchestrator/validator.go b/internal/orchestrator/validator.go new file mode 100644 index 0000000..8958e90 --- /dev/null +++ b/internal/orchestrator/validator.go @@ -0,0 +1,164 @@ +package orchestrator + +import ( + "context" + "fmt" + "math/big" + + "github.com/rs/zerolog/log" + "github.com/thirdweb-dev/indexer/internal/common" + "github.com/thirdweb-dev/indexer/internal/rpc" + "github.com/thirdweb-dev/indexer/internal/storage" + "github.com/thirdweb-dev/indexer/internal/validation" +) + +type Validator struct { + storage storage.IStorage + rpc rpc.IRPCClient + poller *Poller +} + +func NewValidator(rpcClient rpc.IRPCClient, s storage.IStorage) *Validator { + return &Validator{ + rpc: rpcClient, + storage: s, + poller: NewBoundlessPoller(rpcClient, s), + } +} + +/** + * Validate blocks in the range of startBlock to endBlock + * @param startBlock - The start block number (inclusive) + * @param endBlock - The end block number (inclusive) + * @return error - An error if the validation fails + */ +func (v *Validator) ValidateBlockRange(startBlock *big.Int, endBlock *big.Int) (validBlocks []common.BlockData, invalidBlocks []common.BlockData, err error) { + dbData, err := v.storage.MainStorage.GetValidationBlockData(v.rpc.GetChainID(), startBlock, endBlock) + if err != nil { + return nil, nil, err + } + validBlocks, invalidBlocks, err = v.ValidateBlocks(dbData) + if err != nil { + return nil, nil, err + } + return validBlocks, invalidBlocks, nil +} + +func (v *Validator) ValidateBlocks(blocks []common.BlockData) (validBlocks []common.BlockData, invalidBlocks []common.BlockData, err error) { + invalidBlocks = make([]common.BlockData, 0) + validBlocks = make([]common.BlockData, 0) + for _, blockData := range blocks { + valid, err := ValidateBlock(blockData) + if err != nil { + log.Error().Err(err).Msgf("Block verification failed for block %s", blockData.Block.Number) + return nil, nil, err + } + if valid { + validBlocks = append(validBlocks, blockData) + } else { + invalidBlocks = append(invalidBlocks, blockData) + } + } + return validBlocks, invalidBlocks, nil +} + +func ValidateBlock(blockData common.BlockData) (valid bool, err error) { + if blockData.Block.TransactionCount != uint64(len(blockData.Transactions)) { + log.Error().Msgf("Block verification failed for block %s: transaction count mismatch: expected=%d, fetched from DB=%d", blockData.Block.Number, blockData.Block.TransactionCount, len(blockData.Transactions)) + return false, nil + } + + // Calculate logsBloom from logs + calculatedLogsBloom := validation.CalculateLogsBloom(blockData.Logs) + // Compare calculated logsBloom with block's logsBloom + if calculatedLogsBloom != blockData.Block.LogsBloom { + log.Error().Msgf("Block verification failed for block %s: logsBloom mismatch: calculated=%s, block=%s", blockData.Block.Number, calculatedLogsBloom, blockData.Block.LogsBloom) + return false, nil + } + + // TODO: remove this once we know how to validate all tx types + for _, tx := range blockData.Transactions { + if tx.TransactionType > 4 { // Currently supported types are 0-4 + log.Warn().Msgf("Skipping transaction root validation for block %s due to unsupported transaction type %d", blockData.Block.Number, tx.TransactionType) + return true, nil + } + } + + // Calculate transactionsRoot from transactions + calculatedTransactionsRoot, err := validation.CalculateTransactionsRoot(blockData.Transactions) + if err != nil { + return false, fmt.Errorf("failed to calculate transactionsRoot: %v", err) + } + + // Compare calculated transactionsRoot with block's transactionsRoot + if calculatedTransactionsRoot != blockData.Block.TransactionsRoot { + log.Error().Msgf("Block verification failed for block %s: transactionsRoot mismatch: calculated=%s, block=%s", blockData.Block.Number, calculatedTransactionsRoot, blockData.Block.TransactionsRoot) + return false, nil + } + + return true, nil +} + +func (v *Validator) FixBlocks(invalidBlocks []*big.Int, fixBatchSize int) error { + if len(invalidBlocks) == 0 { + log.Debug().Msg("No invalid blocks") + return nil + } + + if fixBatchSize == 0 { + fixBatchSize = len(invalidBlocks) + } + + log.Debug().Msgf("Fixing invalid blocks %d to %d", invalidBlocks[0], invalidBlocks[len(invalidBlocks)-1]) + + // Process blocks in batches + for i := 0; i < len(invalidBlocks); i += fixBatchSize { + end := i + fixBatchSize + if end > len(invalidBlocks) { + end = len(invalidBlocks) + } + batch := invalidBlocks[i:end] + + polledBlocks, failedBlocks := v.poller.PollWithoutSaving(context.Background(), batch) + log.Debug().Msgf("Batch of invalid blocks polled: %d to %d", batch[0], batch[len(batch)-1]) + if len(failedBlocks) > 0 { + log.Error().Msgf("Failed to poll %d blocks: %v", len(failedBlocks), failedBlocks) + return fmt.Errorf("failed to poll %d blocks: %v", len(failedBlocks), failedBlocks) + } + + _, err := v.storage.MainStorage.ReplaceBlockData(polledBlocks) + if err != nil { + log.Error().Err(err).Msgf("Failed to replace blocks: %v", polledBlocks) + return err + } + } + log.Info().Msgf("Fixed %d blocks", len(invalidBlocks)) + return nil +} + +func (v *Validator) FindAndFixGaps(startBlock *big.Int, endBlock *big.Int) error { + missingBlockNumbers, err := v.storage.MainStorage.FindMissingBlockNumbers(v.rpc.GetChainID(), startBlock, endBlock) + if err != nil { + return err + } + if len(missingBlockNumbers) == 0 { + log.Debug().Msg("No missing blocks found") + return nil + } + log.Debug().Msgf("Found %d missing blocks: %v", len(missingBlockNumbers), missingBlockNumbers) + + // query missing blocks + polledBlocks, failedBlocks := v.poller.PollWithoutSaving(context.Background(), missingBlockNumbers) + log.Debug().Msg("Missing blocks polled") + if len(failedBlocks) > 0 { + log.Error().Msgf("Failed to poll %d blocks: %v", len(failedBlocks), failedBlocks) + return fmt.Errorf("failed to poll %d blocks: %v", len(failedBlocks), failedBlocks) + } + + err = v.storage.MainStorage.InsertBlockData(polledBlocks) + if err != nil { + log.Error().Err(err).Msgf("Failed to insert missing blocks: %v", polledBlocks) + return err + } + return nil +} diff --git a/internal/rpc/rpc.go b/internal/rpc/rpc.go index a39c454..88ea884 100644 --- a/internal/rpc/rpc.go +++ b/internal/rpc/rpc.go @@ -107,7 +107,11 @@ func InitializeSimpleRPCWithUrl(url string) (IRPCClient, error) { EthClient: ethClient, url: url, } - rpc.setChainID(context.Background()) + + chainIdErr := rpc.setChainID(context.Background()) + if chainIdErr != nil { + return nil, chainIdErr + } return IRPCClient(rpc), nil } diff --git a/internal/rpc/serializer.go b/internal/rpc/serializer.go index 01f6346..53c2f5e 100644 --- a/internal/rpc/serializer.go +++ b/internal/rpc/serializer.go @@ -379,8 +379,8 @@ func serializeTrace(chainId *big.Int, trace map[string]interface{}, block common Error: interfaceToString(trace["error"]), FromAddress: interfaceToString(action["from"]), ToAddress: interfaceToString(action["to"]), - Gas: hexToBigInt(action["gas"]), - GasUsed: hexToBigInt(result["gasUsed"]), + Gas: hexToUint64(action["gas"]), + GasUsed: hexToUint64(result["gasUsed"]), Input: interfaceToString(action["input"]), Output: interfaceToString(result["output"]), Value: hexToBigInt(action["value"]), @@ -399,15 +399,15 @@ func hexToBigInt(hex interface{}) *big.Int { return v } -func serializeTraceAddress(traceAddress interface{}) []uint64 { +func serializeTraceAddress(traceAddress interface{}) []int64 { if traceAddressSlice, ok := traceAddress.([]interface{}); ok { - var addresses []uint64 + var addresses []int64 for _, addr := range traceAddressSlice { - addresses = append(addresses, uint64(addr.(float64))) + addresses = append(addresses, int64(addr.(float64))) } return addresses } - return []uint64{} + return []int64{} } func hexToTime(hex interface{}) time.Time { diff --git a/internal/storage/clickhouse.go b/internal/storage/clickhouse.go index ef16a10..7831043 100644 --- a/internal/storage/clickhouse.go +++ b/internal/storage/clickhouse.go @@ -380,8 +380,8 @@ func (c *ClickHouseConnector) insertTraces(traces []common.Trace, opt InsertOpti trace.Error, trace.FromAddress, trace.ToAddress, - trace.Gas.Uint64(), - trace.GasUsed.Uint64(), + trace.Gas, + trace.GasUsed, trace.Input, trace.Output, trace.Value, @@ -480,6 +480,10 @@ func (c *ClickHouseConnector) GetAggregations(table string, qf QueryFilter) (Que if qf.ChainId != nil && qf.ChainId.Sign() > 0 { whereClauses = append(whereClauses, createFilterClause("chain_id", qf.ChainId.String())) } + blockNumbersClause := createBlockNumbersClause(qf.BlockNumbers) + if blockNumbersClause != "" { + whereClauses = append(whereClauses, blockNumbersClause) + } contractAddressClause := createContractAddressClause(table, qf.ContractAddress) if contractAddressClause != "" { whereClauses = append(whereClauses, contractAddressClause) @@ -940,7 +944,7 @@ func (c *ClickHouseConnector) InsertStagingData(data []common.BlockData) error { } func (c *ClickHouseConnector) GetStagingData(qf QueryFilter) ([]common.BlockData, error) { - query := fmt.Sprintf("SELECT data FROM %s.block_data WHERE block_number IN (%s) AND is_deleted = 0", + query := fmt.Sprintf("SELECT data FROM %s.block_data FINAL WHERE block_number IN (%s) AND is_deleted = 0", c.cfg.Database, getBlockNumbersStringArray(qf.BlockNumbers)) if qf.ChainId.Sign() != 0 { @@ -1299,8 +1303,8 @@ func (c *ClickHouseConnector) InsertBlockData(data []common.BlockData) error { trace.Error, trace.FromAddress, trace.ToAddress, - trace.Gas.Uint64(), - trace.GasUsed.Uint64(), + trace.Gas, + trace.GasUsed, trace.Input, trace.Output, trace.Value, @@ -1744,3 +1748,165 @@ func (c *ClickHouseConnector) GetTokenBalances(qf BalancesQueryFilter, fields .. return queryResult, nil } + +func (c *ClickHouseConnector) GetValidationBlockData(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) (blocks []common.BlockData, err error) { + if startBlock == nil || endBlock == nil { + return nil, fmt.Errorf("start block and end block must not be nil") + } + + if startBlock.Cmp(endBlock) > 0 { + return nil, fmt.Errorf("start block must be less than or equal to end block") + } + + blockNumbers := make([]*big.Int, 0) + for i := new(big.Int).Set(startBlock); i.Cmp(endBlock) <= 0; i.Add(i, big.NewInt(1)) { + blockNumbers = append(blockNumbers, new(big.Int).Set(i)) + } + // Get blocks, logs and transactions concurrently + type blockResult struct { + blocks []common.Block + err error + } + + type logResult struct { + logMap map[string][]common.Log // blockNumber -> logs + err error + } + + type txResult struct { + txMap map[string][]common.Transaction // blockNumber -> transactions + err error + } + + blocksChan := make(chan blockResult) + logsChan := make(chan logResult) + txsChan := make(chan txResult) + + // Launch goroutines for concurrent fetching + go func() { + blocksResult, err := c.GetBlocks(QueryFilter{ + ChainId: chainId, + BlockNumbers: blockNumbers, + ForceConsistentData: true, + }, "chain_id", "block_number", "transactions_root", "receipts_root", "logs_bloom", "transaction_count") + blocksChan <- blockResult{blocks: blocksResult.Data, err: err} + }() + + go func() { + logsResult, err := c.GetLogs(QueryFilter{ + ChainId: chainId, + BlockNumbers: blockNumbers, + ForceConsistentData: true, + }, "chain_id", "block_number", "address", "log_index", "topic_0", "topic_1", "topic_2", "topic_3") + if err != nil { + logsChan <- logResult{err: err} + return + } + + // Pre-organize logs by block number + logMap := make(map[string][]common.Log) + for _, log := range logsResult.Data { + blockNum := log.BlockNumber.String() + logMap[blockNum] = append(logMap[blockNum], log) + } + logsChan <- logResult{logMap: logMap} + }() + + go func() { + transactionsResult, err := c.GetTransactions(QueryFilter{ + ChainId: chainId, + BlockNumbers: blockNumbers, + ForceConsistentData: true, + }, "chain_id", "block_number", "nonce", "transaction_index", "to_address", "value", "gas", "gas_price", "data", "max_fee_per_gas", "max_priority_fee_per_gas", "max_fee_per_blob_gas", "blob_versioned_hashes", "transaction_type", "r", "s", "v", "access_list", "authorization_list", "blob_gas_used", "blob_gas_price") + if err != nil { + txsChan <- txResult{err: err} + return + } + + // Pre-organize transactions by block number + txMap := make(map[string][]common.Transaction) + for _, tx := range transactionsResult.Data { + blockNum := tx.BlockNumber.String() + txMap[blockNum] = append(txMap[blockNum], tx) + } + txsChan <- txResult{txMap: txMap} + }() + + // Wait for all results + blocksResult := <-blocksChan + logsResult := <-logsChan + txsResult := <-txsChan + + // Check for errors + if blocksResult.err != nil { + return nil, fmt.Errorf("error fetching blocks: %v", blocksResult.err) + } + if logsResult.err != nil { + return nil, fmt.Errorf("error fetching logs: %v", logsResult.err) + } + if txsResult.err != nil { + return nil, fmt.Errorf("error fetching transactions: %v", txsResult.err) + } + + // Build BlockData slice + blockData := make([]common.BlockData, len(blocksResult.blocks)) + + // Build BlockData for each block + for i, block := range blocksResult.blocks { + blockNum := block.Number.String() + blockData[i] = common.BlockData{ + Block: block, + Logs: logsResult.logMap[blockNum], + Transactions: txsResult.txMap[blockNum], + } + } + + return blockData, nil +} + +func (c *ClickHouseConnector) FindMissingBlockNumbers(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) (blockNumbers []*big.Int, err error) { + tableName := c.getTableName(chainId, "blocks") + query := fmt.Sprintf(` + WITH sequence AS ( + SELECT + {startBlock:UInt256} + number AS expected_block_number + FROM + numbers(toUInt64({endBlock:UInt256} - {startBlock:UInt256} + 1)) + ), + existing_blocks AS ( + SELECT DISTINCT + block_number + FROM + %s FINAL + WHERE + chain_id = {chainId:UInt256} + AND block_number >= {startBlock:UInt256} + AND block_number <= {endBlock:UInt256} + ) + SELECT + s.expected_block_number AS missing_block_number + FROM + sequence s + LEFT JOIN + existing_blocks e ON s.expected_block_number = e.block_number + WHERE + e.block_number = 0 + ORDER BY + missing_block_number + `, tableName) + rows, err := c.conn.Query(context.Background(), query, clickhouse.Named("chainId", chainId.String()), clickhouse.Named("startBlock", startBlock.String()), clickhouse.Named("endBlock", endBlock.String())) + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var blockNumber *big.Int + err := rows.Scan(&blockNumber) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, blockNumber) + } + return blockNumbers, nil +} diff --git a/internal/storage/connector.go b/internal/storage/connector.go index 0172418..c5a7487 100644 --- a/internal/storage/connector.go +++ b/internal/storage/connector.go @@ -102,6 +102,15 @@ type IMainStorage interface { GetTokenBalances(qf BalancesQueryFilter, fields ...string) (QueryResult[common.TokenBalance], error) GetTokenTransfers(qf TransfersQueryFilter, fields ...string) (QueryResult[common.TokenTransfer], error) + + /** + * Gets only the data required for validation. + */ + GetValidationBlockData(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) (blocks []common.BlockData, err error) + /** + * Finds missing block numbers in a range. Block numbers should be sequential. + */ + FindMissingBlockNumbers(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) (blockNumbers []*big.Int, err error) } func NewStorageConnector(cfg *config.StorageConfig) (IStorage, error) { diff --git a/internal/validation/cursor.go b/internal/validation/cursor.go new file mode 100644 index 0000000..d954446 --- /dev/null +++ b/internal/validation/cursor.go @@ -0,0 +1,72 @@ +package validation + +import ( + "encoding/json" + "fmt" + "math/big" + "os" + + "github.com/thirdweb-dev/indexer/internal/storage" +) + +type Cursor struct { + LastScannedBlockNumber *big.Int + MaxBlockNumber *big.Int + ChainId *big.Int +} + +func InitCursor(chainId *big.Int, storage storage.IStorage) (*Cursor, error) { + lastScannedBlock := getLastScannedBlock(chainId) + maxBlockNumber, err := storage.MainStorage.GetMaxBlockNumber(chainId) + if err != nil { + return nil, err + } + if maxBlockNumber == nil { + maxBlockNumber = big.NewInt(0) + } + if lastScannedBlock.Cmp(maxBlockNumber) >= 0 { + return nil, fmt.Errorf("last scanned block number is greater than or equal to max block number") + } + return &Cursor{ + LastScannedBlockNumber: lastScannedBlock, + MaxBlockNumber: maxBlockNumber, + ChainId: chainId, + }, nil +} + +func (c *Cursor) Update(blockNumber *big.Int) error { + cursorFile := fmt.Sprintf("validation_cursor_%s.json", c.ChainId.String()) + jsonData, err := json.Marshal(blockNumber.String()) + if err != nil { + return err + } + + err = os.WriteFile(cursorFile, jsonData, 0644) + if err != nil { + return err + } + c.LastScannedBlockNumber = blockNumber + return nil +} + +func getLastScannedBlock(chainId *big.Int) *big.Int { + cursorFile := fmt.Sprintf("validation_cursor_%s.json", chainId.String()) + if _, err := os.Stat(cursorFile); err != nil { + return big.NewInt(0) + } + + fileData, err := os.ReadFile(cursorFile) + if err != nil { + return big.NewInt(0) + } + + var lastBlock string + err = json.Unmarshal(fileData, &lastBlock) + if err != nil { + return big.NewInt(0) + } + + lastBlockBig := new(big.Int) + lastBlockBig.SetString(lastBlock, 10) + return lastBlockBig +} diff --git a/internal/validation/duplicates.go b/internal/validation/duplicates.go new file mode 100644 index 0000000..9ee6164 --- /dev/null +++ b/internal/validation/duplicates.go @@ -0,0 +1,246 @@ +package validation + +import ( + "context" + "fmt" + "math/big" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/rs/zerolog/log" +) + +type DuplicateTransaction struct { + BlockNumber *big.Int `json:"block_number" ch:"block_number"` + Hash string `json:"hash" ch:"hash"` +} + +type DuplicateLog struct { + BlockNumber *big.Int `json:"block_number" ch:"block_number"` + TxHash string `json:"transaction_hash" ch:"transaction_hash"` + LogIndex uint64 `json:"log_index" ch:"log_index"` +} + +func FindAndRemoveDuplicates(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) error { + duplicateBlockNumbers, err := findDuplicateBlocksInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateBlockNumbers) == 0 { + log.Debug().Msg("No duplicate blocks found in range") + } else { + log.Debug().Msgf("Found %d duplicate blocks in range %v-%v: %v", len(duplicateBlockNumbers), startBlock, endBlock, duplicateBlockNumbers) + err = removeDuplicateBlocks(conn, chainId, duplicateBlockNumbers) + if err != nil { + return err + } + } + + duplicateTransactions, err := findDuplicateTransactionsInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateTransactions) == 0 { + log.Debug().Msg("No duplicate transactions found in range") + } else { + log.Debug().Msgf("Found %d duplicate transactions in range %v-%v: %v", len(duplicateTransactions), startBlock, endBlock, duplicateTransactions) + err = removeDuplicateTransactions(conn, chainId, duplicateTransactions) + if err != nil { + return err + } + } + + duplicateLogs, err := findDuplicateLogsInRange(conn, chainId, startBlock, endBlock) + if err != nil { + return err + } + if len(duplicateLogs) == 0 { + log.Debug().Msg("No duplicate logs found in range") + } else { + log.Debug().Msgf("Found %d duplicate logs in range %v-%v: %v", len(duplicateLogs), startBlock, endBlock, duplicateLogs) + err = removeDuplicateLogs(conn, chainId, duplicateLogs) + if err != nil { + return err + } + } + + return nil +} + +func findDuplicateBlocksInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]*big.Int, error) { + query := `SELECT block_number + FROM default.blocks FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + blockNumbers := make([]*big.Int, 0) + + for rows.Next() { + var blockNumber *big.Int + err := rows.Scan(&blockNumber) + if err != nil { + return nil, err + } + blockNumbers = append(blockNumbers, blockNumber) + } + return blockNumbers, nil +} + +func findDuplicateTransactionsInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]DuplicateTransaction, error) { + query := `SELECT block_number, hash + FROM default.transactions FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number, hash + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + duplicateTransactions := make([]DuplicateTransaction, 0) + + for rows.Next() { + var duplicateTransaction DuplicateTransaction + err := rows.ScanStruct(&duplicateTransaction) + if err != nil { + return nil, err + } + duplicateTransactions = append(duplicateTransactions, duplicateTransaction) + } + return duplicateTransactions, nil +} + +func findDuplicateLogsInRange(conn clickhouse.Conn, chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]DuplicateLog, error) { + query := `SELECT block_number, transaction_hash, log_index + FROM default.logs FINAL WHERE chain_id = ? AND block_number >= ? AND block_number <= ? + GROUP BY block_number, transaction_hash, log_index + HAVING sum(sign) != 1 + ORDER BY block_number; + ` + rows, err := conn.Query(context.Background(), query, chainId, startBlock, endBlock) + if err != nil { + return nil, err + } + defer rows.Close() + + duplicateLogs := make([]DuplicateLog, 0) + + for rows.Next() { + var duplicateLog DuplicateLog + err := rows.ScanStruct(&duplicateLog) + if err != nil { + return nil, err + } + duplicateLogs = append(duplicateLogs, duplicateLog) + } + return duplicateLogs, nil +} + +func removeDuplicateBlocks(conn clickhouse.Conn, chainId *big.Int, duplicateBlockNumbers []*big.Int) error { + query := `WITH + to_be_inserted AS ( + SELECT chain_id, block_number, block_timestamp, hash, parent_hash, sha3_uncles, nonce, mix_hash, miner, state_root, + transactions_root, receipts_root, logs_bloom, size, extra_data, difficulty, total_difficulty, transaction_count, + gas_limit, gas_used, withdrawals_root, base_fee_per_gas, insert_timestamp, -sign as sign + FROM default.blocks FINAL + WHERE chain_id = ? AND block_number IN (?) + ) + INSERT INTO blocks ( + chain_id, block_number, block_timestamp, hash, parent_hash, sha3_uncles, nonce, mix_hash, miner, state_root, + transactions_root, receipts_root, logs_bloom, size, extra_data, difficulty, total_difficulty, transaction_count, + gas_limit, gas_used, withdrawals_root, base_fee_per_gas, insert_timestamp, sign + ) SELECT * from to_be_inserted + ` + err := conn.Exec(context.Background(), query, chainId, duplicateBlockNumbers) + if err != nil { + return err + } + return nil +} + +func removeDuplicateTransactions(conn clickhouse.Conn, chainId *big.Int, duplicateTransactions []DuplicateTransaction) error { + query := `WITH + to_be_inserted AS ( + SELECT chain_id, hash, nonce, block_hash, block_number, block_timestamp, transaction_index, from_address, to_address, value, gas, gas_price, data, function_selector, + max_fee_per_gas, max_priority_fee_per_gas, max_fee_per_blob_gas, blob_versioned_hashes, transaction_type, r, s, v, access_list, authorization_list, contract_address, + gas_used, cumulative_gas_used, effective_gas_price, blob_gas_used, blob_gas_price, logs_bloom, status, insert_timestamp, -sign as sign + FROM default.transactions FINAL + WHERE chain_id = ? AND block_number IN (?) AND hash IN (?) + ) + INSERT INTO transactions ( + chain_id, hash, nonce, block_hash, block_number, block_timestamp, transaction_index, from_address, to_address, value, gas, gas_price, data, function_selector, + max_fee_per_gas, max_priority_fee_per_gas, max_fee_per_blob_gas, blob_versioned_hashes, transaction_type, r, s, v, access_list, authorization_list, contract_address, + gas_used, cumulative_gas_used, effective_gas_price, blob_gas_used, blob_gas_price, logs_bloom, status, insert_timestamp, sign + ) SELECT * from to_be_inserted + ` + + const batchSize = 1000 + for i := 0; i < len(duplicateTransactions); i += batchSize { + end := i + batchSize + if end > len(duplicateTransactions) { + end = len(duplicateTransactions) + } + + batch := duplicateTransactions[i:end] + blockNumbers := make([]*big.Int, 0, len(batch)) + hashes := make([]string, 0, len(batch)) + + for _, duplicateTransaction := range batch { + blockNumbers = append(blockNumbers, duplicateTransaction.BlockNumber) + hashes = append(hashes, duplicateTransaction.Hash) + } + + err := conn.Exec(context.Background(), query, chainId, blockNumbers, hashes) + if err != nil { + return err + } + } + return nil +} + +func removeDuplicateLogs(conn clickhouse.Conn, chainId *big.Int, duplicateLogs []DuplicateLog) error { + const batchSize = 1000 + for i := 0; i < len(duplicateLogs); i += batchSize { + end := i + batchSize + if end > len(duplicateLogs) { + end = len(duplicateLogs) + } + + batch := duplicateLogs[i:end] + blockNumbers := make([]*big.Int, 0, len(batch)) + tuples := make([]string, 0, len(batch)) + + for _, duplicateLog := range batch { + blockNumbers = append(blockNumbers, duplicateLog.BlockNumber) + tuples = append(tuples, fmt.Sprintf("('%s', %d)", duplicateLog.TxHash, duplicateLog.LogIndex)) + } + + query := fmt.Sprintf(`WITH + to_be_inserted AS ( + SELECT chain_id, block_number, block_hash, block_timestamp, transaction_hash, transaction_index, log_index, address, + data, topic_0, topic_1, topic_2, topic_3, insert_timestamp, -sign as sign + FROM default.logs FINAL + WHERE chain_id = ? AND block_number IN (?) AND (transaction_hash, log_index) IN (%s) + ) + INSERT INTO logs ( + chain_id, block_number, block_hash, block_timestamp, transaction_hash, transaction_index, log_index, address, + data, topic_0, topic_1, topic_2, topic_3, insert_timestamp, sign + ) SELECT * from to_be_inserted + `, strings.Join(tuples, ",")) + + err := conn.Exec(context.Background(), query, chainId, blockNumbers) + if err != nil { + return err + } + } + return nil +} diff --git a/internal/validation/root_calculator.go b/internal/validation/root_calculator.go new file mode 100644 index 0000000..bed9bec --- /dev/null +++ b/internal/validation/root_calculator.go @@ -0,0 +1,297 @@ +package validation + +import ( + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "sort" + "strings" + + gethCommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" + "github.com/thirdweb-dev/indexer/internal/common" +) + +func CalculateTransactionsRoot(transactions []common.Transaction) (string, error) { + // Sort transactions by transaction index + sort.Slice(transactions, func(i, j int) bool { + return transactions[i].TransactionIndex < transactions[j].TransactionIndex + }) + + // Convert our transactions to go-ethereum types.Transaction + ethTxs := make([]*types.Transaction, 0, len(transactions)) + for _, tx := range transactions { + // Convert to address + to := gethCommon.HexToAddress(tx.ToAddress) + isContractCreation := tx.ToAddress == "" + + // Convert data - ensure we're using the full input data + data, err := hex.DecodeString(strings.TrimPrefix(tx.Data, "0x")) + if err != nil { + return "", fmt.Errorf("failed to decode transaction data: %v", err) + } + + // Create transaction based on type + var ethTx *types.Transaction + switch tx.TransactionType { + case 0: // Legacy transaction + // For legacy transactions, we need to set the signature values + if isContractCreation { + ethTx = types.NewContractCreation( + tx.Nonce, + tx.Value, + tx.Gas, + tx.GasPrice, + data, + ) + } else { + ethTx = types.NewTransaction( + tx.Nonce, + to, + tx.Value, + tx.Gas, + tx.GasPrice, + data, + ) + } + + // Set the signature values + if tx.V != nil && tx.R != nil && tx.S != nil { + v := tx.V.Uint64() + if v >= 35 { + // This is an EIP-155 transaction + signer := types.NewEIP155Signer(tx.ChainId) + sig := make([]byte, 65) + tx.R.FillBytes(sig[:32]) + tx.S.FillBytes(sig[32:64]) + // Calculate recovery_id from v + recovery_id := byte(v - (tx.ChainId.Uint64()*2 + 35)) + sig[64] = recovery_id + ethTx, err = ethTx.WithSignature(signer, sig) + if err != nil { + return "", fmt.Errorf("failed to set EIP-155 signature: %v", err) + } + } else { + // This is a legacy transaction + signer := types.HomesteadSigner{} + sig := make([]byte, 65) + tx.R.FillBytes(sig[:32]) + tx.S.FillBytes(sig[32:64]) + // Calculate recovery_id from v + recovery_id := byte(v - 27) + sig[64] = recovery_id + ethTx, err = ethTx.WithSignature(signer, sig) + if err != nil { + return "", fmt.Errorf("failed to set legacy signature: %v", err) + } + } + } + case 1: // Access list transaction + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var toAddr *gethCommon.Address + if !isContractCreation { + toAddr = &to + } + + ethTx = types.NewTx(&types.AccessListTx{ + ChainID: tx.ChainId, + Nonce: tx.Nonce, + GasPrice: tx.GasPrice, + Gas: tx.Gas, + To: toAddr, + Value: tx.Value, + Data: data, + AccessList: accessList, + V: tx.V, + R: tx.R, + S: tx.S, + }) + case 2: // Dynamic fee transaction + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var toAddr *gethCommon.Address + if !isContractCreation { + toAddr = &to + } + + ethTx = types.NewTx(&types.DynamicFeeTx{ + ChainID: tx.ChainId, + Nonce: tx.Nonce, + GasTipCap: tx.MaxPriorityFeePerGas, + GasFeeCap: tx.MaxFeePerGas, + Gas: tx.Gas, + To: toAddr, + Value: tx.Value, + Data: data, + AccessList: accessList, + V: tx.V, + R: tx.R, + S: tx.S, + }) + case 3: // Blob transaction + // Convert big.Int values to uint256.Int + chainID := safeUint256(tx.ChainId) + gasTipCap := safeUint256(tx.MaxPriorityFeePerGas) + gasFeeCap := safeUint256(tx.MaxFeePerGas) + value := safeUint256(tx.Value) + blobFeeCap := safeUint256(tx.MaxFeePerBlobGas) + v := safeUint256(tx.V) + r := safeUint256(tx.R) + s := safeUint256(tx.S) + + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + blobHashes := make([]gethCommon.Hash, len(tx.BlobVersionedHashes)) + for i, hash := range tx.BlobVersionedHashes { + blobHashes[i] = gethCommon.HexToHash(hash) + } + + var toAddr gethCommon.Address + if !isContractCreation { + toAddr = to + } + + ethTx = types.NewTx(&types.BlobTx{ + ChainID: chainID, + Nonce: tx.Nonce, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Gas: tx.Gas, + To: toAddr, + Value: value, + Data: data, + AccessList: accessList, + BlobFeeCap: blobFeeCap, + BlobHashes: blobHashes, + V: v, + R: r, + S: s, + }) + case 4: // EIP-7702 + // Convert big.Int values to uint256.Int + chainID := safeUint256(tx.ChainId) + gasTipCap := safeUint256(tx.MaxPriorityFeePerGas) + gasFeeCap := safeUint256(tx.MaxFeePerGas) + value := safeUint256(tx.Value) + v := safeUint256(tx.V) + r := safeUint256(tx.R) + s := safeUint256(tx.S) + // Parse access list from JSON if available + var accessList types.AccessList + if tx.AccessListJson != nil { + err = json.Unmarshal([]byte(*tx.AccessListJson), &accessList) + if err != nil { + return "", fmt.Errorf("failed to parse access list: %v", err) + } + } + + var authList []types.SetCodeAuthorization + if tx.AuthorizationListJson != nil { + err = json.Unmarshal([]byte(*tx.AuthorizationListJson), &authList) + if err != nil { + return "", fmt.Errorf("failed to parse authorization list: %v", err) + } + } + + var toAddr gethCommon.Address + if !isContractCreation { + toAddr = to + } + + ethTx = types.NewTx(&types.SetCodeTx{ + ChainID: chainID, + Nonce: tx.Nonce, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Gas: tx.Gas, + To: toAddr, + Value: value, + Data: data, + AccessList: accessList, + AuthList: authList, + V: v, + R: r, + S: s, + }) + default: + return "", fmt.Errorf("unsupported transaction type: %d", tx.TransactionType) + } + + // Set the transaction hash + ethTxs = append(ethTxs, ethTx) + } + + // Create transactions root using go-ethereum's implementation + root := types.DeriveSha(types.Transactions(ethTxs), trie.NewStackTrie(nil)) + calculatedRoot := "0x" + hex.EncodeToString(root[:]) + return calculatedRoot, nil +} + +func CalculateLogsBloom(logs []common.Log) string { + // Convert our logs to go-ethereum types.Log + ethLogs := make([]*types.Log, len(logs)) + for i, log := range logs { + // Convert address + addr := gethCommon.HexToAddress(log.Address) + + // Convert topics + topics := make([]gethCommon.Hash, 0, 4) + if log.Topic0 != "" { + topics = append(topics, gethCommon.HexToHash(log.Topic0)) + } + if log.Topic1 != "" { + topics = append(topics, gethCommon.HexToHash(log.Topic1)) + } + if log.Topic2 != "" { + topics = append(topics, gethCommon.HexToHash(log.Topic2)) + } + if log.Topic3 != "" { + topics = append(topics, gethCommon.HexToHash(log.Topic3)) + } + + ethLogs[i] = &types.Log{ + Address: addr, + Topics: topics, + } + } + + receipt := &types.Receipt{ + Logs: ethLogs, + } + // Create bloom filter using go-ethereum's implementation + bloom := types.CreateBloom(receipt) + return "0x" + hex.EncodeToString(bloom[:]) +} + +func safeUint256(b *big.Int) *uint256.Int { + if b == nil { + return new(uint256.Int) + } + out, _ := uint256.FromBig(b) + return out +} diff --git a/test/mocks/MockIMainStorage.go b/test/mocks/MockIMainStorage.go index eaaac80..390fbfa 100644 --- a/test/mocks/MockIMainStorage.go +++ b/test/mocks/MockIMainStorage.go @@ -26,6 +26,66 @@ func (_m *MockIMainStorage) EXPECT() *MockIMainStorage_Expecter { return &MockIMainStorage_Expecter{mock: &_m.Mock} } +// FindMissingBlockNumbers provides a mock function with given fields: chainId, startBlock, endBlock +func (_m *MockIMainStorage) FindMissingBlockNumbers(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]*big.Int, error) { + ret := _m.Called(chainId, startBlock, endBlock) + + if len(ret) == 0 { + panic("no return value specified for FindMissingBlockNumbers") + } + + var r0 []*big.Int + var r1 error + if rf, ok := ret.Get(0).(func(*big.Int, *big.Int, *big.Int) ([]*big.Int, error)); ok { + return rf(chainId, startBlock, endBlock) + } + if rf, ok := ret.Get(0).(func(*big.Int, *big.Int, *big.Int) []*big.Int); ok { + r0 = rf(chainId, startBlock, endBlock) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*big.Int) + } + } + + if rf, ok := ret.Get(1).(func(*big.Int, *big.Int, *big.Int) error); ok { + r1 = rf(chainId, startBlock, endBlock) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockIMainStorage_FindMissingBlockNumbers_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'FindMissingBlockNumbers' +type MockIMainStorage_FindMissingBlockNumbers_Call struct { + *mock.Call +} + +// FindMissingBlockNumbers is a helper method to define mock.On call +// - chainId *big.Int +// - startBlock *big.Int +// - endBlock *big.Int +func (_e *MockIMainStorage_Expecter) FindMissingBlockNumbers(chainId interface{}, startBlock interface{}, endBlock interface{}) *MockIMainStorage_FindMissingBlockNumbers_Call { + return &MockIMainStorage_FindMissingBlockNumbers_Call{Call: _e.mock.On("FindMissingBlockNumbers", chainId, startBlock, endBlock)} +} + +func (_c *MockIMainStorage_FindMissingBlockNumbers_Call) Run(run func(chainId *big.Int, startBlock *big.Int, endBlock *big.Int)) *MockIMainStorage_FindMissingBlockNumbers_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*big.Int), args[1].(*big.Int), args[2].(*big.Int)) + }) + return _c +} + +func (_c *MockIMainStorage_FindMissingBlockNumbers_Call) Return(blockNumbers []*big.Int, err error) *MockIMainStorage_FindMissingBlockNumbers_Call { + _c.Call.Return(blockNumbers, err) + return _c +} + +func (_c *MockIMainStorage_FindMissingBlockNumbers_Call) RunAndReturn(run func(*big.Int, *big.Int, *big.Int) ([]*big.Int, error)) *MockIMainStorage_FindMissingBlockNumbers_Call { + _c.Call.Return(run) + return _c +} + // GetAggregations provides a mock function with given fields: table, qf func (_m *MockIMainStorage) GetAggregations(table string, qf storage.QueryFilter) (storage.QueryResult[interface{}], error) { ret := _m.Called(table, qf) @@ -627,6 +687,66 @@ func (_c *MockIMainStorage_GetTransactions_Call) RunAndReturn(run func(storage.Q return _c } +// GetValidationBlockData provides a mock function with given fields: chainId, startBlock, endBlock +func (_m *MockIMainStorage) GetValidationBlockData(chainId *big.Int, startBlock *big.Int, endBlock *big.Int) ([]common.BlockData, error) { + ret := _m.Called(chainId, startBlock, endBlock) + + if len(ret) == 0 { + panic("no return value specified for GetValidationBlockData") + } + + var r0 []common.BlockData + var r1 error + if rf, ok := ret.Get(0).(func(*big.Int, *big.Int, *big.Int) ([]common.BlockData, error)); ok { + return rf(chainId, startBlock, endBlock) + } + if rf, ok := ret.Get(0).(func(*big.Int, *big.Int, *big.Int) []common.BlockData); ok { + r0 = rf(chainId, startBlock, endBlock) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]common.BlockData) + } + } + + if rf, ok := ret.Get(1).(func(*big.Int, *big.Int, *big.Int) error); ok { + r1 = rf(chainId, startBlock, endBlock) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockIMainStorage_GetValidationBlockData_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetValidationBlockData' +type MockIMainStorage_GetValidationBlockData_Call struct { + *mock.Call +} + +// GetValidationBlockData is a helper method to define mock.On call +// - chainId *big.Int +// - startBlock *big.Int +// - endBlock *big.Int +func (_e *MockIMainStorage_Expecter) GetValidationBlockData(chainId interface{}, startBlock interface{}, endBlock interface{}) *MockIMainStorage_GetValidationBlockData_Call { + return &MockIMainStorage_GetValidationBlockData_Call{Call: _e.mock.On("GetValidationBlockData", chainId, startBlock, endBlock)} +} + +func (_c *MockIMainStorage_GetValidationBlockData_Call) Run(run func(chainId *big.Int, startBlock *big.Int, endBlock *big.Int)) *MockIMainStorage_GetValidationBlockData_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*big.Int), args[1].(*big.Int), args[2].(*big.Int)) + }) + return _c +} + +func (_c *MockIMainStorage_GetValidationBlockData_Call) Return(blocks []common.BlockData, err error) *MockIMainStorage_GetValidationBlockData_Call { + _c.Call.Return(blocks, err) + return _c +} + +func (_c *MockIMainStorage_GetValidationBlockData_Call) RunAndReturn(run func(*big.Int, *big.Int, *big.Int) ([]common.BlockData, error)) *MockIMainStorage_GetValidationBlockData_Call { + _c.Call.Return(run) + return _c +} + // InsertBlockData provides a mock function with given fields: data func (_m *MockIMainStorage) InsertBlockData(data []common.BlockData) error { ret := _m.Called(data)