diff --git a/cmd/elasticindexer/config/config.toml b/cmd/elasticindexer/config/config.toml index b63329c7..1f443da7 100644 --- a/cmd/elasticindexer/config/config.toml +++ b/cmd/elasticindexer/config/config.toml @@ -2,7 +2,7 @@ available-indices = [ "rating", "transactions", "blocks", "validators", "miniblocks", "rounds", "accounts", "accountshistory", "receipts", "scresults", "accountsesdt", "accountsesdthistory", "epochinfo", "scdeploys", "tokens", "tags", - "logs", "delegators", "operations", "esdts", "values", "events" + "logs", "delegators", "operations", "esdts", "values", "events", "executionresults" ] [config.address-converter] length = 32 diff --git a/cmd/elasticindexer/config/prefs.toml b/cmd/elasticindexer/config/prefs.toml index b60b6ce6..f0fce3e4 100644 --- a/cmd/elasticindexer/config/prefs.toml +++ b/cmd/elasticindexer/config/prefs.toml @@ -22,3 +22,4 @@ username = "" password = "" bulk-request-max-size-in-bytes = 4194304 # 4MB + num-writes-in-parallel = 1 diff --git a/config/config.go b/config/config.go index 56034f7d..39bb6a7c 100644 --- a/config/config.go +++ b/config/config.go @@ -50,6 +50,7 @@ type ClusterConfig struct { UserName string `toml:"username"` Password string `toml:"password"` BulkRequestMaxSizeInBytes int `toml:"bulk-request-max-size-in-bytes"` + NumWritesInParallel int `toml:"num-writes-in-parallel"` } `toml:"elastic-cluster"` } `toml:"config"` } diff --git a/data/block.go b/data/block.go index 53a92569..73a598b4 100644 --- a/data/block.go +++ b/data/block.go @@ -1,6 +1,7 @@ package data import ( + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/api" ) @@ -9,47 +10,85 @@ import ( // to be saved for a block. It has all the default fields // plus some extra information for ease of search and filter type Block struct { - UUID string `json:"uuid"` - Nonce uint64 `json:"nonce"` - Round uint64 `json:"round"` - Epoch uint32 `json:"epoch"` - Hash string `json:"-"` - MiniBlocksHashes []string `json:"miniBlocksHashes"` - MiniBlocksDetails []*MiniBlocksDetails `json:"miniBlocksDetails,omitempty"` - NotarizedBlocksHashes []string `json:"notarizedBlocksHashes"` - Proposer uint64 `json:"proposer"` - Validators []uint64 `json:"validators,omitempty"` - PubKeyBitmap string `json:"pubKeyBitmap"` - Size int64 `json:"size"` - SizeTxs int64 `json:"sizeTxs"` - Timestamp uint64 `json:"timestamp"` - TimestampMs uint64 `json:"timestampMs,omitempty"` - StateRootHash string `json:"stateRootHash"` - PrevHash string `json:"prevHash"` - ShardID uint32 `json:"shardId"` - TxCount uint32 `json:"txCount"` - NotarizedTxsCount uint32 `json:"notarizedTxsCount"` - AccumulatedFees string `json:"accumulatedFees"` - DeveloperFees string `json:"developerFees"` - EpochStartBlock bool `json:"epochStartBlock"` - SearchOrder uint64 `json:"searchOrder"` - EpochStartInfo *EpochStartInfo `json:"epochStartInfo,omitempty"` - GasProvided uint64 `json:"gasProvided"` - GasRefunded uint64 `json:"gasRefunded"` - GasPenalized uint64 `json:"gasPenalized"` - MaxGasLimit uint64 `json:"maxGasLimit"` - ScheduledData *ScheduledData `json:"scheduledData,omitempty"` - EpochStartShardsData []*EpochStartShardData `json:"epochStartShardsData,omitempty"` - Proof *api.HeaderProof `json:"proof,omitempty"` - RandSeed string `json:"randSeed,omitempty"` - PrevRandSeed string `json:"prevRandSeed,omitempty"` - Signature string `json:"signature,omitempty"` - LeaderSignature string `json:"leaderSignature,omitempty"` - ChainID string `json:"chainID,omitempty"` - SoftwareVersion string `json:"softwareVersion,omitempty"` - ReceiptsHash string `json:"receiptsHash,omitempty"` - Reserved []byte `json:"reserved,omitempty"` - ProposerBlsKey string `json:"proposerBlsKey,omitempty"` + UUID string `json:"uuid"` + Nonce uint64 `json:"nonce"` + Round uint64 `json:"round"` + Epoch uint32 `json:"epoch"` + Hash string `json:"-"` + MiniBlocksHashes []string `json:"miniBlocksHashes"` + MiniBlocksDetails []*MiniBlocksDetails `json:"miniBlocksDetails,omitempty"` + NotarizedBlocksHashes []string `json:"notarizedBlocksHashes"` + Proposer uint64 `json:"proposer"` + Validators []uint64 `json:"validators,omitempty"` + PubKeyBitmap string `json:"pubKeyBitmap"` + Size int64 `json:"size"` + SizeTxs int64 `json:"sizeTxs"` + Timestamp uint64 `json:"timestamp"` + TimestampMs uint64 `json:"timestampMs,omitempty"` + StateRootHash string `json:"stateRootHash"` + PrevHash string `json:"prevHash"` + ShardID uint32 `json:"shardId"` + TxCount uint32 `json:"txCount"` + NotarizedTxsCount uint32 `json:"notarizedTxsCount"` + AccumulatedFees string `json:"accumulatedFees"` + DeveloperFees string `json:"developerFees"` + EpochStartBlock bool `json:"epochStartBlock"` + SearchOrder uint64 `json:"searchOrder"` + EpochStartInfo *EpochStartInfo `json:"epochStartInfo,omitempty"` + GasProvided uint64 `json:"gasProvided"` + GasRefunded uint64 `json:"gasRefunded"` + GasPenalized uint64 `json:"gasPenalized"` + MaxGasLimit uint64 `json:"maxGasLimit"` + ScheduledData *ScheduledData `json:"scheduledData,omitempty"` + EpochStartShardsData []*EpochStartShardData `json:"epochStartShardsData,omitempty"` + Proof *api.HeaderProof `json:"proof,omitempty"` + RandSeed string `json:"randSeed,omitempty"` + PrevRandSeed string `json:"prevRandSeed,omitempty"` + Signature string `json:"signature,omitempty"` + LeaderSignature string `json:"leaderSignature,omitempty"` + ChainID string `json:"chainID,omitempty"` + SoftwareVersion string `json:"softwareVersion,omitempty"` + ReceiptsHash string `json:"receiptsHash,omitempty"` + Reserved []byte `json:"reserved,omitempty"` + ProposerBlsKey string `json:"proposerBlsKey,omitempty"` + ExecutionResultBlockHashes []string `json:"executionResultBlockHashes,omitempty"` +} + +// ExecutionResult is a structure containing all the fields that need +// +// to be saved for an execution results. +type ExecutionResult struct { + UUID string `json:"uuid"` + Hash string `json:"-"` + RootHash string `json:"rootHash"` + NotarizedInBlockHash string `json:"notarizedInBlockHash"` + AccumulatedFees string `json:"accumulatedFees"` + DeveloperFees string `json:"developerFees"` + TxCount uint64 `json:"txCount"` + GasUsed uint64 `json:"gasUsed"` + Nonce uint64 `json:"nonce"` + Round uint64 `json:"round"` + Epoch uint32 `json:"epoch"` + MiniBlocksHashes []string `json:"miniBlocksHashes"` + MiniBlocksDetails []*MiniBlocksDetails `json:"miniBlocksDetails,omitempty"` +} + +// HeaderData is the DTO that holds fields from header or execution result +type HeaderData struct { + Timestamp uint64 + TimestampMs uint64 + Round uint64 + ShardID uint32 + NumberOfShards uint32 + Epoch uint32 + HeaderHash []byte + MiniBlockHeaders []coreData.MiniBlockHeaderHandler +} + +// PreparedBlockResults is the DTO that holds all the results after processing a block +type PreparedBlockResults struct { + Block *Block + ExecutionResults []*ExecutionResult } // MiniBlocksDetails is a structure that hold information about mini-blocks execution details diff --git a/factory/wsIndexerFactory.go b/factory/wsIndexerFactory.go index b0c9b86e..4014d877 100644 --- a/factory/wsIndexerFactory.go +++ b/factory/wsIndexerFactory.go @@ -98,6 +98,7 @@ func createDataIndexer( StatusMetrics: statusMetrics, Version: version, EnableEpochsConfig: enableEpochsCfg, + NumWritesInParallel: clusterCfg.Config.ElasticCluster.NumWritesInParallel, }) } diff --git a/go.mod b/go.mod index 315531ff..c13fb074 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,11 @@ go 1.23 require ( github.com/elastic/go-elasticsearch/v7 v7.12.0 - github.com/gin-contrib/cors v1.4.0 + github.com/gin-contrib/cors v1.6.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/multiversx/mx-chain-communication-go v1.3.0 - github.com/multiversx/mx-chain-core-go v1.4.0 + github.com/multiversx/mx-chain-core-go v1.4.2-0.20251113093212-347994c38d2d github.com/multiversx/mx-chain-logger-go v1.1.0 github.com/multiversx/mx-chain-vm-common-go v1.6.0 github.com/prometheus/client_model v0.6.1 @@ -40,6 +40,7 @@ require ( github.com/gorilla/websocket v1.5.3 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/go.sum b/go.sum index 02b1c53c..a3116602 100644 --- a/go.sum +++ b/go.sum @@ -46,26 +46,20 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= -github.com/gin-contrib/cors v1.4.0 h1:oJ6gwtUl3lqV0WEIwM/LxPF1QZ5qe2lGWdY2+bz7y0g= -github.com/gin-contrib/cors v1.4.0/go.mod h1:bs9pNM0x/UsmHPBWT2xZz9ROh8xYjYkiURUfmBoMlcs= +github.com/gin-contrib/cors v1.6.0 h1:0Z7D/bVhE6ja07lI8CTjTonp6SB07o8bNuFyRbsBUQg= +github.com/gin-contrib/cors v1.6.0/go.mod h1:cI+h6iOAyxKRtUtC6iF/Si1KSFvGm/gK+kshxlCi8ro= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= -github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBExVwjEviJTixqxL8= github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -107,18 +101,12 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -130,8 +118,8 @@ github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/multiversx/mx-chain-communication-go v1.3.0 h1:ziNM1dRuiR/7al2L/jGEA/a/hjurtJ/HEqgazHNt9P8= github.com/multiversx/mx-chain-communication-go v1.3.0/go.mod h1:gDVWn6zUW6aCN1YOm/FbbT5MUmhgn/L1Rmpl8EoH3Yg= -github.com/multiversx/mx-chain-core-go v1.4.0 h1:p6FbfCzvMXF54kpS0B5mrjNWYpq4SEQqo0UvrMF7YVY= -github.com/multiversx/mx-chain-core-go v1.4.0/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20251113093212-347994c38d2d h1:5N0OXoa//0hMxQ9PYUjwf3el4cuEU4/E+mpp6Y416Js= +github.com/multiversx/mx-chain-core-go v1.4.2-0.20251113093212-347994c38d2d/go.mod h1:IO+vspNan+gT0WOHnJ95uvWygiziHZvfXpff6KnxV7g= github.com/multiversx/mx-chain-crypto-go v1.3.0 h1:0eK2bkDOMi8VbSPrB1/vGJSYT81IBtfL4zw+C4sWe/k= github.com/multiversx/mx-chain-crypto-go v1.3.0/go.mod h1:nPIkxxzyTP8IquWKds+22Q2OJ9W7LtusC7cAosz7ojM= github.com/multiversx/mx-chain-logger-go v1.1.0 h1:97x84A6L4RfCa6YOx1HpAFxZp1cf/WI0Qh112whgZNM= @@ -151,18 +139,14 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -172,7 +156,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -190,8 +173,6 @@ github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= -github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= @@ -205,7 +186,6 @@ golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -218,7 +198,6 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -235,18 +214,12 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -265,14 +238,11 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -280,7 +250,6 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/integrationtests/executionResults_test.go b/integrationtests/executionResults_test.go new file mode 100644 index 00000000..2d42ddb5 --- /dev/null +++ b/integrationtests/executionResults_test.go @@ -0,0 +1,58 @@ +//go:build integrationtests + +package integrationtests + +import ( + "context" + "encoding/hex" + "testing" + + dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" + indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" + "github.com/stretchr/testify/require" +) + +func TestIndexExecutionResults(t *testing.T) { + esClient, err := createESClient(esURL) + require.Nil(t, err) + + esProc, err := CreateElasticProcessor(esClient) + require.Nil(t, err) + + header := &dataBlock.HeaderV3{ + ExecutionResults: []*dataBlock.ExecutionResult{ + { + BaseExecutionResult: &dataBlock.BaseExecutionResult{ + HeaderHash: []byte("h1"), + HeaderNonce: 1, + HeaderRound: 2, + HeaderEpoch: 3, + }, + }, + { + BaseExecutionResult: &dataBlock.BaseExecutionResult{ + HeaderHash: []byte("h2"), + HeaderNonce: 2, + HeaderRound: 3, + HeaderEpoch: 3, + }, + }, + }, + } + pool := &outport.TransactionPool{} + body := &dataBlock.Body{} + ob := createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards) + ob.BlockData.HeaderHash = []byte("hh1") + ob.HeaderGasConsumption = &outport.HeaderGasConsumption{} + err = esProc.SaveHeader(ob) + require.Nil(t, err) + + ids := []string{hex.EncodeToString([]byte("h1")), hex.EncodeToString([]byte("h2"))} + genericResponse := &GenericResponse{} + err = esClient.DoMultiGet(context.Background(), ids, indexerdata.ExecutionResultsIndex, true, genericResponse) + require.Nil(t, err) + + require.JSONEq(t, readExpectedResult("./testdata/executionResults/execution-result-1.json"), string(genericResponse.Docs[0].Source)) + require.JSONEq(t, readExpectedResult("./testdata/executionResults/execution-result-2.json"), string(genericResponse.Docs[1].Source)) +} diff --git a/integrationtests/miniblocks_test.go b/integrationtests/miniblocks_test.go index 9cde9fe5..b0d99f6f 100644 --- a/integrationtests/miniblocks_test.go +++ b/integrationtests/miniblocks_test.go @@ -4,9 +4,11 @@ package integrationtests import ( "context" + "encoding/hex" "testing" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" + "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/marshal" indexerdata "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/stretchr/testify/require" @@ -31,7 +33,12 @@ func TestIndexMiniBlocksOnSourceAndDestination(t *testing.T) { ReceiverShardID: 2, }, } - err = esProc.SaveMiniblocks(header, miniBlocks, 1234000) + + body := &dataBlock.Body{MiniBlocks: miniBlocks} + pool := &outport.TransactionPool{} + ob := createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards) + ob.BlockData.HeaderHash, _ = hex.DecodeString("3fede8a9a3c4f2ba6d7e6e01541813606cd61c4d3af2940f8e089827b5d94e50") + err = esProc.SaveMiniblocks(ob) require.Nil(t, err) mbHash := "11a1bb4065e16a2e93b2b5ac5957b7b69f1cfba7579b170b24f30dab2d3162e0" ids := []string{mbHash} @@ -57,7 +64,9 @@ func TestIndexMiniBlocksOnSourceAndDestination(t *testing.T) { }, } - err = esProc.SaveMiniblocks(header, miniBlocks, 1234000) + ob = createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards) + ob.BlockData.HeaderHash, _ = hex.DecodeString("d7f1e8003a45c7adbd87bbbb269cb4af3d1f4aedd0c214973bfc096dd0f3b65e") + err = esProc.SaveMiniblocks(ob) require.Nil(t, err) err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) @@ -96,7 +105,12 @@ func TestIndexMiniBlockFirstOnDestinationAndAfterSource(t *testing.T) { }, } - err = esProc.SaveMiniblocks(header, miniBlocks, 54321000) + pool := &outport.TransactionPool{} + body := &dataBlock.Body{MiniBlocks: miniBlocks} + ob := createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards) + ob.BlockData.HeaderHash, _ = hex.DecodeString("b36435faaa72390772da84f418348ce0d477c74432579519bf0ffea1dc4c36e9") + ob.BlockData.TimestampMs = 54321000 + err = esProc.SaveMiniblocks(ob) require.Nil(t, err) genericResponse := &GenericResponse{} @@ -116,7 +130,10 @@ func TestIndexMiniBlockFirstOnDestinationAndAfterSource(t *testing.T) { Reserved: mbhrBytes, }, } - err = esProc.SaveMiniblocks(header, miniBlocks, 54321000) + + ob = createOutportBlockWithHeader(body, header, pool, nil, testNumOfShards) + ob.BlockData.HeaderHash, _ = hex.DecodeString("b601381e1f41df2aa3da9f2b8eb169f14c86418229e30fc65f9e6b37b7f0d902") + err = esProc.SaveMiniblocks(ob) require.Nil(t, err) err = esClient.DoMultiGet(context.Background(), ids, indexerdata.MiniblocksIndex, true, genericResponse) require.Nil(t, err) diff --git a/integrationtests/testdata/executionResults/execution-result-1.json b/integrationtests/testdata/executionResults/execution-result-1.json new file mode 100644 index 00000000..c406d299 --- /dev/null +++ b/integrationtests/testdata/executionResults/execution-result-1.json @@ -0,0 +1,12 @@ +{ + "rootHash": "", + "notarizedInBlockHash": "686831", + "accumulatedFees": "", + "developerFees": "", + "txCount": 0, + "gasUsed": 0, + "nonce": 1, + "round": 2, + "epoch": 3, + "miniBlocksHashes":null +} diff --git a/integrationtests/testdata/executionResults/execution-result-2.json b/integrationtests/testdata/executionResults/execution-result-2.json new file mode 100644 index 00000000..b08b7510 --- /dev/null +++ b/integrationtests/testdata/executionResults/execution-result-2.json @@ -0,0 +1,12 @@ +{ + "rootHash": "", + "notarizedInBlockHash": "686831", + "accumulatedFees": "", + "developerFees": "", + "txCount": 0, + "gasUsed": 0, + "nonce": 2, + "round": 3, + "epoch": 3, + "miniBlocksHashes":null +} diff --git a/integrationtests/utils.go b/integrationtests/utils.go index 0796e9c8..6b8c4d6c 100644 --- a/integrationtests/utils.go +++ b/integrationtests/utils.go @@ -60,11 +60,12 @@ func CreateElasticProcessor( DBClient: esClient, EnabledIndexes: []string{dataindexer.TransactionsIndex, dataindexer.LogsIndex, dataindexer.AccountsESDTIndex, dataindexer.ScResultsIndex, dataindexer.ReceiptsIndex, dataindexer.BlockIndex, dataindexer.AccountsIndex, dataindexer.TokensIndex, dataindexer.TagsIndex, dataindexer.EventsIndex, - dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex}, + dataindexer.OperationsIndex, dataindexer.DelegatorsIndex, dataindexer.ESDTsIndex, dataindexer.SCDeploysIndex, dataindexer.MiniblocksIndex, dataindexer.ValuesIndex, dataindexer.ExecutionResultsIndex}, Denomination: 18, EnableEpochsConfig: config.EnableEpochsConfig{ RelayedTransactionsV1V2DisableEpoch: 1, }, + NumWritesInParallel: 1, } return factory.CreateElasticProcessor(args) diff --git a/mock/dbTransactionsHandlerStub.go b/mock/dbTransactionsHandlerStub.go index b50d85f9..a11bafaa 100644 --- a/mock/dbTransactionsHandlerStub.go +++ b/mock/dbTransactionsHandlerStub.go @@ -1,7 +1,6 @@ package mock import ( - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -9,7 +8,7 @@ import ( // DBTransactionProcessorStub - type DBTransactionProcessorStub struct { - PrepareTransactionsForDatabaseCalled func(mbs []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults + PrepareTransactionsForDatabaseCalled func(mbs []*block.MiniBlock, headerData *data.HeaderData, pool *outport.TransactionPool) *data.PreparedResults SerializeReceiptsCalled func(recs []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeScResultsCalled func(scrs []*data.ScResult, buffSlice *data.BufferSlice, index string) error } @@ -20,16 +19,16 @@ func (tps *DBTransactionProcessorStub) SerializeTransactionsFeeData(_ map[string } // PrepareTransactionsForDatabase - -func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(mbs []*block.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool, _ bool, _ uint32, _ uint64) *data.PreparedResults { +func (tps *DBTransactionProcessorStub) PrepareTransactionsForDatabase(mbs []*block.MiniBlock, headerData *data.HeaderData, pool *outport.TransactionPool, _ bool) *data.PreparedResults { if tps.PrepareTransactionsForDatabaseCalled != nil { - return tps.PrepareTransactionsForDatabaseCalled(mbs, header, pool) + return tps.PrepareTransactionsForDatabaseCalled(mbs, headerData, pool) } return nil } // GetHexEncodedHashesForRemove - -func (tps *DBTransactionProcessorStub) GetHexEncodedHashesForRemove(_ coreData.HeaderHandler, _ *block.Body) ([]string, []string) { +func (tps *DBTransactionProcessorStub) GetHexEncodedHashesForRemove(_ *data.HeaderData, _ *block.Body) ([]string, []string) { return nil, nil } diff --git a/mock/elasticProcessorStub.go b/mock/elasticProcessorStub.go index b9f7dd7a..389f7787 100644 --- a/mock/elasticProcessorStub.go +++ b/mock/elasticProcessorStub.go @@ -10,9 +10,9 @@ import ( type ElasticProcessorStub struct { SaveHeaderCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error RemoveHeaderCalled func(header coreData.HeaderHandler) error - RemoveMiniblocksCalled func(header coreData.HeaderHandler, body *block.Body) error + RemoveMiniblocksCalled func(header coreData.HeaderHandler) error RemoveTransactionsCalled func(header coreData.HeaderHandler, body *block.Body) error - SaveMiniblocksCalled func(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMs uint64) error + SaveMiniblocksCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveTransactionsCalled func(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveValidatorsRatingCalled func(validatorsRating *outport.ValidatorsRating) error SaveRoundsInfoCalled func(infos *outport.RoundsInfo) error @@ -47,9 +47,9 @@ func (eim *ElasticProcessorStub) RemoveHeader(header coreData.HeaderHandler) err } // RemoveMiniblocks - -func (eim *ElasticProcessorStub) RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error { +func (eim *ElasticProcessorStub) RemoveMiniblocks(header coreData.HeaderHandler) error { if eim.RemoveMiniblocksCalled != nil { - return eim.RemoveMiniblocksCalled(header, body) + return eim.RemoveMiniblocksCalled(header) } return nil } @@ -63,9 +63,9 @@ func (eim *ElasticProcessorStub) RemoveTransactions(header coreData.HeaderHandle } // SaveMiniblocks - -func (eim *ElasticProcessorStub) SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMs uint64) error { +func (eim *ElasticProcessorStub) SaveMiniblocks(outportBlockWithHeader *outport.OutportBlockWithHeader) error { if eim.SaveMiniblocksCalled != nil { - return eim.SaveMiniblocksCalled(header, miniBlocks, timestampMs) + return eim.SaveMiniblocksCalled(outportBlockWithHeader) } return nil } diff --git a/process/dataindexer/constants.go b/process/dataindexer/constants.go index 229fc5d8..3ea0073a 100644 --- a/process/dataindexer/constants.go +++ b/process/dataindexer/constants.go @@ -29,8 +29,6 @@ const ( AccountsESDTHistoryIndex = "accountsesdthistory" // EpochInfoIndex is the Elasticsearch index for the epoch information EpochInfoIndex = "epochinfo" - // OpenDistroIndex is the Elasticsearch index for opendistro - OpenDistroIndex = "opendistro" // SCDeploysIndex is the Elasticsearch index for the smart contracts deploy information SCDeploysIndex = "scdeploys" // TokensIndex is the Elasticsearch index for the ESDT tokens @@ -49,29 +47,6 @@ const ( ValuesIndex = "values" // EventsIndex is the Elasticsearch index for log events EventsIndex = "events" - - // TransactionsPolicy is the Elasticsearch policy for the transactions - TransactionsPolicy = "transactions_policy" - // BlockPolicy is the Elasticsearch policy for the blocks - BlockPolicy = "blocks_policy" - // MiniblocksPolicy is the Elasticsearch policy for the miniblocks - MiniblocksPolicy = "miniblocks_policy" - // ValidatorsPolicy is the Elasticsearch policy for the validators information - ValidatorsPolicy = "validators_policy" - // RoundsPolicy is the Elasticsearch policy for the rounds information - RoundsPolicy = "rounds_policy" - // RatingPolicy is the Elasticsearch policy for the rating information - RatingPolicy = "rating_policy" - // AccountsPolicy is the Elasticsearch policy for the accounts - AccountsPolicy = "accounts_policy" - // AccountsHistoryPolicy is the Elasticsearch policy for the accounts history information - AccountsHistoryPolicy = "accountshistory_policy" - // AccountsESDTPolicy is the Elasticsearch policy for the accounts with ESDT balance - AccountsESDTPolicy = "accountsesdt_policy" - // AccountsESDTHistoryPolicy is the Elasticsearch policy for the accounts history information with ESDT - AccountsESDTHistoryPolicy = "accountsesdthistory_policy" - // ScResultsPolicy is the Elasticsearch policy for the smart contract results - ScResultsPolicy = "scresults_policy" - // ReceiptsPolicy is the Elasticsearch policy for the receipts - ReceiptsPolicy = "receipts_policy" + // ExecutionResultsIndex is the Elasticsearch index for execution results + ExecutionResultsIndex = "executionresults" ) diff --git a/process/dataindexer/dataIndexer.go b/process/dataindexer/dataIndexer.go index ca2460d8..9c6dcb18 100644 --- a/process/dataindexer/dataIndexer.go +++ b/process/dataindexer/dataIndexer.go @@ -110,12 +110,7 @@ func (di *dataIndexer) saveBlockData(outportBlock *outport.OutportBlock, header err, hex.EncodeToString(headerHash), headerNonce) } - if len(outportBlock.BlockData.Body.MiniBlocks) == 0 { - return nil - } - - miniBlocks := append(outportBlock.BlockData.Body.MiniBlocks, outportBlock.BlockData.IntraShardMiniBlocks...) - err = di.elasticProcessor.SaveMiniblocks(header, miniBlocks, outportBlock.BlockData.GetTimestampMs()) + err = di.elasticProcessor.SaveMiniblocks(outportBlockWithHeader) if err != nil { return fmt.Errorf("%w when saving miniblocks, block hash %s, nonce %d", err, hex.EncodeToString(headerHash), headerNonce) @@ -147,7 +142,7 @@ func (di *dataIndexer) RevertIndexedBlock(blockData *outport.BlockData) error { return err } - err = di.elasticProcessor.RemoveMiniblocks(header, blockData.Body) + err = di.elasticProcessor.RemoveMiniblocks(header) if err != nil { return err } diff --git a/process/dataindexer/dataIndexer_test.go b/process/dataindexer/dataIndexer_test.go index 0a6404c2..bde18a1d 100644 --- a/process/dataindexer/dataIndexer_test.go +++ b/process/dataindexer/dataIndexer_test.go @@ -62,7 +62,7 @@ func TestDataIndexer_SaveBlock(t *testing.T) { countMap[0]++ return nil }, - SaveMiniblocksCalled: func(header coreData.HeaderHandler, miniBlocks []*dataBlock.MiniBlock, timestampMs uint64) error { + SaveMiniblocksCalled: func(outportBlockWithHeader *outport.OutportBlockWithHeader) error { countMap[1]++ return nil }, @@ -159,7 +159,7 @@ func TestDataIndexer_RevertIndexedBlock(t *testing.T) { countMap[0]++ return nil }, - RemoveMiniblocksCalled: func(header coreData.HeaderHandler, body *dataBlock.Body) error { + RemoveMiniblocksCalled: func(header coreData.HeaderHandler) error { countMap[1]++ return nil }, diff --git a/process/dataindexer/interface.go b/process/dataindexer/interface.go index 1583de6b..e52942c3 100644 --- a/process/dataindexer/interface.go +++ b/process/dataindexer/interface.go @@ -14,10 +14,10 @@ import ( type ElasticProcessor interface { SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error RemoveHeader(header coreData.HeaderHandler) error - RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error + RemoveMiniblocks(header coreData.HeaderHandler) error RemoveTransactions(header coreData.HeaderHandler, body *block.Body, uint65 uint64) error RemoveAccountsESDT(shardID uint32, timestampMS uint64) error - SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMS uint64) error + SaveMiniblocks(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveTransactions(outportBlockWithHeader *outport.OutportBlockWithHeader) error SaveValidatorsRating(ratingData *outport.ValidatorsRating) error SaveRoundsInfo(rounds *outport.RoundsInfo) error diff --git a/process/elasticproc/block/blockProcessor.go b/process/elasticproc/block/blockProcessor.go index 0fcadb60..50bd2151 100644 --- a/process/elasticproc/block/blockProcessor.go +++ b/process/elasticproc/block/blockProcessor.go @@ -57,7 +57,7 @@ func NewBlockProcessor(hasher hashing.Hasher, marshalizer marshal.Marshalizer, v } // PrepareBlockForDB will prepare a database block and serialize it for database -func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.Block, error) { +func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.PreparedBlockResults, error) { if check.IfNil(obh.Header) { return nil, indexer.ErrNilHeaderHandler } @@ -131,14 +131,76 @@ func (bp *blockProcessor) PrepareBlockForDB(obh *outport.OutportBlockWithHeader) } } + for _, executionResult := range obh.Header.GetExecutionResultsHandlers() { + elasticBlock.ExecutionResultBlockHashes = append(elasticBlock.ExecutionResultBlockHashes, hex.EncodeToString(executionResult.GetHeaderHash())) + } + bp.addEpochStartInfoForMeta(obh.Header, elasticBlock) - appendBlockDetailsFromHeaders(elasticBlock, obh.Header, obh.BlockData.Body, obh.TransactionPool) + elasticBlock.MiniBlocksDetails = prepareMiniBlockDetails(obh.Header.GetMiniBlockHeaderHandlers(), obh.BlockData.Body, obh.TransactionPool) + appendBlockDetailsFromIntraShardMbs(elasticBlock, obh.BlockData.IntraShardMiniBlocks, obh.TransactionPool, len(obh.Header.GetMiniBlockHeaderHandlers())) addProofs(elasticBlock, obh) - return elasticBlock, nil + return &data.PreparedBlockResults{ + Block: elasticBlock, + ExecutionResults: bp.prepareExecutionResults(obh), + }, nil +} + +func (bp *blockProcessor) prepareExecutionResults(obh *outport.OutportBlockWithHeader) []*data.ExecutionResult { + if !obh.Header.IsHeaderV3() { + return []*data.ExecutionResult{} + } + + executionResults := make([]*data.ExecutionResult, 0) + for _, executionResultHandler := range obh.Header.GetExecutionResultsHandlers() { + executionResult := bp.prepareExecutionResult(executionResultHandler, obh) + + executionResults = append(executionResults, executionResult) + } + + return executionResults +} + +func (bp *blockProcessor) prepareExecutionResult(baseExecutionResult coreData.BaseExecutionResultHandler, obh *outport.OutportBlockWithHeader) *data.ExecutionResult { + executionResultsHash := hex.EncodeToString(baseExecutionResult.GetHeaderHash()) + executionResult := &data.ExecutionResult{ + UUID: converters.GenerateBase64UUID(), + Hash: executionResultsHash, + RootHash: hex.EncodeToString(baseExecutionResult.GetRootHash()), + NotarizedInBlockHash: hex.EncodeToString(obh.BlockData.GetHeaderHash()), + Nonce: baseExecutionResult.GetHeaderNonce(), + Round: baseExecutionResult.GetHeaderRound(), + Epoch: baseExecutionResult.GetHeaderEpoch(), + GasUsed: baseExecutionResult.GetGasUsed(), + } + + executionResultData, found := obh.BlockData.Results[executionResultsHash] + if !found { + log.Warn("cannot find execution result data for execution result", "hash", executionResultsHash) + return executionResult + } + + executionResult.MiniBlocksHashes = bp.getEncodedMBSHashes(executionResultData.Body, executionResultData.IntraShardMiniBlocks) + + switch t := baseExecutionResult.(type) { + case *nodeBlock.MetaExecutionResult: + executionResult.MiniBlocksDetails = prepareMiniBlockDetails(t.GetMiniBlockHeadersHandlers(), executionResultData.Body, executionResultData.TransactionPool) + executionResult.AccumulatedFees = t.AccumulatedFees.String() + executionResult.DeveloperFees = t.DeveloperFees.String() + executionResult.TxCount = t.ExecutedTxCount + case *nodeBlock.ExecutionResult: + executionResult.MiniBlocksDetails = prepareMiniBlockDetails(t.GetMiniBlockHeadersHandlers(), executionResultData.Body, executionResultData.TransactionPool) + executionResult.AccumulatedFees = t.AccumulatedFees.String() + executionResult.DeveloperFees = t.DeveloperFees.String() + executionResult.TxCount = t.ExecutedTxCount + default: + return executionResult + } + + return executionResult } func getLeaderIndex(obh *outport.OutportBlockWithHeader) uint64 { @@ -287,15 +349,16 @@ func (bp *blockProcessor) getEncodedMBSHashes(body *nodeBlock.Body, intraShardMb return miniblocksHashes } -func appendBlockDetailsFromHeaders(block *data.Block, header coreData.HeaderHandler, body *nodeBlock.Body, pool *outport.TransactionPool) { - for idx, mbHeader := range header.GetMiniBlockHeaderHandlers() { +func prepareMiniBlockDetails(mbHeaders []coreData.MiniBlockHeaderHandler, body *nodeBlock.Body, pool *outport.TransactionPool) []*data.MiniBlocksDetails { + mbsDetails := make([]*data.MiniBlocksDetails, 0, len(mbHeaders)) + for idx, mbHeader := range mbHeaders { mbType := nodeBlock.Type(mbHeader.GetTypeInt32()) if mbType == nodeBlock.PeerBlock { continue } txsHashes := body.MiniBlocks[idx].TxHashes - block.MiniBlocksDetails = append(block.MiniBlocksDetails, &data.MiniBlocksDetails{ + mbsDetails = append(mbsDetails, &data.MiniBlocksDetails{ IndexFirstProcessedTx: mbHeader.GetIndexOfFirstTxProcessed(), IndexLastProcessedTx: mbHeader.GetIndexOfLastTxProcessed(), MBIndex: idx, @@ -307,6 +370,8 @@ func appendBlockDetailsFromHeaders(block *data.Block, header coreData.HeaderHand ExecutionOrderTxsIndices: extractExecutionOrderIndicesFromPool(mbHeader, txsHashes, pool), }) } + + return mbsDetails } func appendBlockDetailsFromIntraShardMbs(block *data.Block, intraShardMbs []*nodeBlock.MiniBlock, pool *outport.TransactionPool, offset int) { diff --git a/process/elasticproc/block/blockProcessor_test.go b/process/elasticproc/block/blockProcessor_test.go index acff1533..3bd02954 100644 --- a/process/elasticproc/block/blockProcessor_test.go +++ b/process/elasticproc/block/blockProcessor_test.go @@ -3,10 +3,11 @@ package block import ( "encoding/hex" "errors" - "github.com/multiversx/mx-chain-core-go/data/api" "math/big" "testing" + "github.com/multiversx/mx-chain-core-go/data/api" + "github.com/multiversx/mx-chain-core-go/core" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -100,10 +101,10 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { HeaderGasConsumption: &outport.HeaderGasConsumption{}, }, } - dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) + blockResults, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Nil(t, err) - dbBlock.UUID = "" + blockResults.Block.UUID = "" expectedBlock := &data.Block{ Hash: "68617368", @@ -123,8 +124,9 @@ func TestBlockProcessor_PrepareBlockForDBShouldWork(t *testing.T) { SoftwareVersion: "31", ReceiptsHash: "68617368", Reserved: []byte("reserved"), + MiniBlocksDetails: []*data.MiniBlocksDetails{}, } - require.Equal(t, expectedBlock, dbBlock) + require.Equal(t, expectedBlock, blockResults.Block) } func TestBlockProcessor_PrepareBlockForDBNilHeader(t *testing.T) { @@ -299,8 +301,9 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { }, } - dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) - dbBlock.UUID = "" + blockResults, err := bp.PrepareBlockForDB(outportBlockWithHeader) + require.Nil(t, err) + blockResults.Block.UUID = "" require.Equal(t, nil, err) require.Equal(t, &data.Block{ @@ -388,7 +391,7 @@ func TestBlockProcessor_PrepareBlockForDBEpochStartMeta(t *testing.T) { TxCount: 170, AccumulatedFees: "0", DeveloperFees: "0", - }, dbBlock) + }, blockResults.Block) } func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { @@ -497,9 +500,10 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { }, } - dbBlock, err := bp.PrepareBlockForDB(outportBlockWithHeader) + blockResults, err := bp.PrepareBlockForDB(outportBlockWithHeader) require.Nil(t, err) - dbBlock.UUID = "" + require.Equal(t, 0, len(blockResults.ExecutionResults)) + blockResults.Block.UUID = "" require.Equal(t, &data.Block{ Hash: "68617368", @@ -557,5 +561,98 @@ func TestBlockProcessor_PrepareBlockForDBMiniBlocksDetails(t *testing.T) { ExecutionOrderTxsIndices: []int{4}, TxsHashes: []string{"696e747261534352"}}, }, - }, dbBlock) + }, blockResults.Block) +} + +func TestPrepareExecutionResult(t *testing.T) { + t.Parallel() + + bp, _ := NewBlockProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}, &mock.PubkeyConverterMock{}) + + executionResultHeaderHash := []byte("h1") + mb1Hash := []byte("mb1") + txHash := []byte("tx1") + + outportBlockWithHeader := &outport.OutportBlockWithHeader{ + Header: &dataBlock.HeaderV3{ + ExecutionResults: []*dataBlock.ExecutionResult{ + { + BaseExecutionResult: &dataBlock.BaseExecutionResult{ + HeaderHash: executionResultHeaderHash, + HeaderNonce: 1, + HeaderRound: 2, + HeaderEpoch: 3, + }, + MiniBlockHeaders: []dataBlock.MiniBlockHeader{ + { + Hash: mb1Hash, + SenderShardID: 0, + ReceiverShardID: 0, + Type: dataBlock.TxBlock, + TxCount: 1, + }, + }, + AccumulatedFees: big.NewInt(1), + DeveloperFees: big.NewInt(2), + }, + }, + }, + OutportBlock: &outport.OutportBlock{ + HeaderGasConsumption: &outport.HeaderGasConsumption{}, + BlockData: &outport.BlockData{ + Body: &dataBlock.Body{}, + Results: map[string]*outport.ExecutionResultData{ + hex.EncodeToString(executionResultHeaderHash): { + Body: &dataBlock.Body{ + MiniBlocks: []*dataBlock.MiniBlock{ + { + SenderShardID: 0, + ReceiverShardID: 0, + Type: dataBlock.TxBlock, + TxHashes: [][]byte{txHash}, + }, + }, + }, + TransactionPool: &outport.TransactionPool{ + Transactions: map[string]*outport.TxInfo{ + hex.EncodeToString(txHash): { + Transaction: &transaction.Transaction{}, + ExecutionOrder: 2, + }, + }, + }, + }, + }, + }, + }, + } + + results, err := bp.PrepareBlockForDB(outportBlockWithHeader) + require.NoError(t, err) + + require.Equal(t, []string{"6831"}, results.Block.ExecutionResultBlockHashes) + results.ExecutionResults[0].UUID = "" + require.Equal(t, &data.ExecutionResult{ + Hash: "6831", + RootHash: "", + NotarizedInBlockHash: "", + AccumulatedFees: "1", + DeveloperFees: "2", + TxCount: 0, + GasUsed: 0, + Nonce: 1, + Round: 2, + Epoch: 3, + MiniBlocksHashes: []string{"2dae16da63bc04a18cf7609e0a79d7867b11463660dbab048b044b8434bf0a82"}, + MiniBlocksDetails: []*data.MiniBlocksDetails{ + { + IndexLastProcessedTx: 0, + IndexFirstProcessedTx: 0, + Type: dataBlock.TxBlock.String(), + ProcessingType: dataBlock.Normal.String(), + TxsHashes: []string{hex.EncodeToString(txHash)}, + ExecutionOrderTxsIndices: []int{2}, + }, + }, + }, results.ExecutionResults[0]) } diff --git a/process/elasticproc/block/serialize.go b/process/elasticproc/block/serialize.go index 3f5f107e..6c8fa008 100644 --- a/process/elasticproc/block/serialize.go +++ b/process/elasticproc/block/serialize.go @@ -27,6 +27,23 @@ func (bp *blockProcessor) SerializeBlock(elasticBlock *data.Block, buffSlice *da return buffSlice.PutData(meta, serializedData) } +// SerializeExecutionResults will serialize execution results slice for database +func (bp *blockProcessor) SerializeExecutionResults(executionResults []*data.ExecutionResult, buffSlice *data.BufferSlice, index string) error { + for _, result := range executionResults { + meta := []byte(fmt.Sprintf(`{ "index" : { "_index":"%s", "_id" : "%s" } }%s`, index, converters.JsonEscape(result.Hash), "\n")) + serializedData, err := json.Marshal(result) + if err != nil { + return err + } + + err = buffSlice.PutData(meta, serializedData) + if err != nil { + return err + } + } + return nil +} + // SerializeEpochInfoData will serialize information about current epoch func (bp *blockProcessor) SerializeEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice, index string) error { if check.IfNil(header) { diff --git a/process/elasticproc/converters/miniblocks.go b/process/elasticproc/converters/miniblocks.go new file mode 100644 index 00000000..23626d65 --- /dev/null +++ b/process/elasticproc/converters/miniblocks.go @@ -0,0 +1,19 @@ +package converters + +import ( + coreData "github.com/multiversx/mx-chain-core-go/data" +) + +type executionResultHandler interface { + GetMiniBlockHeadersHandlers() []coreData.MiniBlockHeaderHandler +} + +// GetMiniBlocksHeaderHandlersFromExecResult returns miniblock handlers based on execution result +func GetMiniBlocksHeaderHandlersFromExecResult(baseExecResult coreData.BaseExecutionResultHandler) []coreData.MiniBlockHeaderHandler { + execResult, ok := baseExecResult.(executionResultHandler) + if !ok { + return []coreData.MiniBlockHeaderHandler{} + } + + return execResult.GetMiniBlockHeadersHandlers() +} diff --git a/process/elasticproc/elasticProcessor.go b/process/elasticproc/elasticProcessor.go index 7394ce70..14ec021b 100644 --- a/process/elasticproc/elasticProcessor.go +++ b/process/elasticproc/elasticProcessor.go @@ -5,7 +5,10 @@ import ( "context" "encoding/hex" "encoding/json" + "errors" "fmt" + "sync" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" @@ -20,7 +23,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/tokeninfo" "github.com/multiversx/mx-chain-es-indexer-go/templates" logger "github.com/multiversx/mx-chain-logger-go" - "sync" ) var ( @@ -30,47 +32,52 @@ var ( elasticIndexer.TransactionsIndex, elasticIndexer.BlockIndex, elasticIndexer.MiniblocksIndex, elasticIndexer.RatingIndex, elasticIndexer.RoundsIndex, elasticIndexer.ValidatorsIndex, elasticIndexer.AccountsIndex, elasticIndexer.AccountsHistoryIndex, elasticIndexer.ReceiptsIndex, elasticIndexer.ScResultsIndex, elasticIndexer.AccountsESDTHistoryIndex, elasticIndexer.AccountsESDTIndex, elasticIndexer.EpochInfoIndex, elasticIndexer.SCDeploysIndex, elasticIndexer.TokensIndex, elasticIndexer.TagsIndex, elasticIndexer.LogsIndex, elasticIndexer.DelegatorsIndex, elasticIndexer.OperationsIndex, - elasticIndexer.ESDTsIndex, elasticIndexer.ValuesIndex, elasticIndexer.EventsIndex, + elasticIndexer.ESDTsIndex, elasticIndexer.ValuesIndex, elasticIndexer.EventsIndex, elasticIndexer.ExecutionResultsIndex, } ) -const versionStr = "indexer-version" +const ( + versionStr = "indexer-version" + minNumWritesInParallel = 1 +) // ArgElasticProcessor holds all dependencies required by the elasticProcessor in order to create // new instances type ArgElasticProcessor struct { - BulkRequestMaxSize int - UseKibana bool - ImportDB bool - EnabledIndexes map[string]struct{} - TransactionsProc DBTransactionsHandler - AccountsProc DBAccountHandler - BlockProc DBBlockHandler - MiniblocksProc DBMiniblocksHandler - StatisticsProc DBStatisticsHandler - ValidatorsProc DBValidatorsHandler - DBClient DatabaseClientHandler - LogsAndEventsProc DBLogsAndEventsHandler - OperationsProc OperationsHandler - MappingsHandler TemplatesAndPoliciesHandler - Version string + NumWritesInParallel int + BulkRequestMaxSize int + UseKibana bool + ImportDB bool + EnabledIndexes map[string]struct{} + TransactionsProc DBTransactionsHandler + AccountsProc DBAccountHandler + BlockProc DBBlockHandler + MiniblocksProc DBMiniblocksHandler + StatisticsProc DBStatisticsHandler + ValidatorsProc DBValidatorsHandler + DBClient DatabaseClientHandler + LogsAndEventsProc DBLogsAndEventsHandler + OperationsProc OperationsHandler + MappingsHandler TemplatesAndPoliciesHandler + Version string } type elasticProcessor struct { - bulkRequestMaxSize int - importDB bool - enabledIndexes map[string]struct{} - mutex sync.RWMutex - elasticClient DatabaseClientHandler - accountsProc DBAccountHandler - blockProc DBBlockHandler - transactionsProc DBTransactionsHandler - miniblocksProc DBMiniblocksHandler - statisticsProc DBStatisticsHandler - validatorsProc DBValidatorsHandler - logsAndEventsProc DBLogsAndEventsHandler - operationsProc OperationsHandler - mappingsHandler TemplatesAndPoliciesHandler + numWritesInParallel int + bulkRequestMaxSize int + importDB bool + enabledIndexes map[string]struct{} + mutex sync.RWMutex + elasticClient DatabaseClientHandler + accountsProc DBAccountHandler + blockProc DBBlockHandler + transactionsProc DBTransactionsHandler + miniblocksProc DBMiniblocksHandler + statisticsProc DBStatisticsHandler + validatorsProc DBValidatorsHandler + logsAndEventsProc DBLogsAndEventsHandler + operationsProc OperationsHandler + mappingsHandler TemplatesAndPoliciesHandler } // NewElasticProcessor handles Elasticsearch operations such as initialization, adding, modifying or removing data @@ -79,20 +86,26 @@ func NewElasticProcessor(arguments *ArgElasticProcessor) (*elasticProcessor, err if err != nil { return nil, err } + numWritesInParallel := arguments.NumWritesInParallel + if numWritesInParallel < minNumWritesInParallel { + log.Warn("elasticProcessor.NewElasticProcessor: provided num writes in parallel is invalid, the minimum value will be set", "min value", minNumWritesInParallel) + numWritesInParallel = minNumWritesInParallel + } ei := &elasticProcessor{ - elasticClient: arguments.DBClient, - enabledIndexes: arguments.EnabledIndexes, - accountsProc: arguments.AccountsProc, - blockProc: arguments.BlockProc, - miniblocksProc: arguments.MiniblocksProc, - transactionsProc: arguments.TransactionsProc, - statisticsProc: arguments.StatisticsProc, - validatorsProc: arguments.ValidatorsProc, - logsAndEventsProc: arguments.LogsAndEventsProc, - operationsProc: arguments.OperationsProc, - bulkRequestMaxSize: arguments.BulkRequestMaxSize, - mappingsHandler: arguments.MappingsHandler, + elasticClient: arguments.DBClient, + enabledIndexes: arguments.EnabledIndexes, + accountsProc: arguments.AccountsProc, + blockProc: arguments.BlockProc, + miniblocksProc: arguments.MiniblocksProc, + transactionsProc: arguments.TransactionsProc, + statisticsProc: arguments.StatisticsProc, + validatorsProc: arguments.ValidatorsProc, + logsAndEventsProc: arguments.LogsAndEventsProc, + operationsProc: arguments.OperationsProc, + bulkRequestMaxSize: arguments.BulkRequestMaxSize, + mappingsHandler: arguments.MappingsHandler, + numWritesInParallel: numWritesInParallel, } err = ei.init() @@ -112,11 +125,6 @@ func (ei *elasticProcessor) init() error { return err } - err = ei.createOpenDistroTemplates(indexTemplates) - if err != nil { - return err - } - err = ei.createIndexTemplates(indexTemplates) if err != nil { return err @@ -177,35 +185,6 @@ func (ei *elasticProcessor) indexVersion(version string) error { return ei.elasticClient.DoBulkRequest(context.Background(), buffSlice.Buffers()[0], "") } -// nolint -func (ei *elasticProcessor) createIndexPolicies(indexPolicies map[string]*bytes.Buffer) error { - indexesPolicies := []string{elasticIndexer.TransactionsPolicy, elasticIndexer.BlockPolicy, elasticIndexer.MiniblocksPolicy, elasticIndexer.RatingPolicy, elasticIndexer.RoundsPolicy, elasticIndexer.ValidatorsPolicy, - elasticIndexer.AccountsPolicy, elasticIndexer.AccountsESDTPolicy, elasticIndexer.AccountsHistoryPolicy, elasticIndexer.AccountsESDTHistoryPolicy, elasticIndexer.AccountsESDTIndex, elasticIndexer.ReceiptsPolicy, elasticIndexer.ScResultsPolicy} - for _, indexPolicyName := range indexesPolicies { - indexPolicy := getTemplateByName(indexPolicyName, indexPolicies) - if indexPolicy != nil { - err := ei.elasticClient.CheckAndCreatePolicy(indexPolicyName, indexPolicy) - if err != nil { - return err - } - } - } - - return nil -} - -func (ei *elasticProcessor) createOpenDistroTemplates(indexTemplates map[string]*bytes.Buffer) error { - opendistroTemplate := getTemplateByName(elasticIndexer.OpenDistroIndex, indexTemplates) - if opendistroTemplate != nil { - err := ei.elasticClient.CheckAndCreateTemplate(elasticIndexer.OpenDistroIndex, opendistroTemplate) - if err != nil { - return err - } - } - - return nil -} - func (ei *elasticProcessor) createIndexTemplates(indexTemplates map[string]*bytes.Buffer) error { for _, index := range indexes { indexTemplate := getTemplateByName(index, indexTemplates) @@ -254,17 +233,13 @@ func getTemplateByName(templateName string, templateList map[string]*bytes.Buffe // SaveHeader will prepare and save information about a header in elasticsearch server func (ei *elasticProcessor) SaveHeader(outportBlockWithHeader *outport.OutportBlockWithHeader) error { - if !ei.isIndexEnabled(elasticIndexer.BlockIndex) { - return nil - } - - elasticBlock, err := ei.blockProc.PrepareBlockForDB(outportBlockWithHeader) + blockResults, err := ei.blockProc.PrepareBlockForDB(outportBlockWithHeader) if err != nil { return err } buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - err = ei.blockProc.SerializeBlock(elasticBlock, buffSlice, elasticIndexer.BlockIndex) + err = ei.indexBlock(blockResults.Block, buffSlice) if err != nil { return err } @@ -274,9 +249,30 @@ func (ei *elasticProcessor) SaveHeader(outportBlockWithHeader *outport.OutportBl return err } + err = ei.indexExecutionResults(blockResults.ExecutionResults, buffSlice) + if err != nil { + return err + } + return ei.doBulkRequests("", buffSlice.Buffers(), outportBlockWithHeader.ShardID) } +func (ei *elasticProcessor) indexBlock(esBlock *data.Block, buffSlice *data.BufferSlice) error { + if !ei.isIndexEnabled(elasticIndexer.BlockIndex) { + return nil + } + + return ei.blockProc.SerializeBlock(esBlock, buffSlice, elasticIndexer.BlockIndex) +} + +func (ei *elasticProcessor) indexExecutionResults(executionResults []*data.ExecutionResult, buffSlice *data.BufferSlice) error { + if !ei.isIndexEnabled(elasticIndexer.ExecutionResultsIndex) { + return nil + } + + return ei.blockProc.SerializeExecutionResults(executionResults, buffSlice, elasticIndexer.ExecutionResultsIndex) +} + func (ei *elasticProcessor) indexEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.EpochInfoIndex) || header.GetShardID() != core.MetachainShardId { @@ -294,19 +290,38 @@ func (ei *elasticProcessor) RemoveHeader(header coreData.HeaderHandler) error { } ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) - return ei.elasticClient.DoQueryRemove( + err = ei.elasticClient.DoQueryRemove( ctxWithValue, elasticIndexer.BlockIndex, converters.PrepareHashesForQueryRemove([]string{hex.EncodeToString(headerHash)}), ) + if err != nil { + return err + } + + if len(header.GetExecutionResultsHandlers()) == 0 { + return nil + } + + executionResultsHashes := make([]string, 0) + for _, executonResult := range header.GetExecutionResultsHandlers() { + executionResultsHashes = append(executionResultsHashes, hex.EncodeToString(executonResult.GetHeaderHash())) + } + + return ei.elasticClient.DoQueryRemove( + ctxWithValue, + elasticIndexer.ExecutionResultsIndex, + converters.PrepareHashesForQueryRemove(executionResultsHashes), + ) } // RemoveMiniblocks will remove all miniblocks that are in header from elasticsearch server -func (ei *elasticProcessor) RemoveMiniblocks(header coreData.HeaderHandler, body *block.Body) error { - encodedMiniblocksHashes := ei.miniblocksProc.GetMiniblocksHashesHexEncoded(header, body) - if len(encodedMiniblocksHashes) == 0 { - return nil +func (ei *elasticProcessor) RemoveMiniblocks(header coreData.HeaderHandler) error { + headerData := &data.HeaderData{ + ShardID: header.GetShardID(), + MiniBlockHeaders: header.GetMiniBlockHeaderHandlers(), } + encodedMiniblocksHashes := ei.miniblocksProc.GetMiniblocksHashesHexEncoded(headerData) ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.RemoveTopic, header.GetShardID())) return ei.elasticClient.DoQueryRemove( @@ -318,7 +333,15 @@ func (ei *elasticProcessor) RemoveMiniblocks(header coreData.HeaderHandler, body // RemoveTransactions will remove transaction that are in miniblock from the elasticsearch server func (ei *elasticProcessor) RemoveTransactions(header coreData.HeaderHandler, body *block.Body, timestampMs uint64) error { - encodedTxsHashes, encodedScrsHashes := ei.transactionsProc.GetHexEncodedHashesForRemove(header, body) + headerData := &data.HeaderData{ + Timestamp: header.GetTimeStamp(), + TimestampMs: timestampMs, + Round: header.GetRound(), + ShardID: header.GetShardID(), + Epoch: header.GetEpoch(), + MiniBlockHeaders: header.GetMiniBlockHeaderHandlers(), + } + encodedTxsHashes, encodedScrsHashes := ei.transactionsProc.GetHexEncodedHashesForRemove(headerData, body) shardID := header.GetShardID() err := ei.removeIfHashesNotEmpty(elasticIndexer.TransactionsIndex, encodedTxsHashes, shardID) @@ -399,37 +422,117 @@ func (ei *elasticProcessor) removeFromIndexByTimestampAndShardID(shardID uint32, } // SaveMiniblocks will prepare and save information about miniblocks in elasticsearch server -func (ei *elasticProcessor) SaveMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMS uint64) error { +func (ei *elasticProcessor) SaveMiniblocks(obh *outport.OutportBlockWithHeader) error { if !ei.isIndexEnabled(elasticIndexer.MiniblocksIndex) { return nil } - mbs := ei.miniblocksProc.PrepareDBMiniblocks(header, miniBlocks, timestampMS) - if len(mbs) == 0 { - return nil + buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) + + headerData := &data.HeaderData{ + Timestamp: converters.MillisecondsToSeconds(obh.BlockData.TimestampMs), + TimestampMs: obh.BlockData.TimestampMs, + Round: obh.Header.GetRound(), + ShardID: obh.Header.GetShardID(), + Epoch: obh.Header.GetEpoch(), + MiniBlockHeaders: obh.Header.GetMiniBlockHeaderHandlers(), + NumberOfShards: obh.NumberOfShards, + HeaderHash: obh.BlockData.HeaderHash, } + miniBlocks := append(obh.BlockData.Body.MiniBlocks, obh.BlockData.IntraShardMiniBlocks...) + mbs := ei.miniblocksProc.PrepareDBMiniblocks(headerData, miniBlocks) + ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, buffSlice, elasticIndexer.MiniblocksIndex, headerData.ShardID) + + for _, executionResult := range obh.Header.GetExecutionResultsHandlers() { + executionResulData, found := obh.BlockData.Results[hex.EncodeToString(executionResult.GetHeaderHash())] + if !found { + log.Warn("elasticProcessor.SaveTransactions: cannot find execution result data", "hash", executionResult.GetHeaderHash()) + continue + } - buffSlice := data.NewBufferSlice(ei.bulkRequestMaxSize) - ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, buffSlice, elasticIndexer.MiniblocksIndex, header.GetShardID()) + headerData = &data.HeaderData{ + Timestamp: converters.MillisecondsToSeconds(executionResulData.TimestampMs), + TimestampMs: executionResulData.TimestampMs, + Round: executionResult.GetHeaderRound(), + ShardID: obh.Header.GetShardID(), + NumberOfShards: obh.NumberOfShards, + Epoch: executionResult.GetHeaderEpoch(), + MiniBlockHeaders: converters.GetMiniBlocksHeaderHandlersFromExecResult(executionResult), + HeaderHash: executionResult.GetHeaderHash(), + } + + miniBlocks = append(executionResulData.Body.MiniBlocks, executionResulData.IntraShardMiniBlocks...) + mbs = ei.miniblocksProc.PrepareDBMiniblocks(headerData, miniBlocks) + ei.miniblocksProc.SerializeBulkMiniBlocks(mbs, buffSlice, elasticIndexer.MiniblocksIndex, headerData.ShardID) + + } - return ei.doBulkRequests("", buffSlice.Buffers(), header.GetShardID()) + return ei.doBulkRequests("", buffSlice.Buffers(), headerData.ShardID) } // SaveTransactions will prepare and save information about a transactions in elasticsearch server func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader) error { - headerTimestamp := obh.Header.GetTimeStamp() - miniBlocks := append(obh.BlockData.Body.MiniBlocks, obh.BlockData.IntraShardMiniBlocks...) - preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(miniBlocks, obh.Header, obh.TransactionPool, ei.isImportDB(), obh.NumberOfShards, obh.BlockData.TimestampMs) - logsData := ei.logsAndEventsProc.ExtractDataFromLogs(obh.TransactionPool.Logs, preparedResults, headerTimestamp, obh.Header.GetShardID(), obh.NumberOfShards, obh.BlockData.TimestampMs) + headerData := &data.HeaderData{ + Timestamp: converters.MillisecondsToSeconds(obh.BlockData.TimestampMs), + TimestampMs: obh.BlockData.TimestampMs, + Round: obh.Header.GetRound(), + ShardID: obh.Header.GetShardID(), + Epoch: obh.Header.GetEpoch(), + MiniBlockHeaders: obh.Header.GetMiniBlockHeaderHandlers(), + NumberOfShards: obh.NumberOfShards, + } buffers := data.NewBufferSlice(ei.bulkRequestMaxSize) - err := ei.indexTransactions(preparedResults.Transactions, logsData.TxHashStatusInfo, obh.Header, buffers) + err := ei.prepareAndSaveTransactionsData(headerData, miniBlocks, obh.TransactionPool, obh.AlteredAccounts, buffers) + if err != nil { + return err + } + + for _, executionResult := range obh.Header.GetExecutionResultsHandlers() { + executionResulData, found := obh.BlockData.Results[hex.EncodeToString(executionResult.GetHeaderHash())] + if !found { + log.Warn("elasticProcessor.SaveTransactions: cannot find execution result data", "hash", executionResult.GetHeaderHash()) + continue + } + + headerData = &data.HeaderData{ + Timestamp: converters.MillisecondsToSeconds(executionResulData.TimestampMs), + TimestampMs: executionResulData.TimestampMs, + Round: executionResult.GetHeaderRound(), + ShardID: obh.Header.GetShardID(), + NumberOfShards: obh.NumberOfShards, + Epoch: executionResult.GetHeaderEpoch(), + MiniBlockHeaders: converters.GetMiniBlocksHeaderHandlersFromExecResult(executionResult), + } + + miniBlocks = append(executionResulData.Body.MiniBlocks, executionResulData.IntraShardMiniBlocks...) + err = ei.prepareAndSaveTransactionsData(headerData, miniBlocks, executionResulData.TransactionPool, executionResulData.AlteredAccounts, buffers) + if err != nil { + return err + } + } + + return ei.doBulkRequests("", buffers.Buffers(), headerData.ShardID) +} + +func (ei *elasticProcessor) prepareAndSaveTransactionsData( + headerData *data.HeaderData, + miniBlocks []*block.MiniBlock, + pool *outport.TransactionPool, + alteredAccounts map[string]*alteredAccount.AlteredAccount, + buffers *data.BufferSlice, +) error { + + preparedResults := ei.transactionsProc.PrepareTransactionsForDatabase(miniBlocks, headerData, pool, ei.isImportDB()) + logsData := ei.logsAndEventsProc.ExtractDataFromLogs(pool.Logs, preparedResults, headerData.ShardID, headerData.NumberOfShards, headerData.TimestampMs) + + err := ei.indexTransactions(preparedResults.Transactions, logsData.TxHashStatusInfo, headerData.ShardID, buffers) if err != nil { return err } - err = ei.prepareAndIndexOperations(preparedResults.Transactions, logsData.TxHashStatusInfo, obh.Header, preparedResults.ScResults, buffers, ei.isImportDB()) + err = ei.prepareAndIndexOperations(preparedResults.Transactions, logsData.TxHashStatusInfo, headerData.ShardID, preparedResults.ScResults, buffers, ei.isImportDB()) if err != nil { return err } @@ -439,7 +542,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexNFTCreateInfo(logsData.Tokens, obh.AlteredAccounts, buffers, obh.ShardID) + err = ei.indexNFTCreateInfo(logsData.Tokens, alteredAccounts, buffers, headerData.ShardID) if err != nil { return err } @@ -465,7 +568,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader } tagsCount := tags.NewTagsCount() - err = ei.indexAlteredAccounts(logsData.NFTsDataUpdates, obh.AlteredAccounts, buffers, tagsCount, obh.Header.GetShardID(), obh.BlockData.TimestampMs) + err = ei.indexAlteredAccounts(logsData.NFTsDataUpdates, alteredAccounts, buffers, tagsCount, headerData.ShardID, headerData.TimestampMs) if err != nil { return err } @@ -475,7 +578,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexTokens(logsData.TokensInfo, logsData.NFTsDataUpdates, buffers, obh.ShardID) + err = ei.indexTokens(logsData.TokensInfo, logsData.NFTsDataUpdates, buffers, headerData.ShardID) if err != nil { return err } @@ -485,7 +588,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexNFTBurnInfo(logsData.TokensSupply, buffers, obh.ShardID) + err = ei.indexNFTBurnInfo(logsData.TokensSupply, buffers, headerData.ShardID) if err != nil { return err } @@ -499,12 +602,7 @@ func (ei *elasticProcessor) SaveTransactions(obh *outport.OutportBlockWithHeader return err } - err = ei.indexScDeploys(logsData.ScDeploys, logsData.ChangeOwnerOperations, buffers) - if err != nil { - return err - } - - return ei.doBulkRequests("", buffers.Buffers(), obh.ShardID) + return ei.indexScDeploys(logsData.ScDeploys, logsData.ChangeOwnerOperations, buffers) } func (ei *elasticProcessor) prepareAndIndexRolesData(tokenRolesAndProperties *tokeninfo.TokenRolesAndProperties, buffSlice *data.BufferSlice, index string) error { @@ -565,18 +663,18 @@ func (ei *elasticProcessor) indexScDeploys(deployData map[string]*data.ScDeployI return ei.logsAndEventsProc.SerializeChangeOwnerOperations(changeOwnerOperation, buffSlice, elasticIndexer.SCDeploysIndex) } -func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, header coreData.HeaderHandler, bytesBuff *data.BufferSlice) error { +func (ei *elasticProcessor) indexTransactions(txs []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, shardID uint32, bytesBuff *data.BufferSlice) error { if !ei.isIndexEnabled(elasticIndexer.TransactionsIndex) { return nil } - return ei.transactionsProc.SerializeTransactions(txs, txHashStatusInfo, header.GetShardID(), bytesBuff, elasticIndexer.TransactionsIndex) + return ei.transactionsProc.SerializeTransactions(txs, txHashStatusInfo, shardID, bytesBuff, elasticIndexer.TransactionsIndex) } func (ei *elasticProcessor) prepareAndIndexOperations( txs []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, - header coreData.HeaderHandler, + shardID uint32, scrs []*data.ScResult, buffSlice *data.BufferSlice, isImportDB bool, @@ -585,14 +683,14 @@ func (ei *elasticProcessor) prepareAndIndexOperations( return nil } - processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB, header.GetShardID()) + processedTxs, processedSCRs := ei.operationsProc.ProcessTransactionsAndSCRs(txs, scrs, isImportDB, shardID) - err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatusInfo, header.GetShardID(), buffSlice, elasticIndexer.OperationsIndex) + err := ei.transactionsProc.SerializeTransactions(processedTxs, txHashStatusInfo, shardID, buffSlice, elasticIndexer.OperationsIndex) if err != nil { return err } - return ei.operationsProc.SerializeSCRs(processedSCRs, buffSlice, elasticIndexer.OperationsIndex, header.GetShardID()) + return ei.operationsProc.SerializeSCRs(processedSCRs, buffSlice, elasticIndexer.OperationsIndex, shardID) } // SaveValidatorsRating will save validators rating @@ -836,13 +934,41 @@ func (ei *elasticProcessor) isIndexEnabled(index string) bool { } func (ei *elasticProcessor) doBulkRequests(index string, buffSlice []*bytes.Buffer, shardID uint32) error { - var err error - for idx := range buffSlice { - ctxWithValue := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, shardID)) - err = ei.elasticClient.DoBulkRequest(ctxWithValue, buffSlice[idx], index) - if err != nil { - return err + jobs := make(chan *bytes.Buffer) + errCh := make(chan error, len(buffSlice)) + + var wg sync.WaitGroup + + for i := 0; i < ei.numWritesInParallel; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for buf := range jobs { + ctx := context.WithValue(context.Background(), request.ContextKey, request.ExtendTopicWithShardID(request.BulkTopic, shardID)) + err := ei.elasticClient.DoBulkRequest(ctx, buf, index) + if err != nil { + errCh <- err + } + } + }() + } + + go func() { + for _, buf := range buffSlice { + jobs <- buf } + close(jobs) + }() + + wg.Wait() + close(errCh) + + var errs []error + for err := range errCh { + errs = append(errs, err) + } + if len(errs) > 0 { + return errors.Join(errs...) } return nil diff --git a/process/elasticproc/elasticProcessor_test.go b/process/elasticproc/elasticProcessor_test.go index e045e129..63def74a 100644 --- a/process/elasticproc/elasticProcessor_test.go +++ b/process/elasticproc/elasticProcessor_test.go @@ -4,11 +4,13 @@ import ( "bytes" "encoding/hex" "errors" + "fmt" + "strconv" "strings" + "sync" "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -31,15 +33,16 @@ import ( func newElasticsearchProcessor(elasticsearchWriter DatabaseClientHandler, arguments *ArgElasticProcessor) *elasticProcessor { return &elasticProcessor{ - elasticClient: elasticsearchWriter, - enabledIndexes: arguments.EnabledIndexes, - blockProc: arguments.BlockProc, - transactionsProc: arguments.TransactionsProc, - miniblocksProc: arguments.MiniblocksProc, - accountsProc: arguments.AccountsProc, - validatorsProc: arguments.ValidatorsProc, - statisticsProc: arguments.StatisticsProc, - logsAndEventsProc: arguments.LogsAndEventsProc, + elasticClient: elasticsearchWriter, + enabledIndexes: arguments.EnabledIndexes, + blockProc: arguments.BlockProc, + transactionsProc: arguments.TransactionsProc, + miniblocksProc: arguments.MiniblocksProc, + accountsProc: arguments.AccountsProc, + validatorsProc: arguments.ValidatorsProc, + statisticsProc: arguments.StatisticsProc, + logsAndEventsProc: arguments.LogsAndEventsProc, + numWritesInParallel: 1, } } @@ -80,15 +83,16 @@ func createMockElasticProcessorArgs() *ArgElasticProcessor { EnabledIndexes: map[string]struct{}{ dataindexer.BlockIndex: {}, dataindexer.TransactionsIndex: {}, dataindexer.MiniblocksIndex: {}, dataindexer.ValidatorsIndex: {}, dataindexer.RoundsIndex: {}, dataindexer.AccountsIndex: {}, dataindexer.RatingIndex: {}, dataindexer.AccountsHistoryIndex: {}, }, - ValidatorsProc: vp, - StatisticsProc: statistics.NewStatisticsProcessor(), - TransactionsProc: &mock.DBTransactionProcessorStub{}, - MiniblocksProc: mp, - AccountsProc: acp, - BlockProc: bp, - LogsAndEventsProc: lp, - OperationsProc: op, - MappingsHandler: templatesAndPolicies.NewTemplatesAndPolicyReader(), + ValidatorsProc: vp, + StatisticsProc: statistics.NewStatisticsProcessor(), + TransactionsProc: &mock.DBTransactionProcessorStub{}, + MiniblocksProc: mp, + AccountsProc: acp, + BlockProc: bp, + LogsAndEventsProc: lp, + OperationsProc: op, + MappingsHandler: templatesAndPolicies.NewTemplatesAndPolicyReader(), + NumWritesInParallel: 1, } } @@ -268,9 +272,6 @@ func TestElasticProcessor_RemoveHeader(t *testing.T) { func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { called := false - mb1 := &dataBlock.MiniBlock{ - Type: dataBlock.PeerBlock, - } mb2 := &dataBlock.MiniBlock{ ReceiverShardID: 0, SenderShardID: 1, @@ -279,10 +280,6 @@ func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { ReceiverShardID: 1, SenderShardID: 1, } // should be removed - mb4 := &dataBlock.MiniBlock{ - ReceiverShardID: 1, - SenderShardID: 0, - } // should NOT be removed args := createMockElasticProcessorArgs() @@ -308,25 +305,27 @@ func TestElasticProcessor_RemoveMiniblocks(t *testing.T) { ShardID: 1, MiniBlockHeaders: []dataBlock.MiniBlockHeader{ { + Type: dataBlock.PeerBlock, Hash: []byte("hash1"), }, { - Hash: []byte("hash2"), + Hash: mbHash2, + ReceiverShardID: 0, + SenderShardID: 1, }, { - Hash: []byte("hash3"), + Hash: mbHash3, + ReceiverShardID: 1, + SenderShardID: 1, }, { - Hash: []byte("hash4"), + Hash: []byte("hash4"), + ReceiverShardID: 1, + SenderShardID: 0, }, }, } - body := &dataBlock.Body{ - MiniBlocks: dataBlock.MiniBlockSlice{ - mb1, mb2, mb3, mb4, - }, - } - err = elasticProc.RemoveMiniblocks(header, body) + err = elasticProc.RemoveMiniblocks(header) require.Nil(t, err) require.True(t, called) } @@ -344,7 +343,8 @@ func TestElasticseachDatabaseSaveHeader_RequestError(t *testing.T) { elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) err := elasticDatabase.SaveHeader(createEmptyOutportBlockWithHeader()) - require.Equal(t, localErr, err) + require.NotNil(t, err) + require.Equal(t, localErr.Error(), err.Error()) } func TestElasticseachSaveTransactions(t *testing.T) { @@ -378,7 +378,8 @@ func TestElasticseachSaveTransactions(t *testing.T) { elasticDatabase := newElasticsearchProcessor(dbWriter, arguments) err := elasticDatabase.SaveTransactions(outportBlock) - require.Equal(t, localErr, err) + require.NotNil(t, err) + require.Equal(t, localErr.Error(), err.Error()) } func TestElasticProcessor_SaveValidatorsRating(t *testing.T) { @@ -399,7 +400,8 @@ func TestElasticProcessor_SaveValidatorsRating(t *testing.T) { Epoch: 1, ValidatorsRatingInfo: []*outport.ValidatorRatingInfo{{}}, }) - require.Equal(t, localErr, err) + require.NotNil(t, err) + require.Equal(t, localErr.Error(), err.Error()) } func TestElasticProcessor_SaveMiniblocks(t *testing.T) { @@ -422,8 +424,14 @@ func TestElasticProcessor_SaveMiniblocks(t *testing.T) { body := &dataBlock.Body{MiniBlocks: dataBlock.MiniBlockSlice{ {SenderShardID: 0, ReceiverShardID: 1}, }} - err := elasticProc.SaveMiniblocks(header, body.MiniBlocks, 0) - require.Equal(t, localErr, err) + + ob := createEmptyOutportBlockWithHeader() + ob.Header = header + ob.BlockData.Body = body + + err := elasticProc.SaveMiniblocks(ob) + require.NotNil(t, err) + require.Equal(t, localErr.Error(), err.Error()) } func TestElasticsearch_saveShardValidatorsPubKeys_RequestError(t *testing.T) { @@ -446,7 +454,8 @@ func TestElasticsearch_saveShardValidatorsPubKeys_RequestError(t *testing.T) { shardID: {Keys: valPubKeys}, }, }) - require.Equal(t, localErr, err) + require.NotNil(t, err) + require.Equal(t, localErr.Error(), err.Error()) } func TestElasticsearch_saveRoundInfoRequestError(t *testing.T) { @@ -550,7 +559,7 @@ func TestElasticProcessor_SaveTransactionNoDataShouldNotDoRequest(t *testing.T) called := false arguments := createMockElasticProcessorArgs() arguments.TransactionsProc = &mock.DBTransactionProcessorStub{ - PrepareTransactionsForDatabaseCalled: func(mbs []*dataBlock.MiniBlock, header coreData.HeaderHandler, pool *outport.TransactionPool) *data.PreparedResults { + PrepareTransactionsForDatabaseCalled: func(mbs []*dataBlock.MiniBlock, headerData *data.HeaderData, pool *outport.TransactionPool) *data.PreparedResults { return &data.PreparedResults{ Transactions: nil, ScResults: nil, @@ -603,3 +612,126 @@ func TestElasticProcessor_IndexAlteredAccounts(t *testing.T) { require.Nil(t, err) require.True(t, called) } + +func TestElasticProcessor_DoBulkRequests_SuccessfulParallelProcessing(t *testing.T) { + arguments := createMockElasticProcessorArgs() + arguments.NumWritesInParallel = 3 + + processedBuffers := make(map[string]bool) + processedBuffersMutex := &sync.Mutex{} + + arguments.DBClient = &mock.DatabaseWriterStub{ + DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { + processedBuffersMutex.Lock() + processedBuffers[buff.String()] = true + processedBuffersMutex.Unlock() + return nil + }, + } + + elasticProc, _ := NewElasticProcessor(arguments) + + // Create 5 test buffers + buffers := make([]*bytes.Buffer, 5) + for i := 0; i < 5; i++ { + buffers[i] = bytes.NewBufferString(fmt.Sprintf("test-buffer-%d", i)) + } + + err := elasticProc.doBulkRequests("test-index", buffers, 0) + require.Nil(t, err) + + // Verify all buffers were processed + processedBuffersMutex.Lock() + defer processedBuffersMutex.Unlock() + require.Equal(t, 5, len(processedBuffers)) + for i := 0; i < 5; i++ { + require.True(t, processedBuffers[fmt.Sprintf("test-buffer-%d", i)]) + } +} + +func TestElasticProcessor_DoBulkRequests_WorkerCancellationOnError(t *testing.T) { + arguments := createMockElasticProcessorArgs() + arguments.NumWritesInParallel = 3 + + processedBuffers := make(map[string]bool) + processedBuffersMutex := &sync.Mutex{} + expectedErr := errors.New("expected error") + + arguments.DBClient = &mock.DatabaseWriterStub{ + DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { + if buff.String() == "test-buffer-1" { + return expectedErr + } + + processedBuffersMutex.Lock() + processedBuffers[buff.String()] = true + processedBuffersMutex.Unlock() + + return nil + }, + } + + elasticProc, _ := NewElasticProcessor(arguments) + + buffers := make([]*bytes.Buffer, 5) + for i := 0; i < 5; i++ { + buffers[i] = bytes.NewBufferString(fmt.Sprintf("test-buffer-%d", i)) + } + + err := elasticProc.doBulkRequests("test-index", buffers, 0) + require.NotNil(t, err) + require.Equal(t, "expected error", err.Error()) + + processedBuffersMutex.Lock() + defer processedBuffersMutex.Unlock() + require.True(t, len(processedBuffers) > 0) + require.True(t, processedBuffers["test-buffer-0"]) + require.False(t, processedBuffers["test-buffer-1"]) + require.True(t, processedBuffers["test-buffer-2"]) + require.True(t, processedBuffers["test-buffer-3"]) + require.True(t, processedBuffers["test-buffer-4"]) +} + +func TestElasticProcessor_DoBulkRequests_DatabaseError(t *testing.T) { + arguments := createMockElasticProcessorArgs() + arguments.NumWritesInParallel = 20 + + expectedErr := errors.New("database connection error") + processedBuffers := make(map[string]int) + processedBuffersMutex := &sync.Mutex{} + + arguments.DBClient = &mock.DatabaseWriterStub{ + DoBulkRequestCalled: func(buff *bytes.Buffer, index string) error { + split := strings.Split(buff.String(), "-") + bufferNum, _ := strconv.Atoi(split[2]) + if bufferNum%3 == 0 { + return expectedErr + } + + processedBuffersMutex.Lock() + processedBuffers[buff.String()]++ + processedBuffersMutex.Unlock() + return nil + }, + } + + elasticProc, _ := NewElasticProcessor(arguments) + buffers := make([]*bytes.Buffer, 6) + for i := 0; i < 6; i++ { + buffers[i] = bytes.NewBufferString(fmt.Sprintf("test-buffer-%d", i+1)) + } + + err := elasticProc.doBulkRequests("test-index", buffers, 0) + require.NotNil(t, err) + require.Equal(t, "database connection error\ndatabase connection error", err.Error()) + + processedBuffersMutex.Lock() + defer processedBuffersMutex.Unlock() + require.True(t, len(processedBuffers) > 0) + require.Equal(t, 1, processedBuffers["test-buffer-1"]) + require.Equal(t, 1, processedBuffers["test-buffer-2"]) + require.Equal(t, 0, processedBuffers["test-buffer-3"]) + require.Equal(t, 1, processedBuffers["test-buffer-4"]) + require.Equal(t, 1, processedBuffers["test-buffer-5"]) + require.Equal(t, 0, processedBuffers["test-buffer-6"]) +} diff --git a/process/elasticproc/factory/elasticProcessorFactory.go b/process/elasticproc/factory/elasticProcessorFactory.go index 3db4cb16..778e152f 100644 --- a/process/elasticproc/factory/elasticProcessorFactory.go +++ b/process/elasticproc/factory/elasticProcessorFactory.go @@ -30,6 +30,7 @@ type ArgElasticProcessorFactory struct { Version string Denomination int BulkRequestMaxSize int + NumWritesInParallel int UseKibana bool ImportDB bool EnableEpochsConfig config.EnableEpochsConfig @@ -105,21 +106,22 @@ func CreateElasticProcessor(arguments ArgElasticProcessorFactory) (dataindexer.E } args := &elasticproc.ArgElasticProcessor{ - BulkRequestMaxSize: arguments.BulkRequestMaxSize, - TransactionsProc: txsProc, - AccountsProc: accountsProc, - BlockProc: blockProcHandler, - MiniblocksProc: miniblocksProc, - ValidatorsProc: validatorsProc, - StatisticsProc: generalInfoProc, - LogsAndEventsProc: logsAndEventsProc, - DBClient: arguments.DBClient, - EnabledIndexes: enabledIndexesMap, - UseKibana: arguments.UseKibana, - OperationsProc: operationsProc, - ImportDB: arguments.ImportDB, - Version: arguments.Version, - MappingsHandler: templatesAndPoliciesReader, + BulkRequestMaxSize: arguments.BulkRequestMaxSize, + TransactionsProc: txsProc, + AccountsProc: accountsProc, + BlockProc: blockProcHandler, + MiniblocksProc: miniblocksProc, + ValidatorsProc: validatorsProc, + StatisticsProc: generalInfoProc, + LogsAndEventsProc: logsAndEventsProc, + DBClient: arguments.DBClient, + EnabledIndexes: enabledIndexesMap, + UseKibana: arguments.UseKibana, + OperationsProc: operationsProc, + ImportDB: arguments.ImportDB, + Version: arguments.Version, + MappingsHandler: templatesAndPoliciesReader, + NumWritesInParallel: arguments.NumWritesInParallel, } return elasticproc.NewElasticProcessor(args) diff --git a/process/elasticproc/interface.go b/process/elasticproc/interface.go index 4caa6af8..8861159f 100644 --- a/process/elasticproc/interface.go +++ b/process/elasticproc/interface.go @@ -48,24 +48,23 @@ type DBAccountHandler interface { // DBBlockHandler defines the actions that a block handler should do type DBBlockHandler interface { - PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.Block, error) + PrepareBlockForDB(obh *outport.OutportBlockWithHeader) (*data.PreparedBlockResults, error) ComputeHeaderHash(header coreData.HeaderHandler) ([]byte, error) SerializeEpochInfoData(header coreData.HeaderHandler, buffSlice *data.BufferSlice, index string) error SerializeBlock(elasticBlock *data.Block, buffSlice *data.BufferSlice, index string) error + SerializeExecutionResults(executionResults []*data.ExecutionResult, buffSlice *data.BufferSlice, index string) error } // DBTransactionsHandler defines the actions that a transactions handler should do type DBTransactionsHandler interface { PrepareTransactionsForDatabase( miniBlocks []*block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, pool *outport.TransactionPool, isImportDB bool, - numOfShards uint32, - timestampMS uint64, ) *data.PreparedResults - GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) + GetHexEncodedHashesForRemove(headerData *data.HeaderData, body *block.Body) ([]string, []string) SerializeReceipts(receipts []*data.Receipt, buffSlice *data.BufferSlice, index string) error SerializeTransactions(transactions []*data.Transaction, txHashStatusInfo map[string]*outport.StatusInfo, selfShardID uint32, buffSlice *data.BufferSlice, index string) error @@ -75,8 +74,8 @@ type DBTransactionsHandler interface { // DBMiniblocksHandler defines the actions that a miniblocks handler should do type DBMiniblocksHandler interface { - PrepareDBMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMS uint64) []*data.Miniblock - GetMiniblocksHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string + PrepareDBMiniblocks(headerData *data.HeaderData, miniBlocks []*block.MiniBlock) []*data.Miniblock + GetMiniblocksHashesHexEncoded(headerData *data.HeaderData) []string SerializeBulkMiniBlocks(bulkMbs []*data.Miniblock, buffSlice *data.BufferSlice, index string, shardID uint32) } @@ -97,7 +96,6 @@ type DBLogsAndEventsHandler interface { ExtractDataFromLogs( logsAndEvents []*outport.LogData, preparedResults *data.PreparedResults, - timestamp uint64, shardID uint32, numOfShards uint32, timestampMs uint64, diff --git a/process/elasticproc/logsevents/delegatorsProcessor.go b/process/elasticproc/logsevents/delegatorsProcessor.go index d4d171e7..522746a1 100644 --- a/process/elasticproc/logsevents/delegatorsProcessor.go +++ b/process/elasticproc/logsevents/delegatorsProcessor.go @@ -2,11 +2,12 @@ package logsevents import ( "encoding/hex" + "math/big" + "strconv" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-es-indexer-go/data" indexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" - "math/big" - "strconv" ) const ( diff --git a/process/elasticproc/logsevents/esdtIssueProcessor.go b/process/elasticproc/logsevents/esdtIssueProcessor.go index e221ad27..1cdff52b 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor.go @@ -1,9 +1,10 @@ package logsevents import ( + "math/big" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-es-indexer-go/data" - "math/big" ) const ( diff --git a/process/elasticproc/logsevents/esdtIssueProcessor_test.go b/process/elasticproc/logsevents/esdtIssueProcessor_test.go index 214a295a..c3fc59c6 100644 --- a/process/elasticproc/logsevents/esdtIssueProcessor_test.go +++ b/process/elasticproc/logsevents/esdtIssueProcessor_test.go @@ -1,12 +1,13 @@ package logsevents import ( + "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" - "testing" ) func TestIssueESDTProcessor(t *testing.T) { diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor.go b/process/elasticproc/logsevents/logsAndEventsProcessor.go index 5b4ffa88..1c440521 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor.go @@ -3,6 +3,7 @@ package logsevents import ( "encoding/hex" "fmt" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" coreData "github.com/multiversx/mx-chain-core-go/data" @@ -90,12 +91,11 @@ func createEventsProcessors(args ArgsLogsAndEventsProcessor) []eventsProcessor { func (lep *logsAndEventsProcessor) ExtractDataFromLogs( logsAndEvents []*outport.LogData, preparedResults *data.PreparedResults, - timestamp uint64, shardID uint32, numOfShards uint32, timestampMs uint64, ) *data.PreparedLogsResults { - lgData := newLogsData(timestamp, preparedResults.Transactions, preparedResults.ScResults, timestampMs) + lgData := newLogsData(preparedResults.Transactions, preparedResults.ScResults, timestampMs) for _, txLog := range logsAndEvents { if txLog == nil { continue @@ -116,7 +116,7 @@ func (lep *logsAndEventsProcessor) ExtractDataFromLogs( } } - dbLogs, dbEvents := lep.prepareLogsForDB(lgData, logsAndEvents, timestamp, shardID, timestampMs) + dbLogs, dbEvents := lep.prepareLogsForDB(lgData, logsAndEvents, shardID) return &data.PreparedLogsResults{ Tokens: lgData.tokens, @@ -192,9 +192,7 @@ func (lep *logsAndEventsProcessor) processEvent(lgData *logsData, logHashHexEnco func (lep *logsAndEventsProcessor) prepareLogsForDB( lgData *logsData, logsAndEvents []*outport.LogData, - timestamp uint64, shardID uint32, - timestampMs uint64, ) ([]*data.Logs, []*data.LogEvent) { logs := make([]*data.Logs, 0, len(logsAndEvents)) events := make([]*data.LogEvent, 0) @@ -204,7 +202,7 @@ func (lep *logsAndEventsProcessor) prepareLogsForDB( continue } - dbLog, logEvents := lep.prepareLog(lgData, txLog.TxHash, txLog.Log, timestamp, shardID, timestampMs) + dbLog, logEvents := lep.prepareLog(lgData, txLog.TxHash, txLog.Log, shardID) logs = append(logs, dbLog) events = append(events, logEvents...) @@ -217,9 +215,7 @@ func (lep *logsAndEventsProcessor) prepareLog( lgData *logsData, logHashHex string, eventLogs *transaction.Log, - timestamp uint64, shardID uint32, - timestampMs uint64, ) (*data.Logs, []*data.LogEvent) { originalTxHash := lep.getOriginalTxHash(lgData, logHashHex) encodedAddr := lep.pubKeyConverter.SilentEncode(eventLogs.GetAddress(), log) @@ -228,9 +224,9 @@ func (lep *logsAndEventsProcessor) prepareLog( ID: logHashHex, OriginalTxHash: originalTxHash, Address: encodedAddr, - Timestamp: timestamp, + Timestamp: lgData.timestamp, Events: make([]*data.Event, 0, len(eventLogs.Events)), - TimestampMs: timestampMs, + TimestampMs: lgData.timestampMs, } dbEvents := make([]*data.LogEvent, 0, len(eventLogs.Events)) @@ -250,13 +246,13 @@ func (lep *logsAndEventsProcessor) prepareLog( logsDB.Events = append(logsDB.Events, logEvent) executionOrder := lep.getExecutionOrder(lgData, logHashHex) - dbEvents = append(dbEvents, lep.prepareLogEvent(logsDB, logEvent, shardID, executionOrder, timestampMs)) + dbEvents = append(dbEvents, lep.prepareLogEvent(logsDB, logEvent, shardID, executionOrder)) } return logsDB, dbEvents } -func (lep *logsAndEventsProcessor) prepareLogEvent(dbLog *data.Logs, event *data.Event, shardID uint32, execOrder int, timestampMs uint64) *data.LogEvent { +func (lep *logsAndEventsProcessor) prepareLogEvent(dbLog *data.Logs, event *data.Event, shardID uint32, execOrder int) *data.LogEvent { dbEvent := &data.LogEvent{ UUID: converters.GenerateBase64UUID(), TxHash: dbLog.ID, @@ -272,7 +268,7 @@ func (lep *logsAndEventsProcessor) prepareLogEvent(dbLog *data.Logs, event *data OriginalTxHash: dbLog.OriginalTxHash, Timestamp: dbLog.Timestamp, ID: fmt.Sprintf(eventIDFormat, dbLog.ID, shardID, event.Order), - TimestampMs: timestampMs, + TimestampMs: dbLog.TimestampMs, } return dbEvent diff --git a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go index 0020af6f..c3ffeb78 100644 --- a/process/elasticproc/logsevents/logsAndEventsProcessor_test.go +++ b/process/elasticproc/logsevents/logsAndEventsProcessor_test.go @@ -2,6 +2,9 @@ package logsevents import ( "encoding/hex" + "math/big" + "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/transaction" @@ -10,8 +13,6 @@ import ( elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" "github.com/stretchr/testify/require" - "math/big" - "testing" ) func createMockArgs() ArgsLogsAndEventsProcessor { @@ -155,7 +156,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsAndPutInAltered(t *testing.T) args.BalanceConverter = balanceConverter proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEvents, res, 1000, core.MetachainShardId, 3, 1000000) + resLogs := proc.ExtractDataFromLogs(logsAndEvents, res, core.MetachainShardId, 3, 1000000) require.NotNil(t, resLogs.Tokens) require.True(t, res.Transactions[0].HasOperations) require.True(t, res.ScResults[0].HasOperations) @@ -237,7 +238,7 @@ func TestLogsAndEventsProcessor_PrepareLogsForDB(t *testing.T) { Hash: "747848617368", OriginalTxHash: "orignalHash", }, - }}, 1234, 0, 3, 1234000) + }}, 0, 3, 1234000) result.DBLogs[0].UUID = "" @@ -294,7 +295,7 @@ func TestLogsAndEventsProcessor_ExtractDataFromLogsNFTBurn(t *testing.T) { args.BalanceConverter = balanceConverter proc, _ := NewLogsAndEventsProcessor(args) - resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 1000, 2, 3, 1000000) + resLogs := proc.ExtractDataFromLogs(logsAndEventsSlice, res, 2, 3, 1000000) require.Equal(t, 1, resLogs.TokensSupply.Len()) tokensSupply := resLogs.TokensSupply.GetAll() @@ -339,7 +340,7 @@ func TestPrepareLogsAndEvents_LogEvents(t *testing.T) { Hash: "747848617368", OriginalTxHash: "originalHash", }, - }}, 1234, 1, 3, 1234000) + }}, 1, 3, 1234000) results.DBEvents[0].UUID = "" results.DBEvents[1].UUID = "" diff --git a/process/elasticproc/logsevents/logsData.go b/process/elasticproc/logsevents/logsData.go index 0b45f903..ed30ef6a 100644 --- a/process/elasticproc/logsevents/logsData.go +++ b/process/elasticproc/logsevents/logsData.go @@ -23,7 +23,6 @@ type logsData struct { } func newLogsData( - timestamp uint64, txs []*data.Transaction, scrs []*data.ScResult, timestampMs uint64, @@ -34,7 +33,7 @@ func newLogsData( ld.scrsMap = converters.ConvertScrsSliceIntoMap(scrs) ld.tokens = data.NewTokensInfo() ld.tokensSupply = data.NewTokensInfo() - ld.timestamp = timestamp + ld.timestamp = converters.MillisecondsToSeconds(timestampMs) ld.scDeploys = make(map[string]*data.ScDeployInfo) ld.tokensInfo = make([]*data.TokenInfo, 0) ld.delegators = make(map[string]*data.Delegator) diff --git a/process/elasticproc/logsevents/nftsProcessor.go b/process/elasticproc/logsevents/nftsProcessor.go index 01618c5c..3bdd58cf 100644 --- a/process/elasticproc/logsevents/nftsProcessor.go +++ b/process/elasticproc/logsevents/nftsProcessor.go @@ -1,6 +1,8 @@ package logsevents import ( + "math/big" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/sharding" coreData "github.com/multiversx/mx-chain-core-go/data" @@ -10,7 +12,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" logger "github.com/multiversx/mx-chain-logger-go" - "math/big" ) const ( diff --git a/process/elasticproc/logsevents/nftsProcessor_test.go b/process/elasticproc/logsevents/nftsProcessor_test.go index 30189b09..cda904dc 100644 --- a/process/elasticproc/logsevents/nftsProcessor_test.go +++ b/process/elasticproc/logsevents/nftsProcessor_test.go @@ -3,14 +3,15 @@ package logsevents import ( "encoding/hex" "encoding/json" + "math/big" + "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/esdt" "github.com/multiversx/mx-chain-core-go/data/transaction" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" - "math/big" - "testing" ) func TestNftsProcessor_processLogAndEventsNFTs(t *testing.T) { diff --git a/process/elasticproc/logsevents/serialize_test.go b/process/elasticproc/logsevents/serialize_test.go index 30c84362..eb8280f4 100644 --- a/process/elasticproc/logsevents/serialize_test.go +++ b/process/elasticproc/logsevents/serialize_test.go @@ -1,12 +1,13 @@ package logsevents import ( + "math/big" + "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/stretchr/testify/require" - "math/big" - "testing" ) func TestLogsAndEventsProcessor_SerializeLogs(t *testing.T) { diff --git a/process/elasticproc/miniblocks/miniblocksProcessor.go b/process/elasticproc/miniblocks/miniblocksProcessor.go index 451991cd..3025d391 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor.go @@ -2,9 +2,9 @@ package miniblocks import ( "encoding/hex" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -39,14 +39,8 @@ func NewMiniblocksProcessor( } // PrepareDBMiniblocks will prepare miniblocks -func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler, miniBlocks []*block.MiniBlock, timestampMS uint64) []*data.Miniblock { - headerHash, err := mp.calculateHash(header) - if err != nil { - log.Warn("indexer: could not calculate header hash", "error", err) - return nil - } - - selfShard := header.GetShardID() +func (mp *miniblocksProcessor) PrepareDBMiniblocks(headerData *data.HeaderData, miniBlocks []*block.MiniBlock) []*data.Miniblock { + selfShard := headerData.ShardID dbMiniblocks := make([]*data.Miniblock, 0) for mbIndex, miniBlock := range miniBlocks { if miniBlock.ReceiverShardID == core.AllShardId && selfShard != core.MetachainShardId { @@ -54,7 +48,7 @@ func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler continue } - dbMiniBlock, errPrepareMiniBlock := mp.prepareMiniblockForDB(mbIndex, miniBlock, header, headerHash, timestampMS) + dbMiniBlock, errPrepareMiniBlock := mp.prepareMiniblockForDB(mbIndex, miniBlock, headerData) if errPrepareMiniBlock != nil { log.Warn("miniblocksProcessor.PrepareDBMiniBlocks cannot prepare miniblock", "error", errPrepareMiniBlock) continue @@ -69,9 +63,7 @@ func (mp *miniblocksProcessor) PrepareDBMiniblocks(header coreData.HeaderHandler func (mp *miniblocksProcessor) prepareMiniblockForDB( mbIndex int, miniblock *block.MiniBlock, - header coreData.HeaderHandler, - headerHash []byte, - timestampMS uint64, + headerData *data.HeaderData, ) (*data.Miniblock, error) { mbHash, err := mp.calculateHash(miniblock) if err != nil { @@ -85,21 +77,21 @@ func (mp *miniblocksProcessor) prepareMiniblockForDB( SenderShardID: miniblock.SenderShardID, ReceiverShardID: miniblock.ReceiverShardID, Type: miniblock.Type.String(), - Timestamp: header.GetTimeStamp(), - TimestampMs: timestampMS, + Timestamp: headerData.Timestamp, + TimestampMs: headerData.TimestampMs, Reserved: miniblock.Reserved, } - encodedHeaderHash := hex.EncodeToString(headerHash) + encodedHeaderHash := hex.EncodeToString(headerData.HeaderHash) isIntraShard := dbMiniblock.SenderShardID == dbMiniblock.ReceiverShardID - isCrossOnSource := !isIntraShard && dbMiniblock.SenderShardID == header.GetShardID() + isCrossOnSource := !isIntraShard && dbMiniblock.SenderShardID == headerData.ShardID if isIntraShard || isCrossOnSource { - mp.setFieldsMBIntraShardAndCrossFromMe(mbIndex, header, encodedHeaderHash, dbMiniblock, isIntraShard) + mp.setFieldsMBIntraShardAndCrossFromMe(mbIndex, headerData, encodedHeaderHash, dbMiniblock, isIntraShard) return dbMiniblock, nil } - processingType, _ := mp.computeProcessingTypeAndConstructionState(mbIndex, header) + processingType, _ := mp.computeProcessingTypeAndConstructionState(mbIndex, headerData) dbMiniblock.ProcessingTypeOnDestination = processingType dbMiniblock.ReceiverBlockHash = encodedHeaderHash @@ -108,12 +100,12 @@ func (mp *miniblocksProcessor) prepareMiniblockForDB( func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( mbIndex int, - header coreData.HeaderHandler, + headerData *data.HeaderData, headerHash string, dbMiniblock *data.Miniblock, isIntraShard bool, ) { - processingType, constructionState := mp.computeProcessingTypeAndConstructionState(mbIndex, header) + processingType, constructionState := mp.computeProcessingTypeAndConstructionState(mbIndex, headerData) dbMiniblock.ProcessingTypeOnSource = processingType switch { @@ -133,8 +125,8 @@ func (mp *miniblocksProcessor) setFieldsMBIntraShardAndCrossFromMe( } } -func (mp *miniblocksProcessor) computeProcessingTypeAndConstructionState(mbIndex int, header coreData.HeaderHandler) (string, int32) { - miniblockHeaders := header.GetMiniBlockHeaderHandlers() +func (mp *miniblocksProcessor) computeProcessingTypeAndConstructionState(mbIndex int, headerData *data.HeaderData) (string, int32) { + miniblockHeaders := headerData.MiniBlockHeaders if len(miniblockHeaders) <= mbIndex { return block.Normal.String(), int32(block.Final) } @@ -153,31 +145,25 @@ func (mp *miniblocksProcessor) computeProcessingTypeAndConstructionState(mbIndex } // GetMiniblocksHashesHexEncoded will compute miniblocks hashes in a hexadecimal encoding -func (mp *miniblocksProcessor) GetMiniblocksHashesHexEncoded(header coreData.HeaderHandler, body *block.Body) []string { - if body == nil || len(header.GetMiniBlockHeadersHashes()) == 0 { +func (mp *miniblocksProcessor) GetMiniblocksHashesHexEncoded(headerData *data.HeaderData) []string { + if len(headerData.MiniBlockHeaders) == 0 { return nil } encodedMiniblocksHashes := make([]string, 0) - selfShardID := header.GetShardID() - for _, miniblock := range body.MiniBlocks { - if miniblock.Type == block.PeerBlock { + selfShardID := headerData.ShardID + for _, miniblock := range headerData.MiniBlockHeaders { + if miniblock.GetTypeInt32() == int32(block.PeerBlock) { continue } - isCrossShard := miniblock.ReceiverShardID != miniblock.SenderShardID - isCrossShardOnDestination := selfShardID == miniblock.ReceiverShardID && isCrossShard + isCrossShard := miniblock.GetReceiverShardID() != miniblock.GetSenderShardID() + isCrossShardOnDestination := selfShardID == miniblock.GetReceiverShardID() && isCrossShard if isCrossShardOnDestination { continue } - miniblockHash, err := mp.calculateHash(miniblock) - if err != nil { - log.Debug("miniblocksProcessor.GetMiniblocksHashesHexEncoded cannot calculate miniblock hash", - "error", err) - continue - } - encodedMiniblocksHashes = append(encodedMiniblocksHashes, hex.EncodeToString(miniblockHash)) + encodedMiniblocksHashes = append(encodedMiniblocksHashes, hex.EncodeToString(miniblock.GetHash())) } return encodedMiniblocksHashes diff --git a/process/elasticproc/miniblocks/miniblocksProcessor_test.go b/process/elasticproc/miniblocks/miniblocksProcessor_test.go index 180b16fe..251e6925 100644 --- a/process/elasticproc/miniblocks/miniblocksProcessor_test.go +++ b/process/elasticproc/miniblocks/miniblocksProcessor_test.go @@ -1,8 +1,10 @@ package miniblocks import ( + "encoding/hex" "testing" + coreData "github.com/multiversx/mx-chain-core-go/data" dataBlock "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/hashing" "github.com/multiversx/mx-chain-core-go/marshal" @@ -47,7 +49,6 @@ func TestMiniblocksProcessor_PrepareDBMiniblocks(t *testing.T) { mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - header := &dataBlock.Header{} body := &dataBlock.Body{ MiniBlocks: []*dataBlock.MiniBlock{ { @@ -65,7 +66,11 @@ func TestMiniblocksProcessor_PrepareDBMiniblocks(t *testing.T) { }, } - miniblocks := mp.PrepareDBMiniblocks(header, body.MiniBlocks, 1234000) + headerData := &data.HeaderData{ + TimestampMs: 1234000, + } + + miniblocks := mp.PrepareDBMiniblocks(headerData, body.MiniBlocks) require.Len(t, miniblocks, 3) } @@ -81,17 +86,6 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { mbhrBytes, _ := marshalizer.Marshal(mbhr) - header := &dataBlock.Header{ - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - { - Reserved: []byte{0}, - }, - { - Reserved: mbhrBytes, - }, - }, - TimeStamp: 1234, - } miniBlocks := []*dataBlock.MiniBlock{ { SenderShardID: 0, @@ -107,7 +101,22 @@ func TestMiniblocksProcessor_PrepareScheduledMB(t *testing.T) { }, } - miniblocks := mp.PrepareDBMiniblocks(header, miniBlocks, 1234000) + headerHash, _ := hex.DecodeString("64ad61aaddb68f8d0b38ceda3b2b1f76a6749a0e848ed9e95bdaff46b4e73423") + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + MiniBlockHeaders: []coreData.MiniBlockHeaderHandler{ + &dataBlock.MiniBlockHeader{ + Reserved: []byte{0}, + }, + &dataBlock.MiniBlockHeader{ + Reserved: mbhrBytes, + }, + }, + HeaderHash: headerHash, + } + + miniblocks := mp.PrepareDBMiniblocks(headerData, miniBlocks) require.Len(t, miniblocks, 3) require.Equal(t, dataBlock.Scheduled.String(), miniblocks[1].ProcessingTypeOnSource) @@ -130,34 +139,37 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncoded(t *testing.T) { mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - header := &dataBlock.Header{ - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - {}, {}, {}, - }, + expectedHashes := []string{ + "0796d34e8d443fd31bf4d9ec4051421b4d5d0e8c1db9ff942d6f4dc3a9ca2803", + "4cc379ab1f0aef6602e85a0a7ffabb5bc9a2ba646dc0fd720028e06527bf873f", + "8748c4677b01f7db984004fa8465afbf55feaab4b573174c8c0afa282941b9e4", } - body := &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - { + + decode := func(e string) []byte { + d, _ := hex.DecodeString(e) + return d + } + headerData := &data.HeaderData{ + ShardID: 0, + MiniBlockHeaders: []coreData.MiniBlockHeaderHandler{ + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[0]), SenderShardID: 0, ReceiverShardID: 1, }, - { + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[1]), SenderShardID: 0, ReceiverShardID: 2, }, - { + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[2]), SenderShardID: 0, ReceiverShardID: 0, }, }, } - - expectedHashes := []string{ - "0796d34e8d443fd31bf4d9ec4051421b4d5d0e8c1db9ff942d6f4dc3a9ca2803", - "4cc379ab1f0aef6602e85a0a7ffabb5bc9a2ba646dc0fd720028e06527bf873f", - "8748c4677b01f7db984004fa8465afbf55feaab4b573174c8c0afa282941b9e4", - } - miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(header, body) + miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(headerData) require.Equal(t, expectedHashes, miniblocksHashes) } @@ -166,38 +178,41 @@ func TestMiniblocksProcessor_GetMiniblocksHashesHexEncodedImportDBMode(t *testin mp, _ := NewMiniblocksProcessor(&mock.HasherMock{}, &mock.MarshalizerMock{}) - header := &dataBlock.Header{ - MiniBlockHeaders: []dataBlock.MiniBlockHeader{ - {}, {}, {}, - }, - ShardID: 1, + expectedHashes := []string{ + "11a1bb4065e16a2e93b2b5ac5957b7b69f1cfba7579b170b24f30dab2d3162e0", + "68e9a4360489ab7a6e99f92e05d1a3f06a982b7b157ac23fdf39f2392bf88e15", + "d1fd2a5c95c8899ebbaad035b6b0f77c5103b3aacfe630b1a7c51468d682bb1b", } - body := &dataBlock.Body{ - MiniBlocks: []*dataBlock.MiniBlock{ - { + decode := func(e string) []byte { + d, _ := hex.DecodeString(e) + return d + } + + headerData := &data.HeaderData{ + ShardID: 1, + MiniBlockHeaders: []coreData.MiniBlockHeaderHandler{ + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[0]), SenderShardID: 1, ReceiverShardID: 2, }, - { + &dataBlock.MiniBlockHeader{ SenderShardID: 0, ReceiverShardID: 1, }, - { + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[1]), SenderShardID: 1, ReceiverShardID: 0, }, - { + &dataBlock.MiniBlockHeader{ + Hash: decode(expectedHashes[2]), SenderShardID: 1, ReceiverShardID: 1, }, }, } - expectedHashes := []string{ - "11a1bb4065e16a2e93b2b5ac5957b7b69f1cfba7579b170b24f30dab2d3162e0", - "68e9a4360489ab7a6e99f92e05d1a3f06a982b7b157ac23fdf39f2392bf88e15", - "d1fd2a5c95c8899ebbaad035b6b0f77c5103b3aacfe630b1a7c51468d682bb1b", - } - miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(header, body) + miniblocksHashes := mp.GetMiniblocksHashesHexEncoded(headerData) require.Equal(t, expectedHashes, miniblocksHashes) } diff --git a/process/elasticproc/templatesAndPolicies/reader.go b/process/elasticproc/templatesAndPolicies/reader.go index 35db2efb..93c1d5aa 100644 --- a/process/elasticproc/templatesAndPolicies/reader.go +++ b/process/elasticproc/templatesAndPolicies/reader.go @@ -43,6 +43,7 @@ func (tr *templatesAndPolicyReader) GetElasticTemplatesAndPolicies() (map[string indexTemplates[indexer.ESDTsIndex] = indices.ESDTs.ToBuffer() indexTemplates[indexer.ValuesIndex] = indices.Values.ToBuffer() indexTemplates[indexer.EventsIndex] = indices.Events.ToBuffer() + indexTemplates[indexer.ExecutionResultsIndex] = indices.ExecutionResults.ToBuffer() return indexTemplates, indexPolicies, nil } diff --git a/process/elasticproc/templatesAndPolicies/reader_test.go b/process/elasticproc/templatesAndPolicies/reader_test.go index 6626311e..9c345615 100644 --- a/process/elasticproc/templatesAndPolicies/reader_test.go +++ b/process/elasticproc/templatesAndPolicies/reader_test.go @@ -14,5 +14,5 @@ func TestTemplatesAndPolicyReaderNoKibana_GetElasticTemplatesAndPolicies(t *test templates, policies, err := reader.GetElasticTemplatesAndPolicies() require.Nil(t, err) require.Len(t, policies, 0) - require.Len(t, templates, 23) + require.Len(t, templates, 24) } diff --git a/process/elasticproc/transactions/checkers.go b/process/elasticproc/transactions/checkers.go index 7d9fcaf7..624b9156 100644 --- a/process/elasticproc/transactions/checkers.go +++ b/process/elasticproc/transactions/checkers.go @@ -8,7 +8,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/check" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" elasticIndexer "github.com/multiversx/mx-chain-es-indexer-go/process/dataindexer" @@ -50,13 +49,8 @@ func areESDTValuesOK(values []string) bool { } func checkPrepareTransactionForDatabaseArguments( - header coreData.HeaderHandler, pool *outport.TransactionPool, ) error { - - if check.IfNil(header) { - return elasticIndexer.ErrNilHeaderHandler - } if pool == nil { return elasticIndexer.ErrNilPool } diff --git a/process/elasticproc/transactions/checkers_test.go b/process/elasticproc/transactions/checkers_test.go index 06ebe9e9..77534225 100644 --- a/process/elasticproc/transactions/checkers_test.go +++ b/process/elasticproc/transactions/checkers_test.go @@ -6,8 +6,6 @@ import ( "testing" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" - "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" @@ -72,20 +70,13 @@ func TestCheckTxsProcessorArg(t *testing.T) { tests := []struct { name string - args func() (header coreData.HeaderHandler, pool *outport.TransactionPool) + args func() (pool *outport.TransactionPool) exErr error }{ - { - name: "NilHeaderHandler", - args: func() (header coreData.HeaderHandler, pool *outport.TransactionPool) { - return nil, &outport.TransactionPool{} - }, - exErr: elasticIndexer.ErrNilHeaderHandler, - }, { name: "NilPool", - args: func() (header coreData.HeaderHandler, pool *outport.TransactionPool) { - return &block.Header{}, nil + args: func() (pool *outport.TransactionPool) { + return nil }, exErr: elasticIndexer.ErrNilPool, }, diff --git a/process/elasticproc/transactions/smartContractResultsProcessor.go b/process/elasticproc/transactions/smartContractResultsProcessor.go index 6ef4a904..4b6b7bde 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor.go @@ -5,7 +5,6 @@ import ( "strconv" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" @@ -44,10 +43,8 @@ func newSmartContractResultsProcessor( func (proc *smartContractResultsProcessor) processSCRs( miniBlocks []*block.MiniBlock, - header coreData.HeaderHandler, + headerData *indexerData.HeaderData, scrs map[string]*outport.SCRInfo, - numOfShards uint32, - timestampMs uint64, ) []*indexerData.ScResult { allSCRs := make([]*indexerData.ScResult, 0, len(scrs)) @@ -58,14 +55,13 @@ func (proc *smartContractResultsProcessor) processSCRs( continue } - indexerSCRs := proc.processSCRsFromMiniblock(header, mb, workingSCRSMap, numOfShards, timestampMs) + indexerSCRs := proc.processSCRsFromMiniblock(headerData, mb, workingSCRSMap) allSCRs = append(allSCRs, indexerSCRs...) } - selfShardID := header.GetShardID() for scrHashHex, noMBScrInfo := range workingSCRSMap { - indexerScr := proc.prepareSmartContractResult(scrHashHex, nil, noMBScrInfo, header, selfShardID, selfShardID, numOfShards, timestampMs) + indexerScr := proc.prepareSmartContractResult(scrHashHex, nil, noMBScrInfo, headerData, headerData.ShardID, headerData.ShardID) allSCRs = append(allSCRs, indexerScr) } @@ -74,11 +70,9 @@ func (proc *smartContractResultsProcessor) processSCRs( } func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( - header coreData.HeaderHandler, + headerData *indexerData.HeaderData, mb *block.MiniBlock, scrs map[string]*outport.SCRInfo, - numOfShards uint32, - timestampMs uint64, ) []*indexerData.ScResult { mbHash, err := core.CalculateHash(proc.marshalizer, proc.hasher, mb) if err != nil { @@ -97,7 +91,7 @@ func (proc *smartContractResultsProcessor) processSCRsFromMiniblock( continue } - indexerSCR := proc.prepareSmartContractResult(hex.EncodeToString(scrHash), mbHash, scrInfo, header, mb.SenderShardID, mb.ReceiverShardID, numOfShards, timestampMs) + indexerSCR := proc.prepareSmartContractResult(hex.EncodeToString(scrHash), mbHash, scrInfo, headerData, mb.SenderShardID, mb.ReceiverShardID) indexerSCRs = append(indexerSCRs, indexerSCR) delete(scrs, scrHashHex) @@ -110,11 +104,9 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( scrHashHex string, mbHash []byte, scrInfo *outport.SCRInfo, - header coreData.HeaderHandler, + headerData *indexerData.HeaderData, senderShard uint32, receiverShard uint32, - numOfShards uint32, - timestampMs uint64, ) *indexerData.ScResult { scr := scrInfo.SmartContractResult hexEncodedMBHash := "" @@ -136,7 +128,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( originalSenderAddr = proc.pubKeyConverter.SilentEncode(scr.OriginalSender, log) } - res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, numOfShards) + res := proc.dataFieldParser.Parse(scr.Data, scr.SndAddr, scr.RcvAddr, headerData.NumberOfShards) senderAddr := proc.pubKeyConverter.SilentEncode(scr.SndAddr, log) receiverAddr := proc.pubKeyConverter.SilentEncode(scr.RcvAddr, log) @@ -159,7 +151,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( esdtValues = res.ESDTValues } - isRelayed := res.IsRelayed && header.GetEpoch() < proc.relayedV1V2DisableEpoch + isRelayed := res.IsRelayed && headerData.Epoch < proc.relayedV1V2DisableEpoch feeInfo := getFeeInfo(scrInfo) return &indexerData.ScResult{ @@ -181,7 +173,7 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( CallType: strconv.Itoa(int(scr.CallType)), CodeMetadata: scr.CodeMetadata, ReturnMessage: string(scr.ReturnMessage), - Timestamp: header.GetTimeStamp(), + Timestamp: headerData.Timestamp, SenderAddressBytes: scr.SndAddr, SenderShard: senderShard, ReceiverShard: receiverShard, @@ -199,8 +191,8 @@ func (proc *smartContractResultsProcessor) prepareSmartContractResult( GasRefunded: feeInfo.GasRefunded, ExecutionOrder: int(scrInfo.ExecutionOrder), UUID: converters.GenerateBase64UUID(), - Epoch: header.GetEpoch(), - TimestampMs: timestampMs, + Epoch: headerData.Epoch, + TimestampMs: headerData.TimestampMs, } } diff --git a/process/elasticproc/transactions/smartContractResultsProcessor_test.go b/process/elasticproc/transactions/smartContractResultsProcessor_test.go index 73c5a99e..e586983e 100644 --- a/process/elasticproc/transactions/smartContractResultsProcessor_test.go +++ b/process/elasticproc/transactions/smartContractResultsProcessor_test.go @@ -5,7 +5,6 @@ import ( "math/big" "testing" - "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/smartContractResult" "github.com/multiversx/mx-chain-es-indexer-go/data" @@ -56,10 +55,13 @@ func TestPrepareSmartContractResult(t *testing.T) { }, } - header := &block.Header{TimeStamp: 100} - mbHash := []byte("hash") - scRes := scrsProc.prepareSmartContractResult(scHash, mbHash, scrInfo, header, 0, 1, 3, 100000) + headerData := &data.HeaderData{ + Timestamp: 100, + TimestampMs: 100000, + NumberOfShards: 3, + } + scRes := scrsProc.prepareSmartContractResult(scHash, mbHash, scrInfo, headerData, 0, 1) scRes.UUID = "" senderAddr, err := pubKeyConverter.Encode(sndAddr) diff --git a/process/elasticproc/transactions/transactionDBBuilder.go b/process/elasticproc/transactions/transactionDBBuilder.go index 1164eb81..ce925b2e 100644 --- a/process/elasticproc/transactions/transactionDBBuilder.go +++ b/process/elasticproc/transactions/transactionDBBuilder.go @@ -6,7 +6,6 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/sharding" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -41,15 +40,13 @@ func (dtb *dbTransactionBuilder) prepareTransaction( txHash []byte, mbHash []byte, mb *block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, txStatus string, - numOfShards uint32, - timestampMs uint64, ) *data.Transaction { tx := txInfo.Transaction isScCall := core.IsSmartContractAddress(tx.RcvAddr) - res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, numOfShards) + res := dtb.dataFieldParser.Parse(tx.Data, tx.SndAddr, tx.RcvAddr, headerData.NumberOfShards) receiverAddr := dtb.addressPubkeyConverter.SilentEncode(tx.RcvAddr, log) senderAddr := dtb.addressPubkeyConverter.SilentEncode(tx.SndAddr, log) @@ -57,7 +54,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( receiverShardID := mb.ReceiverShardID if mb.Type == block.InvalidBlock { - receiverShardID = sharding.ComputeShardID(tx.RcvAddr, numOfShards) + receiverShardID = sharding.ComputeShardID(tx.RcvAddr, headerData.NumberOfShards) } valueNum, err := dtb.balanceConverter.ConvertBigValueToFloat(tx.Value) @@ -98,7 +95,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, - Round: header.GetRound(), + Round: headerData.Round, Value: tx.Value.String(), Receiver: receiverAddr, Sender: senderAddr, @@ -109,7 +106,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( GasLimit: tx.GasLimit, Data: tx.Data, Signature: hex.EncodeToString(tx.Signature), - Timestamp: header.GetTimeStamp(), + Timestamp: headerData.Timestamp, Status: txStatus, GasUsed: feeInfo.GasUsed, InitialPaidFee: feeInfo.InitialPaidFee.String(), @@ -130,8 +127,8 @@ func (dtb *dbTransactionBuilder) prepareTransaction( RelayedAddr: relayedAddress, HadRefund: feeInfo.HadRefund, UUID: converters.GenerateBase64UUID(), - Epoch: header.GetEpoch(), - TimestampMs: timestampMs, + Epoch: headerData.Epoch, + TimestampMs: headerData.TimestampMs, } hasValidRelayer := len(eTx.RelayedAddr) == len(eTx.Sender) && len(eTx.RelayedAddr) > 0 @@ -142,7 +139,7 @@ func (dtb *dbTransactionBuilder) prepareTransaction( eTx.Tokens = converters.TruncateSliceElementsIfExceedsMaxLength(res.Tokens) eTx.ReceiversShardIDs = res.ReceiversShardID - relayedV1V2Enabled := header.GetEpoch() < dtb.relayedV1V2DisableEpoch + relayedV1V2Enabled := headerData.Epoch < dtb.relayedV1V2DisableEpoch eTx.IsRelayed = res.IsRelayed || isRelayedV3 if res.IsRelayed && !relayedV1V2Enabled { @@ -163,9 +160,8 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( txHash []byte, mbHash []byte, mb *block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, txStatus string, - timestampMs uint64, ) *data.Transaction { rTx := rTxInfo.Reward valueNum, err := dtb.balanceConverter.ConvertBigValueToFloat(rTx.Value) @@ -191,21 +187,20 @@ func (dtb *dbTransactionBuilder) prepareRewardTransaction( GasLimit: 0, Data: make([]byte, 0), Signature: "", - Timestamp: header.GetTimeStamp(), + Timestamp: headerData.Timestamp, Status: txStatus, Operation: rewardsOperation, ExecutionOrder: int(rTxInfo.ExecutionOrder), UUID: converters.GenerateBase64UUID(), - Epoch: header.GetEpoch(), - TimestampMs: timestampMs, + Epoch: headerData.Epoch, + TimestampMs: headerData.TimestampMs, } } func (dtb *dbTransactionBuilder) prepareReceipt( recHashHex string, rec *receipt.Receipt, - header coreData.HeaderHandler, - timestampMs uint64, + headerData *data.HeaderData, ) *data.Receipt { senderAddr := dtb.addressPubkeyConverter.SilentEncode(rec.SndAddr, log) @@ -215,7 +210,7 @@ func (dtb *dbTransactionBuilder) prepareReceipt( Sender: senderAddr, Data: string(rec.Data), TxHash: hex.EncodeToString(rec.TxHash), - Timestamp: header.GetTimeStamp(), - TimestampMs: timestampMs, + Timestamp: headerData.Timestamp, + TimestampMs: headerData.TimestampMs, } } diff --git a/process/elasticproc/transactions/transactionDBBuilder_test.go b/process/elasticproc/transactions/transactionDBBuilder_test.go index 74be210a..b1ae1f85 100644 --- a/process/elasticproc/transactions/transactionDBBuilder_test.go +++ b/process/elasticproc/transactions/transactionDBBuilder_test.go @@ -3,6 +3,9 @@ package transactions import ( "encoding/hex" "fmt" + "math/big" + "testing" + "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" @@ -12,8 +15,6 @@ import ( "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" "github.com/stretchr/testify/require" - "math/big" - "testing" ) func createCommonProcessor() dbTransactionBuilder { @@ -31,7 +32,11 @@ func TestGetMoveBalanceTransaction(t *testing.T) { txHash := []byte("txHash") mbHash := []byte("mbHash") mb := &block.MiniBlock{TxHashes: [][]byte{txHash}} - header := &block.Header{Nonce: 2, TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + NumberOfShards: 3, + } status := "Success" gasPrice := uint64(1000) gasLimit := uint64(1000) @@ -71,7 +76,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, - Round: header.Round, + Round: headerData.Round, Value: tx.Value.String(), ValueNum: 1e-15, Receiver: senderAddr, @@ -97,7 +102,7 @@ func TestGetMoveBalanceTransaction(t *testing.T) { TimestampMs: 1234000, } - dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3, 1234000) + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, headerData, status) dbTx.UUID = "" require.Equal(t, expectedTx, dbTx) } @@ -113,13 +118,16 @@ func TestGetTransactionByType_RewardTx(t *testing.T) { txHash := []byte("txHash") mbHash := []byte("mbHash") mb := &block.MiniBlock{TxHashes: [][]byte{txHash}} - header := &block.Header{Nonce: 2, TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + } status := "Success" rewardInfo := &outport.RewardInfo{ Reward: rwdTx, } - resultTx := cp.prepareRewardTransaction(rewardInfo, txHash, mbHash, mb, header, status, 1234000) + resultTx := cp.prepareRewardTransaction(rewardInfo, txHash, mbHash, mb, headerData, status) resultTx.UUID = "" expectedTx := &data.Transaction{ Hash: hex.EncodeToString(txHash), @@ -144,7 +152,11 @@ func TestRelayedV3Transaction(t *testing.T) { txHash := []byte("txHash") mbHash := []byte("mbHash") mb := &block.MiniBlock{TxHashes: [][]byte{txHash}, Type: block.InvalidBlock} - header := &block.Header{Nonce: 2, TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + NumberOfShards: 3, + } status := transaction.TxStatusInvalid.String() gasPrice := uint64(1000) gasLimit := uint64(1000) @@ -171,7 +183,7 @@ func TestRelayedV3Transaction(t *testing.T) { Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, - Round: header.Round, + Round: headerData.Round, Value: tx.Value.String(), ValueNum: 1e-15, Receiver: cp.addressPubkeyConverter.SilentEncode(tx.RcvAddr, log), @@ -209,7 +221,7 @@ func TestRelayedV3Transaction(t *testing.T) { ExecutionOrder: 0, } - dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3, 1234000) + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, headerData, status) dbTx.UUID = "" require.Equal(t, expectedTx, dbTx) } @@ -220,7 +232,11 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { txHash := []byte("txHash") mbHash := []byte("mbHash") mb := &block.MiniBlock{TxHashes: [][]byte{txHash}, Type: block.InvalidBlock} - header := &block.Header{Nonce: 2, TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + NumberOfShards: 3, + } status := transaction.TxStatusInvalid.String() gasPrice := uint64(1000) gasLimit := uint64(1000) @@ -245,7 +261,7 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { Hash: hex.EncodeToString(txHash), MBHash: hex.EncodeToString(mbHash), Nonce: tx.Nonce, - Round: header.Round, + Round: headerData.Round, Value: tx.Value.String(), ValueNum: 1e-15, Receiver: cp.addressPubkeyConverter.SilentEncode(tx.RcvAddr, log), @@ -281,7 +297,7 @@ func TestGetMoveBalanceTransactionInvalid(t *testing.T) { ExecutionOrder: 0, } - dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, header, status, 3, 1234000) + dbTx := cp.prepareTransaction(txInfo, txHash, mbHash, mb, headerData, status) dbTx.UUID = "" require.Equal(t, expectedTx, dbTx) } diff --git a/process/elasticproc/transactions/transactionsGrouper.go b/process/elasticproc/transactions/transactionsGrouper.go index 1c7e237a..86b7cab5 100644 --- a/process/elasticproc/transactions/transactionsGrouper.go +++ b/process/elasticproc/transactions/transactionsGrouper.go @@ -4,7 +4,6 @@ import ( "encoding/hex" "github.com/multiversx/mx-chain-core-go/core" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -39,11 +38,9 @@ func newTxsGrouper( func (tg *txsGrouper) groupNormalTxs( mbIndex int, mb *block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, txs map[string]*outport.TxInfo, isImportDB bool, - numOfShards uint32, - timestampMs uint64, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) @@ -52,16 +49,15 @@ func (tg *txsGrouper) groupNormalTxs( return nil, err } - selfShardID := header.GetShardID() - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) - mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) + executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, headerData) + mbStatus := computeStatus(headerData.ShardID, mb.ReceiverShardID) for _, txHash := range executedTxHashes { - dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, header, numOfShards, timestampMs) + dbTx, ok := tg.prepareNormalTxForDB(mbHash, mb, mbStatus, txHash, txs, headerData) if !ok { continue } - if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { + if tg.shouldIndex(mb.ReceiverShardID, isImportDB, headerData.ShardID) { transactions[string(txHash)] = dbTx } } @@ -69,8 +65,8 @@ func (tg *txsGrouper) groupNormalTxs( return transactions, nil } -func extractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, header coreData.HeaderHandler) [][]byte { - miniblockHeaders := header.GetMiniBlockHeaderHandlers() +func extractExecutedTxHashes(mbIndex int, mbTxHashes [][]byte, headerData *data.HeaderData) [][]byte { + miniblockHeaders := headerData.MiniBlockHeaders if len(miniblockHeaders) <= mbIndex { return mbTxHashes } @@ -96,16 +92,14 @@ func (tg *txsGrouper) prepareNormalTxForDB( mbStatus string, txHash []byte, txs map[string]*outport.TxInfo, - header coreData.HeaderHandler, - numOfShards uint32, - timestampMs uint64, + headerData *data.HeaderData, ) (*data.Transaction, bool) { txInfo, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } - dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, header, mbStatus, numOfShards, timestampMs) + dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, headerData, mbStatus) return dbTx, true } @@ -113,10 +107,9 @@ func (tg *txsGrouper) prepareNormalTxForDB( func (tg *txsGrouper) groupRewardsTxs( mbIndex int, mb *block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, txs map[string]*outport.RewardInfo, isImportDB bool, - timestampMs uint64, ) (map[string]*data.Transaction, error) { rewardsTxs := make(map[string]*data.Transaction) mbHash, err := core.CalculateHash(tg.marshalizer, tg.hasher, mb) @@ -124,16 +117,15 @@ func (tg *txsGrouper) groupRewardsTxs( return nil, err } - selfShardID := header.GetShardID() - mbStatus := computeStatus(selfShardID, mb.ReceiverShardID) - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + mbStatus := computeStatus(headerData.ShardID, mb.ReceiverShardID) + executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, headerData) for _, txHash := range executedTxHashes { - rewardDBTx, ok := tg.prepareRewardTxForDB(mbHash, mb, mbStatus, txHash, txs, header, timestampMs) + rewardDBTx, ok := tg.prepareRewardTxForDB(mbHash, mb, mbStatus, txHash, txs, headerData) if !ok { continue } - if tg.shouldIndex(mb.ReceiverShardID, isImportDB, selfShardID) { + if tg.shouldIndex(mb.ReceiverShardID, isImportDB, headerData.ShardID) { rewardsTxs[string(txHash)] = rewardDBTx } } @@ -147,15 +139,14 @@ func (tg *txsGrouper) prepareRewardTxForDB( mbStatus string, txHash []byte, txs map[string]*outport.RewardInfo, - header coreData.HeaderHandler, - timestampMs uint64, + headerData *data.HeaderData, ) (*data.Transaction, bool) { rtx, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } - dbTx := tg.txBuilder.prepareRewardTransaction(rtx, txHash, mbHash, mb, header, mbStatus, timestampMs) + dbTx := tg.txBuilder.prepareRewardTransaction(rtx, txHash, mbHash, mb, headerData, mbStatus) return dbTx, true } @@ -163,10 +154,8 @@ func (tg *txsGrouper) prepareRewardTxForDB( func (tg *txsGrouper) groupInvalidTxs( mbIndex int, mb *block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, txs map[string]*outport.TxInfo, - numOfShards uint32, - timestampMs uint64, ) (map[string]*data.Transaction, error) { transactions := make(map[string]*data.Transaction) mbHash, err := core.CalculateHash(tg.marshalizer, tg.hasher, mb) @@ -174,9 +163,9 @@ func (tg *txsGrouper) groupInvalidTxs( return nil, err } - executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, header) + executedTxHashes := extractExecutedTxHashes(mbIndex, mb.TxHashes, headerData) for _, txHash := range executedTxHashes { - invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, header, numOfShards, timestampMs) + invalidDBTx, ok := tg.prepareInvalidTxForDB(mbHash, mb, txHash, txs, headerData) if !ok { continue } @@ -192,16 +181,14 @@ func (tg *txsGrouper) prepareInvalidTxForDB( mb *block.MiniBlock, txHash []byte, txs map[string]*outport.TxInfo, - header coreData.HeaderHandler, - numOfShards uint32, - timestampMs uint64, + headerDta *data.HeaderData, ) (*data.Transaction, bool) { txInfo, okGet := txs[hex.EncodeToString(txHash)] if !okGet { return nil, false } - dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, header, transaction.TxStatusInvalid.String(), numOfShards, timestampMs) + dbTx := tg.txBuilder.prepareTransaction(txInfo, txHash, mbHash, mb, headerDta, transaction.TxStatusInvalid.String()) return dbTx, true } @@ -214,10 +201,10 @@ func (tg *txsGrouper) shouldIndex(destinationShardID uint32, isImportDB bool, se return selfShardID == destinationShardID } -func (tg *txsGrouper) groupReceipts(header coreData.HeaderHandler, txsPool map[string]*receipt.Receipt, timestampMs uint64) []*data.Receipt { +func (tg *txsGrouper) groupReceipts(headerData *data.HeaderData, txsPool map[string]*receipt.Receipt) []*data.Receipt { dbReceipts := make([]*data.Receipt, 0) for hashHex, rec := range txsPool { - dbReceipts = append(dbReceipts, tg.txBuilder.prepareReceipt(hashHex, rec, header, timestampMs)) + dbReceipts = append(dbReceipts, tg.txBuilder.prepareReceipt(hashHex, rec, headerData)) } return dbReceipts diff --git a/process/elasticproc/transactions/transactionsGrouper_test.go b/process/elasticproc/transactions/transactionsGrouper_test.go index 16faac4c..d5f3a7f0 100644 --- a/process/elasticproc/transactions/transactionsGrouper_test.go +++ b/process/elasticproc/transactions/transactionsGrouper_test.go @@ -9,6 +9,7 @@ import ( "github.com/multiversx/mx-chain-core-go/data/receipt" "github.com/multiversx/mx-chain-core-go/data/rewardTx" "github.com/multiversx/mx-chain-core-go/data/transaction" + "github.com/multiversx/mx-chain-es-indexer-go/data" "github.com/multiversx/mx-chain-es-indexer-go/mock" "github.com/multiversx/mx-chain-es-indexer-go/process/elasticproc/converters" "github.com/stretchr/testify/require" @@ -28,7 +29,11 @@ func TestGroupNormalTxs(t *testing.T) { TxHashes: [][]byte{txHash1, txHash2}, Type: block.TxBlock, } - header := &block.Header{TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + NumberOfShards: 3, + } txs := map[string]*outport.TxInfo{ hex.EncodeToString(txHash1): { Transaction: &transaction.Transaction{ @@ -46,7 +51,7 @@ func TestGroupNormalTxs(t *testing.T) { }, } - normalTxs, _ := grouper.groupNormalTxs(0, mb, header, txs, false, 3, 1234000) + normalTxs, _ := grouper.groupNormalTxs(0, mb, headerData, txs, false) require.Len(t, normalTxs, 2) require.Equal(t, uint64(1234), normalTxs[string(txHash1)].Timestamp) require.Equal(t, uint64(1234000), normalTxs[string(txHash1)].TimestampMs) @@ -66,7 +71,10 @@ func TestGroupRewardsTxs(t *testing.T) { TxHashes: [][]byte{txHash1, txHash2}, Type: block.RewardsBlock, } - header := &block.Header{TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + } txs := map[string]*outport.RewardInfo{ hex.EncodeToString(txHash1): {Reward: &rewardTx.RewardTx{ RcvAddr: []byte("receiver1"), @@ -76,7 +84,7 @@ func TestGroupRewardsTxs(t *testing.T) { }}, } - normalTxs, _ := grouper.groupRewardsTxs(0, mb, header, txs, false, 1234000) + normalTxs, _ := grouper.groupRewardsTxs(0, mb, headerData, txs, false) require.Len(t, normalTxs, 2) require.Equal(t, uint64(1234), normalTxs[string(txHash1)].Timestamp) require.Equal(t, uint64(1234000), normalTxs[string(txHash1)].TimestampMs) @@ -96,7 +104,11 @@ func TestGroupInvalidTxs(t *testing.T) { TxHashes: [][]byte{txHash1, txHash2}, Type: block.InvalidBlock, } - header := &block.Header{TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + NumberOfShards: 3, + } txs := map[string]*outport.TxInfo{ hex.EncodeToString(txHash1): { Transaction: &transaction.Transaction{ @@ -110,7 +122,7 @@ func TestGroupInvalidTxs(t *testing.T) { }, FeeInfo: &outport.FeeInfo{}}, } - normalTxs, _ := grouper.groupInvalidTxs(0, mb, header, txs, 3, 1234000) + normalTxs, _ := grouper.groupInvalidTxs(0, mb, headerData, txs) require.Len(t, normalTxs, 2) require.Equal(t, uint64(1234), normalTxs[string(txHash1)].Timestamp) require.Equal(t, uint64(1234000), normalTxs[string(txHash1)].TimestampMs) @@ -126,7 +138,10 @@ func TestGroupReceipts(t *testing.T) { txHash1 := []byte("txHash1") txHash2 := []byte("txHash2") - header := &block.Header{TimeStamp: 1234} + headerData := &data.HeaderData{ + Timestamp: 1234, + TimestampMs: 1234000, + } txs := map[string]*receipt.Receipt{ hex.EncodeToString(txHash1): { SndAddr: []byte("sender1"), @@ -136,7 +151,7 @@ func TestGroupReceipts(t *testing.T) { }, } - receipts := grouper.groupReceipts(header, txs, 1234000) + receipts := grouper.groupReceipts(headerData, txs) require.Len(t, receipts, 2) require.Equal(t, uint64(1234), receipts[0].Timestamp) require.Equal(t, uint64(1234000), receipts[0].TimestampMs) diff --git a/process/elasticproc/transactions/transactionsProcessor.go b/process/elasticproc/transactions/transactionsProcessor.go index 890f9332..1670def3 100644 --- a/process/elasticproc/transactions/transactionsProcessor.go +++ b/process/elasticproc/transactions/transactionsProcessor.go @@ -5,8 +5,6 @@ import ( "math/big" "github.com/multiversx/mx-chain-core-go/core" - "github.com/multiversx/mx-chain-core-go/core/check" - coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/hashing" @@ -69,13 +67,11 @@ func NewTransactionsProcessor(args *ArgsTransactionProcessor) (*txsDatabaseProce // PrepareTransactionsForDatabase will prepare transactions for database func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( miniBlocks []*block.MiniBlock, - header coreData.HeaderHandler, + headerData *data.HeaderData, pool *outport.TransactionPool, isImportDB bool, - numOfShards uint32, - timestampMs uint64, ) *data.PreparedResults { - err := checkPrepareTransactionForDatabaseArguments(header, pool) + err := checkPrepareTransactionForDatabaseArguments(pool) if err != nil { log.Warn("checkPrepareTransactionForDatabaseArguments", "error", err) @@ -92,25 +88,25 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( for mbIndex, mb := range miniBlocks { switch mb.Type { case block.TxBlock: - if shouldIgnoreProcessedMBScheduled(header, mbIndex) { + if shouldIgnoreProcessedMBScheduled(headerData, mbIndex) { continue } - txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, header, pool.Transactions, isImportDB, numOfShards, timestampMs) + txs, errGroup := tdp.txsGrouper.groupNormalTxs(mbIndex, mb, headerData, pool.Transactions, isImportDB) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupNormalTxs", "error", errGroup) continue } mergeTxsMaps(normalTxs, txs) case block.RewardsBlock: - txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, header, pool.Rewards, isImportDB, timestampMs) + txs, errGroup := tdp.txsGrouper.groupRewardsTxs(mbIndex, mb, headerData, pool.Rewards, isImportDB) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupRewardsTxs", "error", errGroup) continue } mergeTxsMaps(rewardsTxs, txs) case block.InvalidBlock: - txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, header, pool.InvalidTxs, numOfShards, timestampMs) + txs, errGroup := tdp.txsGrouper.groupInvalidTxs(mbIndex, mb, headerData, pool.InvalidTxs) if errGroup != nil { log.Warn("txsDatabaseProcessor.groupInvalidTxs", "error", errGroup) continue @@ -122,8 +118,8 @@ func (tdp *txsDatabaseProcessor) PrepareTransactionsForDatabase( } normalTxs = tdp.setTransactionSearchOrder(normalTxs) - dbReceipts := tdp.txsGrouper.groupReceipts(header, pool.Receipts, timestampMs) - dbSCResults := tdp.scrsProc.processSCRs(miniBlocks, header, pool.SmartContractResults, numOfShards, timestampMs) + dbReceipts := tdp.txsGrouper.groupReceipts(headerData, pool.Receipts) + dbSCResults := tdp.scrsProc.processSCRs(miniBlocks, headerData, pool.SmartContractResults) srcsNoTxInCurrentShard := tdp.scrsDataToTxs.attachSCRsToTransactionsAndReturnSCRsWithoutTx(normalTxs, dbSCResults) tdp.scrsDataToTxs.processTransactionsAfterSCRsWereAttached(normalTxs) @@ -152,20 +148,19 @@ func (tdp *txsDatabaseProcessor) setTransactionSearchOrder(transactions map[stri } // GetHexEncodedHashesForRemove will return hex encoded transaction hashes and smart contract result hashes from body -func (tdp *txsDatabaseProcessor) GetHexEncodedHashesForRemove(header coreData.HeaderHandler, body *block.Body) ([]string, []string) { - if body == nil || check.IfNil(header) || len(header.GetMiniBlockHeadersHashes()) == 0 { +func (tdp *txsDatabaseProcessor) GetHexEncodedHashesForRemove(headerData *data.HeaderData, body *block.Body) ([]string, []string) { + if body == nil || headerData == nil || len(headerData.MiniBlockHeaders) == 0 { return nil, nil } - selfShardID := header.GetShardID() encodedTxsHashes := make([]string, 0) encodedScrsHashes := make([]string, 0) for mbIndex, miniblock := range body.MiniBlocks { - if shouldIgnoreProcessedMBScheduled(header, mbIndex) { + if shouldIgnoreProcessedMBScheduled(headerData, mbIndex) { continue } - shouldIgnore := isCrossShardAtSourceNormalTx(selfShardID, miniblock) + shouldIgnore := isCrossShardAtSourceNormalTx(headerData.ShardID, miniblock) if shouldIgnore { // ignore cross-shard miniblocks at source with normal txs continue @@ -190,8 +185,8 @@ func isCrossShardAtSourceNormalTx(selfShardID uint32, miniblock *block.MiniBlock return isCrossShard && isAtSource && txBlock } -func shouldIgnoreProcessedMBScheduled(header coreData.HeaderHandler, mbIndex int) bool { - miniBlockHeaders := header.GetMiniBlockHeaderHandlers() +func shouldIgnoreProcessedMBScheduled(headerData *data.HeaderData, mbIndex int) bool { + miniBlockHeaders := headerData.MiniBlockHeaders if len(miniBlockHeaders) <= mbIndex { return false } diff --git a/process/elasticproc/transactions/transactionsProcessor_test.go b/process/elasticproc/transactions/transactionsProcessor_test.go index 230c2bd6..b3a43161 100644 --- a/process/elasticproc/transactions/transactionsProcessor_test.go +++ b/process/elasticproc/transactions/transactionsProcessor_test.go @@ -7,6 +7,7 @@ import ( "github.com/multiversx/mx-chain-core-go/core" "github.com/multiversx/mx-chain-core-go/core/pubkeyConverter" + coreData "github.com/multiversx/mx-chain-core-go/data" "github.com/multiversx/mx-chain-core-go/data/block" "github.com/multiversx/mx-chain-core-go/data/outport" "github.com/multiversx/mx-chain-core-go/data/receipt" @@ -182,7 +183,9 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { }, } - header := &block.Header{} + headerData := &data.HeaderData{ + NumberOfShards: 3, + } pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ @@ -211,7 +214,7 @@ func TestPrepareTransactionsForDatabase(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + results := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) assert.Equal(t, 7, len(results.Transactions)) } @@ -253,7 +256,9 @@ func TestRelayedTransactions(t *testing.T) { }, } - header := &block.Header{} + headerData := &data.HeaderData{ + NumberOfShards: 3, + } pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ @@ -267,7 +272,7 @@ func TestRelayedTransactions(t *testing.T) { txDbProc, _ := NewTransactionsProcessor(createMockArgsTxsDBProc()) - results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + results := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) assert.Equal(t, 1, len(results.Transactions)) assert.Equal(t, 2, len(results.Transactions[0].SmartContractResults)) assert.Equal(t, transaction.TxStatusSuccess.String(), results.Transactions[0].Status) @@ -364,7 +369,9 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { Type: block.ReceiptBlock, }, } - header := &block.Header{} + headerData := &data.HeaderData{ + NumberOfShards: 3, + } pool := &outport.TransactionPool{ InvalidTxs: map[string]*outport.TxInfo{ @@ -375,7 +382,7 @@ func TestCheckGasUsedInvalidTransaction(t *testing.T) { }, } - results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + results := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) require.Len(t, results.Transactions, 1) require.Equal(t, tx1.Transaction.GetGasLimit(), results.Transactions[0].GasUsed) } @@ -388,11 +395,12 @@ func TestGetRewardsTxsHashesHexEncoded(t *testing.T) { res, _ := txDBProc.GetHexEncodedHashesForRemove(nil, nil) require.Nil(t, res) - header := &block.Header{ - ShardID: core.MetachainShardId, - MiniBlockHeaders: []block.MiniBlockHeader{ - {}, - }, + mbsHeader := []coreData.MiniBlockHeaderHandler{ + &block.MiniBlockHeader{}, + } + header := &data.HeaderData{ + ShardID: core.MetachainShardId, + MiniBlockHeaders: mbsHeader, } body := &block.Body{ MiniBlocks: []*block.MiniBlock{ @@ -494,7 +502,9 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * }, } - header := &block.Header{} + headerData := &data.HeaderData{ + NumberOfShards: 3, + } pool := &outport.TransactionPool{ InvalidTxs: map[string]*outport.TxInfo{ @@ -505,7 +515,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseInvalidTxWithSCR(t * }, } - results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + results := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -551,7 +561,9 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, } - header := &block.Header{} + headerData := &data.HeaderData{ + NumberOfShards: 3, + } pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ @@ -562,7 +574,7 @@ func TestTxsDatabaseProcessor_PrepareTransactionsForDatabaseESDTNFTTransfer(t *t }, } - results := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + results := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) require.NotNil(t, results) require.Len(t, results.Transactions, 1) require.Len(t, results.ScResults, 1) @@ -599,8 +611,9 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { Type: block.SmartContractResultBlock, }, } - header := &block.Header{ - ShardID: core.MetachainShardId, + headerData := &data.HeaderData{ + ShardID: core.MetachainShardId, + NumberOfShards: 3, } pool := &outport.TransactionPool{ Transactions: map[string]*outport.TxInfo{ @@ -626,7 +639,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res := txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + res := txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 2, len(res.ScResults)) @@ -649,7 +662,7 @@ func TestTxsDatabaseProcessor_IssueESDTTx(t *testing.T) { }, } - res = txDbProc.PrepareTransactionsForDatabase(mbs, header, pool, false, 3, 1234000) + res = txDbProc.PrepareTransactionsForDatabase(mbs, headerData, pool, false) require.Equal(t, "success", res.Transactions[0].Status) require.Equal(t, 1, len(res.ScResults)) } diff --git a/process/factory/indexerFactory.go b/process/factory/indexerFactory.go index e193e36a..eb422705 100644 --- a/process/factory/indexerFactory.go +++ b/process/factory/indexerFactory.go @@ -33,6 +33,7 @@ type ArgsIndexerFactory struct { ImportDB bool Denomination int BulkRequestMaxSize int + NumWritesInParallel int Url string UserName string Password string @@ -161,10 +162,18 @@ func createBlockCreatorsContainer() (dataindexer.BlockContainerHandler, error) { if err != nil { return nil, err } + err = container.Add(core.ShardHeaderV3, block.NewEmptyHeaderV3Creator()) + if err != nil { + return nil, err + } err = container.Add(core.MetaHeader, block.NewEmptyMetaBlockCreator()) if err != nil { return nil, err } + err = container.Add(core.MetaHeaderV3, block.NewEmptyMetaBlockV3Creator()) + if err != nil { + return nil, err + } return container, nil } diff --git a/templates/indices/executionResults.go b/templates/indices/executionResults.go new file mode 100644 index 00000000..70cb6a24 --- /dev/null +++ b/templates/indices/executionResults.go @@ -0,0 +1,80 @@ +package indices + +// ExecutionResults will hold the configuration for the execution results index +var ExecutionResults = Object{ + "index_patterns": Array{ + "executionresults-*", + }, + "template": Object{ + "settings": Object{ + "number_of_shards": 3, + "number_of_replicas": 0, + "index": Object{ + "sort.field": Array{ + "timestampMs", "nonce", + }, + "sort.order": Array{ + "desc", "desc", + }, + }, + }, + "mappings": Object{ + "properties": Object{ + "miniBlocksDetails": Object{ + "properties": Object{ + "firstProcessedTx": Object{ + "index": "false", + "type": "long", + }, + "lastProcessedTx": Object{ + "index": "false", + "type": "long", + }, + "mbIndex": Object{ + "index": "false", + "type": "long", + }, + }, + }, + "miniBlocksHashes": Object{ + "type": "keyword", + }, + "nonce": Object{ + "type": "double", + }, + "round": Object{ + "type": "double", + }, + "rootHash": Object{ + "index": "false", + "type": "keyword", + }, + "notarizedInBlockHash": Object{ + "type": "keyword", + }, + "epoch": Object{ + "type": "long", + }, + "gasUsed": Object{ + "type": "double", + }, + "txCount": Object{ + "index": "false", + "type": "long", + }, + "accumulatedFees": Object{ + "index": "false", + "type": "keyword", + }, + "developerFees": Object{ + "index": "false", + "type": "keyword", + }, + "timestampMs": Object{ + "type": "date", + "format": "epoch_millis", + }, + }, + }, + }, +}