diff --git a/go.mod b/go.mod index 9eeace25..d80c245e 100644 --- a/go.mod +++ b/go.mod @@ -78,7 +78,6 @@ require ( github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect - github.com/decred/dcrd/container/lru v1.0.0 // indirect github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect github.com/deso-protocol/go-merkle-tree v1.0.0 // indirect github.com/dgraph-io/ristretto v0.2.0 // indirect diff --git a/go.sum b/go.sum index 248d5b7c..54bcb038 100644 --- a/go.sum +++ b/go.sum @@ -134,8 +134,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI= -github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8= github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= diff --git a/routes/admin_node.go b/routes/admin_node.go index c1cd6112..159433ae 100644 --- a/routes/admin_node.go +++ b/routes/admin_node.go @@ -338,7 +338,7 @@ func (fes *APIServer) _handleConnectDeSoNode( // Note: We don't need to acquire the ChainLock because our parent does it. // Grab the ChainLock since we might do a blockchain lookup below. - locator := fes.blockchain.LatestHeaderLocator() + locator, _ := fes.blockchain.LatestHeaderLocator() desoPeer.AddDeSoMessage(&lib.MsgDeSoGetHeaders{ StopHash: &lib.BlockHash{}, diff --git a/routes/exchange.go b/routes/exchange.go index a8251eed..e6e2c4e8 100644 --- a/routes/exchange.go +++ b/routes/exchange.go @@ -1375,7 +1375,7 @@ func (fes *APIServer) getTxindexMetadataForUncommittedBlock( uncommittedTxnMetaMap := make(map[lib.BlockHash]*lib.TransactionMetadata) txindexUtxoView := lib.NewUtxoView(fes.blockchain.DB(), fes.Params, nil, nil, nil) if blockNode.Header.PrevBlockHash != nil && !txindexUtxoView.TipHash.IsEqual(blockNode.Header.PrevBlockHash) { - utxoViewAndUtxoOps, err := fes.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockNode.Header.PrevBlockHash) + utxoViewAndUtxoOps, err := fes.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockNode.Header.PrevBlockHash, blockNode.Header.Height-1) if err != nil { return nil, errors.Wrap(err, "getTxindexMetadataForUncommittedBlock: Problem fetching utxoView for uncommitted block") @@ -1411,7 +1411,7 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) { // For this endpoint we need to lock the blockchain for reading. // If the HashHex is set, look the block up using that. - numBlocks := len(fes.blockchain.BestChain()) + numBlocks := fes.blockchain.BlockTip().Height + 1 var blockNode *lib.BlockNode var blockHash *lib.BlockHash @@ -1433,7 +1433,7 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) { } else { // Find the block node with the corresponding height on the best chain. if blockRequest.Height >= int64(numBlocks) || blockRequest.Height < 0 { - maxHeight := len(fes.blockchain.BestChain()) - 1 + maxHeight := fes.blockchain.BlockTip().Height APIAddError(ww, fmt.Sprintf("APIBlockRequest: Height requested "+ "%d must be >= 0 and <= "+ @@ -1441,7 +1441,17 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) { maxHeight)) return } - blockNode = fes.blockchain.BestChain()[blockRequest.Height] + var exists bool + var err error + blockNode, exists, err = fes.blockchain.GetBlockFromBestChainByHeight(uint64(blockRequest.Height), false) + if err != nil { + APIAddError(ww, fmt.Sprintf("APIBlockRequest: Problem fetching block: %v", err)) + return + } + if !exists { + APIAddError(ww, fmt.Sprintf("APIBlockRequest: Block with height %d not found", blockRequest.Height)) + return + } blockHash = blockNode.Hash } diff --git a/routes/exchange_test.go b/routes/exchange_test.go index 5ee8a6d4..af8c8b26 100644 --- a/routes/exchange_test.go +++ b/routes/exchange_test.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/hex" "encoding/json" + "github.com/deso-protocol/core/collections" "io" "log" "net/http" @@ -110,6 +111,13 @@ func NewLowDifficultyBlockchainWithParams(t *testing.T, params *lib.DeSoParams) return chain, ¶msCopy, db, dir } +func GetBlockFromBestChainByHeight(t *testing.T, blockchain *lib.Blockchain, height uint32) *lib.BlockNode { + blockNode, exists, err := blockchain.GetBlockFromBestChainByHeight(uint64(height), false) + require.NoError(t, err) + require.True(t, exists) + return blockNode +} + func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, isSender bool) (*lib.DeSoMempool, *lib.DeSoMiner) { assert := assert.New(t) require := require.New(t) @@ -559,16 +567,18 @@ func TestAPI(t *testing.T) { } assert.Equal("", transactionInfoRes.Error) assert.Equal(2, len(transactionInfoRes.Transactions)) + block1 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1) + block2 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 2) assert.Contains( []string{ - hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]), - hex.EncodeToString(apiServer.blockchain.BestChain()[2].Hash[:]), + hex.EncodeToString(block1.Hash[:]), + hex.EncodeToString(block2.Hash[:]), }, transactionInfoRes.Transactions[0].BlockHashHex) assert.Contains( []string{ - hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]), - hex.EncodeToString(apiServer.blockchain.BestChain()[2].Hash[:]), + hex.EncodeToString(block1.Hash[:]), + hex.EncodeToString(block2.Hash[:]), }, transactionInfoRes.Transactions[1].BlockHashHex) assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs)) @@ -578,10 +588,10 @@ func TestAPI(t *testing.T) { var firstBlockTxn *lib.MsgDeSoTxn var secondBlockTxn *lib.MsgDeSoTxn { - blockHash := apiServer.blockchain.BestChain()[1].Hash + blockHash := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash blockLookup, err := lib.GetBlock(blockHash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot()) require.NoError(err) - block2Lookup, err := lib.GetBlock(apiServer.blockchain.BestChain()[2].Hash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot()) + block2Lookup, err := lib.GetBlock(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 2).Hash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot()) firstBlockTxn = blockLookup.Txns[0] secondBlockTxn = block2Lookup.Txns[0] @@ -607,7 +617,7 @@ func TestAPI(t *testing.T) { assert.Equal(1, len(transactionInfoRes.Transactions)) assert.Contains( []string{ - hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]), + hex.EncodeToString(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash[:]), }, transactionInfoRes.Transactions[0].BlockHashHex) assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs)) @@ -1299,8 +1309,16 @@ func TestAPI(t *testing.T) { // Set the tip to the first block and make sure all txns get deleted. { - chainWithFirstBlockOnly := apiServer.blockchain.BestChain()[:2] - oldBestChain := apiServer.blockchain.BestChain() + block0 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 0) + block1 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1) + chainWithFirstBlockOnly := []*lib.BlockNode{block0, block1} + oldBestChainTip := apiServer.blockchain.BlockTip() + oldBestChain := []*lib.BlockNode{} + for oldBestChainTip != nil { + oldBestChain = append(oldBestChain, oldBestChainTip) + oldBestChainTip = oldBestChainTip.GetParent(apiServer.blockchain.GetBlockIndex()) + } + oldBestChain = collections.Reverse(oldBestChain) apiServer.blockchain.SetBestChain(chainWithFirstBlockOnly) require.NoError(apiServer.TXIndex.Update()) @@ -1309,12 +1327,12 @@ func TestAPI(t *testing.T) { { prefix := lib.DbTxindexTxIDKey(&lib.BlockHash{})[0] txnsInTransactionIndex, _ := lib.EnumerateKeysForPrefix( - apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true) + apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true, false) require.Equal(1+len(apiServer.Params.SeedTxns)+len(apiServer.Params.SeedBalances), len(txnsInTransactionIndex)) } { keysInPublicKeyTable, _ := lib.EnumerateKeysForPrefix( - apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true) + apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true, false) // There should be two keys since one is the miner public key and // the other is a dummy public key corresponding to the input of // a block reward txn. Plus one for the seed balance, which creates @@ -1355,7 +1373,7 @@ func TestAPI(t *testing.T) { assert.Equal(1, len(transactionInfoRes.Transactions)) assert.Contains( []string{ - hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]), + hex.EncodeToString(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash[:]), }, transactionInfoRes.Transactions[0].BlockHashHex) assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs)) @@ -1395,12 +1413,12 @@ func TestAPI(t *testing.T) { { prefix := lib.DbTxindexTxIDKey(&lib.BlockHash{})[0] txnsInTransactionIndex, _ := lib.EnumerateKeysForPrefix( - apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true) + apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true, false) require.Equal(5+len(apiServer.Params.SeedTxns)+len(apiServer.Params.SeedBalances), len(txnsInTransactionIndex)) } { keysInPublicKeyTable, _ := lib.EnumerateKeysForPrefix( - apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true) + apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true, false) // Three pairs for the block rewards and two pairs for the transactions // we created. require.Equal(10+ diff --git a/routes/hot_feed.go b/routes/hot_feed.go index dd4619a3..e43eebe5 100644 --- a/routes/hot_feed.go +++ b/routes/hot_feed.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/deso-protocol/core/collections" "io" "math" "net/http" @@ -79,7 +80,11 @@ func (fes *APIServer) StartHotFeedRoutine() { fes.PostTagToOrderedHotFeedEntries = make(map[string][]*HotFeedEntry) fes.PostTagToOrderedNewestEntries = make(map[string][]*HotFeedEntry) fes.PostHashToPostTagsMap = make(map[lib.BlockHash][]string) - fes.HotFeedBlockCache = make(map[lib.BlockHash]*lib.MsgDeSoBlock) + var err error + fes.HotFeedBlockCache, err = collections.NewLruCache[lib.BlockHash, []*lib.MsgDeSoTxn](LookbackWindowBlocks + 50000) // Give it a buffer of 50k blocks. + if err != nil { + glog.Errorf("StartHotFeedRoutine: Error initializing HotFeedBlockCache: %v", err) + } cacheResetCounter := 0 go func() { out: @@ -117,7 +122,6 @@ func (fes *APIServer) UpdateHotFeed(resetCache bool) { glog.V(2).Info("Resetting hot feed cache.") fes.PostTagToPostHashesMap = make(map[string]map[lib.BlockHash]bool) fes.PostHashToPostTagsMap = make(map[lib.BlockHash][]string) - fes.HotFeedBlockCache = make(map[lib.BlockHash]*lib.MsgDeSoBlock) } // We copy the HotFeedApprovedPosts map and HotFeedPKIDMultiplier maps so we can access @@ -403,15 +407,11 @@ func (fes *APIServer) UpdateHotFeedOrderedList( // This offset allows us to see what the hot feed would look like in the past, // which is useful for testing purposes. - blockOffsetForTesting := 0 + // TODO: this is a little more annoying to support without the full history of best chain. + // we can revisit implementing later if needed. Just takes a few more minutes. + // blockOffsetForTesting := 0 lookbackWindowBlocks := LookbackWindowBlocks - // Check if the most recent blocks that we'll be considering in hot feed computation have been processed. - for _, blockNode := range fes.blockchain.BestChain() { - if blockNode.Height < blockTip.Height-uint32(lookbackWindowBlocks+blockOffsetForTesting) { - continue - } - } // Log how long this routine takes, since it could be heavy. glog.V(2).Info("UpdateHotFeedOrderedList: Starting new update cycle.") @@ -425,23 +425,26 @@ func (fes *APIServer) UpdateHotFeedOrderedList( } // Grab the last 24 hours worth of blocks (288 blocks @ 5min/block). - blockTipIndex := len(fes.blockchain.BestChain()) - 1 - blockOffsetForTesting - relevantNodes := fes.blockchain.BestChain() - if len(fes.blockchain.BestChain()) > (lookbackWindowBlocks + blockOffsetForTesting) { - relevantNodes = fes.blockchain.BestChain()[blockTipIndex-lookbackWindowBlocks-blockOffsetForTesting : blockTipIndex] + startNode := fes.blockchain.BlockTip() + relevantNodes := []*lib.BlockNode{} + for len(relevantNodes) < lookbackWindowBlocks && startNode != nil { + relevantNodes = append(relevantNodes, startNode) + startNode = startNode.GetParent(fes.blockchain.GetBlockIndex()) } + relevantNodes = collections.Reverse(relevantNodes) var hotnessInfoBlocks []*HotnessInfoBlock for blockIdx, node := range relevantNodes { - var block *lib.MsgDeSoBlock - if cachedBlock, ok := fes.HotFeedBlockCache[*node.Hash]; ok { - block = cachedBlock + var txns []*lib.MsgDeSoTxn + if cachedBlock, ok := fes.HotFeedBlockCache.Get(*node.Hash); ok { + txns = cachedBlock } else { - block, _ = lib.GetBlock(node.Hash, utxoView.Handle, fes.blockchain.Snapshot()) - fes.HotFeedBlockCache[*node.Hash] = block + block, _ := lib.GetBlock(node.Hash, utxoView.Handle, fes.blockchain.Snapshot()) + fes.HotFeedBlockCache.Put(*node.Hash, block.Txns) + txns = block.Txns } hotnessInfoBlocks = append(hotnessInfoBlocks, &HotnessInfoBlock{ - Block: block, + BlockTxns: txns, // For time decay, we care about how many blocks away from the tip this block is. BlockAge: len(relevantNodes) - blockIdx, }) @@ -465,14 +468,10 @@ func (fes *APIServer) UpdateHotFeedOrderedList( txnsFromMempoolOrderedByTime = append(txnsFromMempoolOrderedByTime, mempoolTxn.Tx) } - if err != nil { - glog.Errorf("Error getting mempool transactions: %v", err) - } else if len(txnsFromMempoolOrderedByTime) > 0 { + if len(txnsFromMempoolOrderedByTime) > 0 { hotnessInfoBlocks = append(hotnessInfoBlocks, &HotnessInfoBlock{ - Block: &lib.MsgDeSoBlock{ - Txns: txnsFromMempoolOrderedByTime, - }, - BlockAge: mempoolBlockHeight, + BlockTxns: txnsFromMempoolOrderedByTime, + BlockAge: mempoolBlockHeight, }) } @@ -529,8 +528,8 @@ func (fes *APIServer) UpdateHotFeedOrderedList( } type HotnessInfoBlock struct { - Block *lib.MsgDeSoBlock - BlockAge int + BlockTxns []*lib.MsgDeSoTxn + BlockAge int } func (fes *APIServer) PopulateHotnessInfoMap( @@ -545,12 +544,12 @@ func (fes *APIServer) PopulateHotnessInfoMap( postInteractionMap := make(map[HotFeedInteractionKey]uint64) for _, hotnessInfoBlock := range hotnessInfoBlocks { - block := hotnessInfoBlock.Block - blockAgee := hotnessInfoBlock.BlockAge - if block == nil { + + if hotnessInfoBlock == nil { continue } - for _, txn := range block.Txns { + blockAgee := hotnessInfoBlock.BlockAge + for _, txn := range hotnessInfoBlock.BlockTxns { // We only care about posts created in the specified look-back period. There should always be a // transaction that creates a given post before someone interacts with it. By only // scoring posts that meet this condition, we can restrict the HotFeedOrderedList diff --git a/routes/server.go b/routes/server.go index 8fd4260f..a9734fa5 100644 --- a/routes/server.go +++ b/routes/server.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" fmt "fmt" + "github.com/deso-protocol/core/collections" "io" "io/ioutil" "net" @@ -426,7 +427,7 @@ type APIServer struct { // The height of the last block evaluated by the hotness routine. HotFeedBlockHeight uint32 // A cache to store blocks for the block feed - in order to reduce processing time. - HotFeedBlockCache map[lib.BlockHash]*lib.MsgDeSoBlock + HotFeedBlockCache *collections.LruCache[lib.BlockHash, []*lib.MsgDeSoTxn] // Map of whitelisted post hashes used for serving the hot feed. // The float64 value is a multiplier than can be modified and used in scoring. HotFeedApprovedPostsToMultipliers map[lib.BlockHash]float64 @@ -2551,7 +2552,7 @@ func Logger(inner http.Handler, name string) http.Handler { inner.ServeHTTP(w, r) - glog.V(2).Infof( + glog.V(3).Infof( "%s\t%s\t%s\t%s", r.Method, r.RequestURI, diff --git a/routes/transaction.go b/routes/transaction.go index af67db93..b93d63eb 100644 --- a/routes/transaction.go +++ b/routes/transaction.go @@ -4801,9 +4801,9 @@ func (fes *APIServer) GetTxnConstructionParams(ww http.ResponseWriter, req *http func (fes *APIServer) GetCommittedTipBlockInfo(ww http.ResponseWriter, req *http.Request) { // Get the block tip from the blockchain. fes.backendServer.GetBlockchain().ChainLock.RLock() - blockTip, idx := fes.backendServer.GetBlockchain().GetCommittedTip() + blockTip, exists := fes.backendServer.GetBlockchain().GetCommittedTip() fes.backendServer.GetBlockchain().ChainLock.RUnlock() - if idx == -1 { + if !exists { _AddBadRequestError(ww, "GetCommittedTipBlockInfo: Problem getting block tip") return } diff --git a/scripts/tools/toolslib/chain.go b/scripts/tools/toolslib/chain.go index 50c7bae0..d232c345 100644 --- a/scripts/tools/toolslib/chain.go +++ b/scripts/tools/toolslib/chain.go @@ -18,6 +18,8 @@ func OpenDataDir(dataDir string) (*badger.DB, error) { return db, nil } +// TODO: This utility function needs to be updated to account for not having the entire chain in +// memory (no more BEST CHAIN representing the entire history of the blockchain). // Returns the best chain associated with a badgerDB handle. func GetBestChainFromBadger(syncedDBHandle *badger.DB, params *lib.DeSoParams) ([]*lib.BlockNode, error) { bestBlockHash := lib.DbGetBestHash(syncedDBHandle, nil, lib.ChainTypeDeSoBlock) @@ -38,7 +40,8 @@ func GetBestChainFromBadger(syncedDBHandle *badger.DB, params *lib.DeSoParams) ( } // Walk back from the best node to the genesis block and store them all in bestChain. - bestChain, err := lib.GetBestChain(tipNode) + bi := lib.NewBlockIndex(syncedDBHandle, nil, tipNode) + bestChain, err := lib.GetBestChain(tipNode, bi) if err != nil { return nil, errors.Wrap(err, "GetBestChainFromBadger() failed to GetBestChain") }