updates to support no longer having best chain

go.mod

@@ -78,7 +78,6 @@ require (
 	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
-	github.com/decred/dcrd/container/lru v1.0.0 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
 	github.com/deso-protocol/go-merkle-tree v1.0.0 // indirect
 	github.com/dgraph-io/ristretto v0.2.0 // indirect

go.sum

@@ -134,8 +134,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/decred/dcrd/container/lru v1.0.0 h1:7foQymtbu18aQWYiY9RnNIeE+kvpiN+fiBQ3+viyJjI=
-github.com/decred/dcrd/container/lru v1.0.0/go.mod h1:vlPwj0l+IzAHhQSsbgQnJgO5Cte78+yI065V+Mc5PRQ=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
 github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
 github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=

routes/admin_node.go

@@ -338,7 +338,7 @@ func (fes *APIServer) _handleConnectDeSoNode(
 				// Note: We don't need to acquire the ChainLock because our parent does it.
 
 				// Grab the ChainLock since we might do a blockchain lookup below.
-				locator := fes.blockchain.LatestHeaderLocator()
+				locator, _ := fes.blockchain.LatestHeaderLocator()
 
 				desoPeer.AddDeSoMessage(&lib.MsgDeSoGetHeaders{
 					StopHash:     &lib.BlockHash{},

routes/exchange.go

@@ -1375,7 +1375,7 @@ func (fes *APIServer) getTxindexMetadataForUncommittedBlock(
 	uncommittedTxnMetaMap := make(map[lib.BlockHash]*lib.TransactionMetadata)
 	txindexUtxoView := lib.NewUtxoView(fes.blockchain.DB(), fes.Params, nil, nil, nil)
 	if blockNode.Header.PrevBlockHash != nil && !txindexUtxoView.TipHash.IsEqual(blockNode.Header.PrevBlockHash) {
-		utxoViewAndUtxoOps, err := fes.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockNode.Header.PrevBlockHash)
+		utxoViewAndUtxoOps, err := fes.blockchain.GetUtxoViewAndUtxoOpsAtBlockHash(*blockNode.Header.PrevBlockHash, blockNode.Header.Height-1)
 		if err != nil {
 			return nil, errors.Wrap(err,
 				"getTxindexMetadataForUncommittedBlock: Problem fetching utxoView for uncommitted block")
@@ -1411,7 +1411,7 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) {
 
 	// For this endpoint we need to lock the blockchain for reading.
 	// If the HashHex is set, look the block up using that.
-	numBlocks := len(fes.blockchain.BestChain())
+	numBlocks := fes.blockchain.BlockTip().Height + 1
 
 	var blockNode *lib.BlockNode
 	var blockHash *lib.BlockHash
@@ -1433,15 +1433,25 @@ func (fes *APIServer) APIBlock(ww http.ResponseWriter, rr *http.Request) {
 	} else {
 		// Find the block node with the corresponding height on the best chain.
 		if blockRequest.Height >= int64(numBlocks) || blockRequest.Height < 0 {
-			maxHeight := len(fes.blockchain.BestChain()) - 1
+			maxHeight := fes.blockchain.BlockTip().Height
 
 			APIAddError(ww, fmt.Sprintf("APIBlockRequest: Height requested "+
 				"%d must be >= 0 and <= "+
 				"height of best block chain tip %d", blockRequest.Height,
 				maxHeight))
 			return
 		}
-		blockNode = fes.blockchain.BestChain()[blockRequest.Height]
+		var exists bool
+		var err error
+		blockNode, exists, err = fes.blockchain.GetBlockFromBestChainByHeight(uint64(blockRequest.Height), false)
+		if err != nil {
+			APIAddError(ww, fmt.Sprintf("APIBlockRequest: Problem fetching block: %v", err))
+			return
+		}
+		if !exists {
+			APIAddError(ww, fmt.Sprintf("APIBlockRequest: Block with height %d not found", blockRequest.Height))
+			return
+		}
 		blockHash = blockNode.Hash
 	}
 

routes/exchange_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/hex"
 	"encoding/json"
+	"github.com/deso-protocol/core/collections"
 	"io"
 	"log"
 	"net/http"
@@ -110,6 +111,13 @@ func NewLowDifficultyBlockchainWithParams(t *testing.T, params *lib.DeSoParams)
 	return chain, &paramsCopy, db, dir
 }
 
+func GetBlockFromBestChainByHeight(t *testing.T, blockchain *lib.Blockchain, height uint32) *lib.BlockNode {
+	blockNode, exists, err := blockchain.GetBlockFromBestChainByHeight(uint64(height), false)
+	require.NoError(t, err)
+	require.True(t, exists)
+	return blockNode
+}
+
 func NewTestMiner(t *testing.T, chain *lib.Blockchain, params *lib.DeSoParams, isSender bool) (*lib.DeSoMempool, *lib.DeSoMiner) {
 	assert := assert.New(t)
 	require := require.New(t)
@@ -559,16 +567,18 @@ func TestAPI(t *testing.T) {
 		}
 		assert.Equal("", transactionInfoRes.Error)
 		assert.Equal(2, len(transactionInfoRes.Transactions))
+		block1 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1)
+		block2 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 2)
 		assert.Contains(
 			[]string{
-				hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]),
-				hex.EncodeToString(apiServer.blockchain.BestChain()[2].Hash[:]),
+				hex.EncodeToString(block1.Hash[:]),
+				hex.EncodeToString(block2.Hash[:]),
 			},
 			transactionInfoRes.Transactions[0].BlockHashHex)
 		assert.Contains(
 			[]string{
-				hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]),
-				hex.EncodeToString(apiServer.blockchain.BestChain()[2].Hash[:]),
+				hex.EncodeToString(block1.Hash[:]),
+				hex.EncodeToString(block2.Hash[:]),
 			},
 			transactionInfoRes.Transactions[1].BlockHashHex)
 		assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs))
@@ -578,10 +588,10 @@ func TestAPI(t *testing.T) {
 	var firstBlockTxn *lib.MsgDeSoTxn
 	var secondBlockTxn *lib.MsgDeSoTxn
 	{
-		blockHash := apiServer.blockchain.BestChain()[1].Hash
+		blockHash := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash
 		blockLookup, err := lib.GetBlock(blockHash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot())
 		require.NoError(err)
-		block2Lookup, err := lib.GetBlock(apiServer.blockchain.BestChain()[2].Hash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot())
+		block2Lookup, err := lib.GetBlock(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 2).Hash, apiServer.blockchain.DB(), apiServer.blockchain.Snapshot())
 
 		firstBlockTxn = blockLookup.Txns[0]
 		secondBlockTxn = block2Lookup.Txns[0]
@@ -607,7 +617,7 @@ func TestAPI(t *testing.T) {
 		assert.Equal(1, len(transactionInfoRes.Transactions))
 		assert.Contains(
 			[]string{
-				hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]),
+				hex.EncodeToString(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash[:]),
 			},
 			transactionInfoRes.Transactions[0].BlockHashHex)
 		assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs))
@@ -1299,8 +1309,16 @@ func TestAPI(t *testing.T) {
 
 	// Set the tip to the first block and make sure all txns get deleted.
 	{
-		chainWithFirstBlockOnly := apiServer.blockchain.BestChain()[:2]
-		oldBestChain := apiServer.blockchain.BestChain()
+		block0 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 0)
+		block1 := GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1)
+		chainWithFirstBlockOnly := []*lib.BlockNode{block0, block1}
+		oldBestChainTip := apiServer.blockchain.BlockTip()
+		oldBestChain := []*lib.BlockNode{}
+		for oldBestChainTip != nil {
+			oldBestChain = append(oldBestChain, oldBestChainTip)
+			oldBestChainTip = oldBestChainTip.GetParent(apiServer.blockchain.GetBlockIndex())
+		}
+		oldBestChain = collections.Reverse(oldBestChain)
 		apiServer.blockchain.SetBestChain(chainWithFirstBlockOnly)
 		require.NoError(apiServer.TXIndex.Update())
 
@@ -1309,12 +1327,12 @@ func TestAPI(t *testing.T) {
 		{
 			prefix := lib.DbTxindexTxIDKey(&lib.BlockHash{})[0]
 			txnsInTransactionIndex, _ := lib.EnumerateKeysForPrefix(
-				apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true)
+				apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true, false)
 			require.Equal(1+len(apiServer.Params.SeedTxns)+len(apiServer.Params.SeedBalances), len(txnsInTransactionIndex))
 		}
 		{
 			keysInPublicKeyTable, _ := lib.EnumerateKeysForPrefix(
-				apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true)
+				apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true, false)
 			// There should be two keys since one is the miner public key and
 			// the other is a dummy public key corresponding to the input of
 			// a block reward txn. Plus one for the seed balance, which creates
@@ -1355,7 +1373,7 @@ func TestAPI(t *testing.T) {
 			assert.Equal(1, len(transactionInfoRes.Transactions))
 			assert.Contains(
 				[]string{
-					hex.EncodeToString(apiServer.blockchain.BestChain()[1].Hash[:]),
+					hex.EncodeToString(GetBlockFromBestChainByHeight(t, apiServer.blockchain, 1).Hash[:]),
 				},
 				transactionInfoRes.Transactions[0].BlockHashHex)
 			assert.Equal(0, len(transactionInfoRes.Transactions[0].Inputs))
@@ -1395,12 +1413,12 @@ func TestAPI(t *testing.T) {
 		{
 			prefix := lib.DbTxindexTxIDKey(&lib.BlockHash{})[0]
 			txnsInTransactionIndex, _ := lib.EnumerateKeysForPrefix(
-				apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true)
+				apiServer.TXIndex.TXIndexChain.DB(), []byte{prefix}, true, false)
 			require.Equal(5+len(apiServer.Params.SeedTxns)+len(apiServer.Params.SeedBalances), len(txnsInTransactionIndex))
 		}
 		{
 			keysInPublicKeyTable, _ := lib.EnumerateKeysForPrefix(
-				apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true)
+				apiServer.TXIndex.TXIndexChain.DB(), lib.DbTxindexPublicKeyPrefix([]byte{}), true, false)
 			// Three pairs for the block rewards and two pairs for the transactions
 			// we created.
 			require.Equal(10+

routes/hot_feed.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"github.com/deso-protocol/core/collections"
 	"io"
 	"math"
 	"net/http"
@@ -79,7 +80,11 @@ func (fes *APIServer) StartHotFeedRoutine() {
 	fes.PostTagToOrderedHotFeedEntries = make(map[string][]*HotFeedEntry)
 	fes.PostTagToOrderedNewestEntries = make(map[string][]*HotFeedEntry)
 	fes.PostHashToPostTagsMap = make(map[lib.BlockHash][]string)
-	fes.HotFeedBlockCache = make(map[lib.BlockHash]*lib.MsgDeSoBlock)
+	var err error
+	fes.HotFeedBlockCache, err = collections.NewLruCache[lib.BlockHash, []*lib.MsgDeSoTxn](LookbackWindowBlocks + 50000) // Give it a buffer of 50k blocks.
+	if err != nil {
+		glog.Errorf("StartHotFeedRoutine: Error initializing HotFeedBlockCache: %v", err)
+	}
 	cacheResetCounter := 0
 	go func() {
 	out:
@@ -117,7 +122,6 @@ func (fes *APIServer) UpdateHotFeed(resetCache bool) {
 		glog.V(2).Info("Resetting hot feed cache.")
 		fes.PostTagToPostHashesMap = make(map[string]map[lib.BlockHash]bool)
 		fes.PostHashToPostTagsMap = make(map[lib.BlockHash][]string)
-		fes.HotFeedBlockCache = make(map[lib.BlockHash]*lib.MsgDeSoBlock)
 	}
 
 	// We copy the HotFeedApprovedPosts map and HotFeedPKIDMultiplier maps so we can access
@@ -403,15 +407,11 @@ func (fes *APIServer) UpdateHotFeedOrderedList(
 
 	// This offset allows us to see what the hot feed would look like in the past,
 	// which is useful for testing purposes.
-	blockOffsetForTesting := 0
+	// TODO: this is a little more annoying to support without the full history of best chain.
+	// we can revisit implementing later if needed. Just takes a few more minutes.
+	// blockOffsetForTesting := 0
 
 	lookbackWindowBlocks := LookbackWindowBlocks
-	// Check if the most recent blocks that we'll be considering in hot feed computation have been processed.
-	for _, blockNode := range fes.blockchain.BestChain() {
-		if blockNode.Height < blockTip.Height-uint32(lookbackWindowBlocks+blockOffsetForTesting) {
-			continue
-		}
-	}
 
 	// Log how long this routine takes, since it could be heavy.
 	glog.V(2).Info("UpdateHotFeedOrderedList: Starting new update cycle.")
@@ -425,23 +425,26 @@ func (fes *APIServer) UpdateHotFeedOrderedList(
 	}
 
 	// Grab the last 24 hours worth of blocks (288 blocks @ 5min/block).
-	blockTipIndex := len(fes.blockchain.BestChain()) - 1 - blockOffsetForTesting
-	relevantNodes := fes.blockchain.BestChain()
-	if len(fes.blockchain.BestChain()) > (lookbackWindowBlocks + blockOffsetForTesting) {
-		relevantNodes = fes.blockchain.BestChain()[blockTipIndex-lookbackWindowBlocks-blockOffsetForTesting : blockTipIndex]
+	startNode := fes.blockchain.BlockTip()
+	relevantNodes := []*lib.BlockNode{}
+	for len(relevantNodes) < lookbackWindowBlocks && startNode != nil {
+		relevantNodes = append(relevantNodes, startNode)
+		startNode = startNode.GetParent(fes.blockchain.GetBlockIndex())
 	}
+	relevantNodes = collections.Reverse(relevantNodes)
 
 	var hotnessInfoBlocks []*HotnessInfoBlock
 	for blockIdx, node := range relevantNodes {
-		var block *lib.MsgDeSoBlock
-		if cachedBlock, ok := fes.HotFeedBlockCache[*node.Hash]; ok {
-			block = cachedBlock
+		var txns []*lib.MsgDeSoTxn
+		if cachedBlock, ok := fes.HotFeedBlockCache.Get(*node.Hash); ok {
+			txns = cachedBlock
 		} else {
-			block, _ = lib.GetBlock(node.Hash, utxoView.Handle, fes.blockchain.Snapshot())
-			fes.HotFeedBlockCache[*node.Hash] = block
+			block, _ := lib.GetBlock(node.Hash, utxoView.Handle, fes.blockchain.Snapshot())
+			fes.HotFeedBlockCache.Put(*node.Hash, block.Txns)
+			txns = block.Txns
 		}
 		hotnessInfoBlocks = append(hotnessInfoBlocks, &HotnessInfoBlock{
-			Block: block,
+			BlockTxns: txns,
 			// For time decay, we care about how many blocks away from the tip this block is.
 			BlockAge: len(relevantNodes) - blockIdx,
 		})
@@ -465,14 +468,10 @@ func (fes *APIServer) UpdateHotFeedOrderedList(
 		txnsFromMempoolOrderedByTime = append(txnsFromMempoolOrderedByTime, mempoolTxn.Tx)
 	}
 
-	if err != nil {
-		glog.Errorf("Error getting mempool transactions: %v", err)
-	} else if len(txnsFromMempoolOrderedByTime) > 0 {
+	if len(txnsFromMempoolOrderedByTime) > 0 {
 		hotnessInfoBlocks = append(hotnessInfoBlocks, &HotnessInfoBlock{
-			Block: &lib.MsgDeSoBlock{
-				Txns: txnsFromMempoolOrderedByTime,
-			},
-			BlockAge: mempoolBlockHeight,
+			BlockTxns: txnsFromMempoolOrderedByTime,
+			BlockAge:  mempoolBlockHeight,
 		})
 	}
 
@@ -529,8 +528,8 @@ func (fes *APIServer) UpdateHotFeedOrderedList(
 }
 
 type HotnessInfoBlock struct {
-	Block    *lib.MsgDeSoBlock
-	BlockAge int
+	BlockTxns []*lib.MsgDeSoTxn
+	BlockAge  int
 }
 
 func (fes *APIServer) PopulateHotnessInfoMap(
@@ -545,12 +544,12 @@ func (fes *APIServer) PopulateHotnessInfoMap(
 	postInteractionMap := make(map[HotFeedInteractionKey]uint64)
 
 	for _, hotnessInfoBlock := range hotnessInfoBlocks {
-		block := hotnessInfoBlock.Block
-		blockAgee := hotnessInfoBlock.BlockAge
-		if block == nil {
+
+		if hotnessInfoBlock == nil {
 			continue
 		}
-		for _, txn := range block.Txns {
+		blockAgee := hotnessInfoBlock.BlockAge
+		for _, txn := range hotnessInfoBlock.BlockTxns {
 			// We only care about posts created in the specified look-back period. There should always be a
 			// transaction that creates a given post before someone interacts with it. By only
 			// scoring posts that meet this condition, we can restrict the HotFeedOrderedList

routes/server.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	fmt "fmt"
+	"github.com/deso-protocol/core/collections"
 	"io"
 	"io/ioutil"
 	"net"
@@ -426,7 +427,7 @@ type APIServer struct {
 	// The height of the last block evaluated by the hotness routine.
 	HotFeedBlockHeight uint32
 	// A cache to store blocks for the block feed - in order to reduce processing time.
-	HotFeedBlockCache map[lib.BlockHash]*lib.MsgDeSoBlock
+	HotFeedBlockCache *collections.LruCache[lib.BlockHash, []*lib.MsgDeSoTxn]
 	// Map of whitelisted post hashes used for serving the hot feed.
 	// The float64 value is a multiplier than can be modified and used in scoring.
 	HotFeedApprovedPostsToMultipliers             map[lib.BlockHash]float64
@@ -2551,7 +2552,7 @@ func Logger(inner http.Handler, name string) http.Handler {
 
 		inner.ServeHTTP(w, r)
 
-		glog.V(2).Infof(
+		glog.V(3).Infof(
 			"%s\t%s\t%s\t%s",
 			r.Method,
 			r.RequestURI,

routes/transaction.go

@@ -4801,9 +4801,9 @@ func (fes *APIServer) GetTxnConstructionParams(ww http.ResponseWriter, req *http
 func (fes *APIServer) GetCommittedTipBlockInfo(ww http.ResponseWriter, req *http.Request) {
 	// Get the block tip from the blockchain.
 	fes.backendServer.GetBlockchain().ChainLock.RLock()
-	blockTip, idx := fes.backendServer.GetBlockchain().GetCommittedTip()
+	blockTip, exists := fes.backendServer.GetBlockchain().GetCommittedTip()
 	fes.backendServer.GetBlockchain().ChainLock.RUnlock()
-	if idx == -1 {
+	if !exists {
 		_AddBadRequestError(ww, "GetCommittedTipBlockInfo: Problem getting block tip")
 		return
 	}

scripts/tools/toolslib/chain.go

@@ -18,6 +18,8 @@ func OpenDataDir(dataDir string) (*badger.DB, error) {
 	return db, nil
 }
 
+// TODO: This utility function needs to be updated to account for not having the entire chain in
+// memory (no more BEST CHAIN representing the entire history of the blockchain).
 // Returns the best chain associated with a badgerDB handle.
 func GetBestChainFromBadger(syncedDBHandle *badger.DB, params *lib.DeSoParams) ([]*lib.BlockNode, error) {
 	bestBlockHash := lib.DbGetBestHash(syncedDBHandle, nil, lib.ChainTypeDeSoBlock)
@@ -38,7 +40,8 @@ func GetBestChainFromBadger(syncedDBHandle *badger.DB, params *lib.DeSoParams) (
 	}
 
 	// Walk back from the best node to the genesis block and store them all in bestChain.
-	bestChain, err := lib.GetBestChain(tipNode)
+	bi := lib.NewBlockIndex(syncedDBHandle, nil, tipNode)
+	bestChain, err := lib.GetBestChain(tipNode, bi)
 	if err != nil {
 		return nil, errors.Wrap(err, "GetBestChainFromBadger() failed to GetBestChain")
 	}