[Phase 1] Feeds optimization (#164)

* optimize validator by power access in endblock

* fix from comments

* remove get oracle status from loop

* change validator price to store as a list

* grpc query validator prices returns empty slice

* fix test for validator prices query

* change bothan version

* fix from comment

* [Phase] Fix feeds by spec (#165)

* validate submit signals max size by max supported feeds

* make new error type for submit too large signals list

* change transition time to grace period and add description

* change price service to price service config

* power threshold to power step threshold

* move deviation to only query

* change blocks per feeds update to supported feeds update interval

* put power in feeds, use it to calculate deviation, change deviation param name

* remove deviation from median calculation

* change SubmitPrice to SignalPrice

* remove MaxSignalIDCharacters from param

* delete signal total power when the power is zero

* fix lint

* fix from comments

* fix readme to match spec

* remove unnecessary map

* merge validator price query path

* separate check miss report from calculate price fn

* fix from comments

* update some naming

* initiate slice with capacity

* fix from comments

* move MaximumGuaranteeBlockTime to constant.go

* fix CalculatePrice Test

* fix CheckMissReport

* fix lint

* update reference source config

* update SignalID of bench test to fit the new format

* fix grogu errors

* change deviation in thousandth to deviation basis point

Author: colmazia

benchmark/feeds_bench_test.go

@@ -3,6 +3,7 @@ package benchmark
 import (
 	"fmt"
 	"math/rand"
+	"sort"
 	"testing"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -15,8 +16,41 @@ import (
 	oracletypes "github.com/bandprotocol/chain/v2/x/oracle/types"
 )
 
-// benchmark test for delivering MsgSubmitPrices
-func BenchmarkSubmitPricesDeliver(b *testing.B) {
+var (
+	ValidValidator  = sdk.ValAddress("1000000001")
+	ValidValidator2 = sdk.ValAddress("1000000002")
+)
+
+func BenchmarkSortMap(b *testing.B) {
+	b.ResetTimer()
+	b.StopTimer()
+	ba := InitializeBenchmarkApp(b, -1)
+	expValPrices := generateValidatorPrices(300, ValidValidator.String(), ba.Ctx.BlockTime().Unix())
+	valPricesMap := make(map[string]types.ValidatorPrice)
+	for _, valPrice := range expValPrices {
+		valPricesMap[valPrice.SignalID] = valPrice
+	}
+	for i := 0; i < b.N; i++ {
+		for j := 0; j < 2000; j++ {
+			b.StartTimer()
+			keys := make([]string, len(valPricesMap))
+			k := int(0)
+			for key := range valPricesMap {
+				keys[k] = key
+				k++
+			}
+			sort.Strings(keys)
+			valPrices := make([]types.ValidatorPrice, len(valPricesMap))
+			for idx, key := range keys {
+				valPrices[idx] = valPricesMap[key]
+			}
+			b.StopTimer()
+		}
+	}
+}
+
+// benchmark test for delivering MsgSubmitSignalPrices
+func BenchmarkSubmitSignalPricesDeliver(b *testing.B) {
 	b.ResetTimer()
 	b.StopTimer()
 
@@ -31,13 +65,16 @@ func BenchmarkSubmitPricesDeliver(b *testing.B) {
 		err = setupFeeds(ba)
 		require.NoError(b, err)
 
+		err = setupValidatorPriceList(ba, vals)
+		require.NoError(b, err)
+
 		ba.CallBeginBlock()
 
 		txs := []sdk.Tx{}
 		for _, val := range vals {
 			tx := GenSequenceOfTxs(
 				ba.TxConfig,
-				GenMsgSubmitPrices(
+				GenMsgSubmitSignalPrices(
 					val,
 					ba.FeedsKeeper.GetSupportedFeeds(ba.Ctx).Feeds,
 					ba.Ctx.BlockTime().Unix(),
@@ -105,9 +142,8 @@ func setupFeeds(ba *BenchmarkApp) error {
 	feeds := []types.Feed{}
 	for i := int64(0); i < numFeeds; i++ {
 		feeds = append(feeds, types.Feed{
-			SignalID:              fmt.Sprintf("signal.%d", i),
-			Interval:              60,
-			DeviationInThousandth: 5,
+			SignalID: fmt.Sprintf("signal.%d", i),
+			Interval: 60,
 		})
 	}
 	ba.FeedsKeeper.SetSupportedFeeds(ba.Ctx, feeds)
@@ -118,22 +154,51 @@ func setupFeeds(ba *BenchmarkApp) error {
 	return nil
 }
 
+func setupValidatorPriceList(ba *BenchmarkApp, vals []*Account) error {
+	sfs := ba.FeedsKeeper.GetSupportedFeeds(ba.Ctx)
+
+	ba.CallBeginBlock()
+	for valIdx, val := range vals {
+		valPrices := []types.ValidatorPrice{}
+		for _, feed := range sfs.Feeds {
+			valPrices = append(valPrices, types.ValidatorPrice{
+				PriceStatus: types.PriceStatusAvailable,
+				Validator:   val.ValAddress.String(),
+				SignalID:    feed.SignalID,
+				Price:       (10000 + uint64(valIdx)) * 10e9,
+				Timestamp:   ba.Ctx.BlockTime().Unix() - 40,
+			})
+		}
+		err := ba.FeedsKeeper.SetValidatorPriceList(ba.Ctx, val.ValAddress, valPrices)
+		if err != nil {
+			return err
+		}
+	}
+	ba.CallEndBlock()
+	ba.Commit()
+
+	return nil
+}
+
 func setupValidatorPrices(ba *BenchmarkApp, vals []*Account) error {
 	sfs := ba.FeedsKeeper.GetSupportedFeeds(ba.Ctx)
 
 	ba.CallBeginBlock()
-	for _, feed := range sfs.Feeds {
-		for valIdx, val := range vals {
-			err := ba.FeedsKeeper.SetValidatorPrice(ba.Ctx, types.ValidatorPrice{
+	for valIdx, val := range vals {
+		valPrices := []types.ValidatorPrice{}
+		for _, feed := range sfs.Feeds {
+			valPrices = append(valPrices, types.ValidatorPrice{
 				PriceStatus: types.PriceStatusAvailable,
 				Validator:   val.ValAddress.String(),
 				SignalID:    feed.SignalID,
 				Price:       (10000 + uint64(valIdx)) * 10e9,
 				Timestamp:   ba.Ctx.BlockTime().Unix(),
 			})
-			if err != nil {
-				return err
-			}
+		}
+
+		err := ba.FeedsKeeper.SetValidatorPriceList(ba.Ctx, val.ValAddress, valPrices)
+		if err != nil {
+			return err
 		}
 	}
 	ba.CallEndBlock()
@@ -213,3 +278,18 @@ func generateValidators(ba *BenchmarkApp, num int) ([]*Account, error) {
 
 	return accs, nil
 }
+
+// generateValidatorPrices generates a slice of ValidatorPrice with the specified number of elements.
+func generateValidatorPrices(numElements int, validatorAddress string, timestamp int64) []types.ValidatorPrice {
+	prices := make([]types.ValidatorPrice, numElements)
+
+	for i := 0; i < numElements; i++ {
+		prices[i] = types.ValidatorPrice{
+			Validator: validatorAddress,
+			SignalID:  fmt.Sprintf("CS:BAND%d-USD", i),
+			Price:     1e10,
+			Timestamp: timestamp,
+		}
+	}
+	return prices
+}

benchmark/helper_test.go

@@ -71,21 +71,21 @@ func GenMsgRequestData(
 	return []sdk.Msg{&msg}
 }
 
-func GenMsgSubmitPrices(
+func GenMsgSubmitSignalPrices(
 	sender *Account,
 	feeds []feedstypes.Feed,
 	timestamp int64,
 ) []sdk.Msg {
-	prices := []feedstypes.SubmitPrice{}
+	prices := make([]feedstypes.SignalPrice, 0, len(feeds))
 	for _, feed := range feeds {
-		prices = append(prices, feedstypes.SubmitPrice{
+		prices = append(prices, feedstypes.SignalPrice{
 			PriceStatus: feedstypes.PriceStatusAvailable,
 			SignalID:    feed.SignalID,
 			Price:       60000,
 		})
 	}
 
-	msg := feedstypes.NewMsgSubmitPrices(sender.ValAddress.String(), timestamp, prices)
+	msg := feedstypes.NewMsgSubmitSignalPrices(sender.ValAddress.String(), timestamp, prices)
 
 	return []sdk.Msg{msg}
 }

grogu/cmd/run.go

@@ -127,7 +127,7 @@ func createRunE(ctx *context.Context) func(cmd *cobra.Command, args []string) er
 		}
 
 		// Create submit channel
-		submitPriceCh := make(chan []types.SubmitPrice, 300)
+		submitSignalPriceCh := make(chan []types.SignalPrice, 300)
 
 		// Parse validator address
 		valAddr, err := sdk.ValAddressFromBech32(ctx.Config.Validator)
@@ -155,7 +155,7 @@ func createRunE(ctx *context.Context) func(cmd *cobra.Command, args []string) er
 			feedQuerier,
 			bothanService,
 			time.Second,
-			submitPriceCh,
+			submitSignalPriceCh,
 			l,
 			valAddr,
 			&pendingSignalIDs,
@@ -168,7 +168,7 @@ func createRunE(ctx *context.Context) func(cmd *cobra.Command, args []string) er
 			clients,
 			l,
 			ctx.Keyring,
-			submitPriceCh,
+			submitSignalPriceCh,
 			authQuerier,
 			txQuerier,
 			valAddr,

grogu/main.go

@@ -34,7 +34,7 @@ func Main() {
 func createRootCmd(ctx *context.Context) *cobra.Command {
 	rootCmd := &cobra.Command{
 		Use:   "grogu",
-		Short: "BandChain daemon to submit prices for feeds module",
+		Short: "BandChain daemon to submit signal prices for feeds module",
 	}
 
 	rootCmd.AddCommand(

grogu/signaller/signaller.go

@@ -26,15 +26,15 @@ type Signaller struct {
 	bothanClient bothan.Client
 	// How often to check for signal changes
 	interval         time.Duration
-	submitCh         chan<- []types.SubmitPrice
+	submitCh         chan<- []types.SignalPrice
 	logger           *logger.Logger
 	valAddress       sdk.ValAddress
 	pendingSignalIDs *sync.Map
 
 	distributionStartPercentage  uint64
 	distributionOffsetPercentage uint64
 
-	signalIDToFeed           map[string]types.Feed
+	signalIDToFeed           map[string]types.FeedWithDeviation
 	signalIDToValidatorPrice map[string]types.ValidatorPrice
 	params                   *types.Params
 }
@@ -43,7 +43,7 @@ func New(
 	feedQuerier *querier.FeedQuerier,
 	bothanClient bothan.Client,
 	interval time.Duration,
-	submitCh chan<- []types.SubmitPrice,
+	submitCh chan<- []types.SignalPrice,
 	logger *logger.Logger,
 	valAddress sdk.ValAddress,
 	pendingSignalIDs *sync.Map,
@@ -60,7 +60,7 @@ func New(
 		pendingSignalIDs:             pendingSignalIDs,
 		distributionStartPercentage:  distributionStartPercentage,
 		distributionOffsetPercentage: distributionOffsetPercentage,
-		signalIDToFeed:               make(map[string]types.Feed),
+		signalIDToFeed:               make(map[string]types.FeedWithDeviation),
 		signalIDToValidatorPrice:     make(map[string]types.ValidatorPrice),
 		params:                       nil,
 	}
@@ -135,7 +135,7 @@ func (h *Signaller) updateFeedMap() bool {
 		return false
 	}
 
-	h.signalIDToFeed = sliceToMap(resp.SupportedFeeds.Feeds, func(feed types.Feed) string {
+	h.signalIDToFeed = sliceToMap(resp.SupportedFeeds.Feeds, func(feed types.FeedWithDeviation) string {
 		return feed.SignalID
 	})
 
@@ -281,7 +281,7 @@ func (h *Signaller) isPriceValid(
 }
 
 func (h *Signaller) shouldUpdatePrice(
-	feed types.Feed,
+	feed types.FeedWithDeviation,
 	valPrice types.ValidatorPrice,
 	newPrice uint64,
 	now time.Time,
@@ -304,15 +304,15 @@ func (h *Signaller) shouldUpdatePrice(
 	}
 
 	// Check if the price is deviated from the last submission, if it is, add it to the list of prices to update
-	if isDeviated(feed.DeviationInThousandth, valPrice.Price, newPrice) {
+	if isDeviated(feed.DeviationBasisPoint, valPrice.Price, newPrice) {
 		return true
 	}
 
 	return false
 }
 
-func (h *Signaller) prepareSubmitPrices(filteredPrices []*proto.PriceData, now int64) ([]types.SubmitPrice, []string) {
-	submitPrices := make([]types.SubmitPrice, 0, len(filteredPrices))
+func (h *Signaller) prepareSubmitPrices(filteredPrices []*proto.PriceData, now int64) ([]types.SignalPrice, []string) {
+	submitPrices := make([]types.SignalPrice, 0, len(filteredPrices))
 	unusedSignalIDs := make([]string, 0, len(filteredPrices))
 
 	for _, price := range filteredPrices {

grogu/signaller/utils.go

@@ -15,10 +15,10 @@ import (
 )
 
 // isDeviated checks if the deviation between the old price and the new price
-// exceeds a given threshold in thousandths.
+// exceeds a given threshold in basis points.
 
 // Parameters:
-// - deviationInThousandth: the allowable deviation in thousandths (1/1000th)
+// - deviationBasisPoint: the allowable deviation in basis points (1/1000th)
 // - oldPrice: the original price
 // - newPrice: the new price to compare against the original
 //
@@ -28,48 +28,48 @@ import (
 //
 // The deviation is calculated as follows:
 //  1. Calculate the absolute difference between the new price and the old price.
-//  2. Compute the deviation in thousandths by dividing the difference by the
-//     original price and multiplying by 1000.
+//  2. Compute the deviation in basis points by dividing the difference by the
+//     original price and multiplying by 10000.
 //  3. Check if the calculated deviation meets or exceeds the allowable deviation.
-func isDeviated(deviationInThousandth int64, oldPrice uint64, newPrice uint64) bool {
+func isDeviated(deviationBasisPoint int64, oldPrice uint64, newPrice uint64) bool {
 	// Calculate the deviation
 	diff := math.Abs(float64(newPrice) - float64(oldPrice))
-	dev := int64((diff / float64(oldPrice)) * 1000)
+	dev := int64((diff / float64(oldPrice)) * 10000)
 
 	// Check if the new price deviation is meets or exceeds the bounds
-	return deviationInThousandth <= dev
+	return deviationBasisPoint <= dev
 }
 
-func convertPriceData(priceData *proto.PriceData) (types.SubmitPrice, error) {
+func convertPriceData(priceData *proto.PriceData) (types.SignalPrice, error) {
 	switch priceData.PriceStatus {
 	case proto.PriceStatus_PRICE_STATUS_UNSPECIFIED:
 		// This should never happen
 		panic("unspecified price status")
 	case proto.PriceStatus_PRICE_STATUS_UNSUPPORTED:
-		return types.SubmitPrice{
+		return types.SignalPrice{
 			PriceStatus: types.PriceStatusUnsupported,
 			SignalID:    priceData.SignalId,
 			Price:       0,
 		}, nil
 	case proto.PriceStatus_PRICE_STATUS_UNAVAILABLE:
-		return types.SubmitPrice{
+		return types.SignalPrice{
 			PriceStatus: types.PriceStatusUnavailable,
 			SignalID:    priceData.SignalId,
 			Price:       0,
 		}, nil
 	case proto.PriceStatus_PRICE_STATUS_AVAILABLE:
 		price, err := safeConvert(priceData.Price)
 		if err != nil {
-			return types.SubmitPrice{}, err
+			return types.SignalPrice{}, err
 		}
-		return types.SubmitPrice{
+		return types.SignalPrice{
 			PriceStatus: types.PriceStatusAvailable,
 			SignalID:    priceData.SignalId,
 			Price:       price,
 		}, nil
 	default:
 		// Handle unexpected price status
-		return types.SubmitPrice{}, fmt.Errorf("unexpected price status: %v", priceData.PriceStatus)
+		return types.SignalPrice{}, fmt.Errorf("unexpected price status: %v", priceData.PriceStatus)
 	}
 }