feat(mempool): backport Lanes and DOG from upstream

.changelog/v1.0.0/breaking-changes/2803-mempool-NewCListMempool.md

@@ -0,0 +1,2 @@
+- `[mempool]` Add new parameter `lanesInfo *LanesInfo` to `NewCListMempool`
+([\#2803](https://github.com/cometbft/cometbft/issues/2803))

.changelog/v1.0.0/breaking-changes/2803-mempool-replace-iterator.md

@@ -0,0 +1,2 @@
+- `[mempool]` Removed `CListIterator`; use `BlockingIterator` instead
+  ([\#2803](https://github.com/cometbft/cometbft/issues/2803)).

.changelog/v1.0.0/breaking-changes/2803-proto-mempool-lane.md

@@ -0,0 +1,3 @@
+- `[types/proto]` Extend `CheckTxResponse` with new `lane_id` field and `InfoResponse` with
+  `lane_priorities` and `default_lane` fields
+  ([#2803](https://github.com/cometbft/cometbft/issues/2803))

.changelog/v1.0.0/breaking-changes/2803-remove-TxsFront-TxsWaitChan.md

@@ -0,0 +1,2 @@
+- `[mempool]` Remove methods `TxsFront` and `TxsWaitChan` from `CListMempool`. They should be
+  replaced by the new iterators ([\#2803](https://github.com/cometbft/cometbft/issues/2803)).

.changelog/v1.0.0/breaking-changes/3297-mempool-interface-senders.md

@@ -0,0 +1,2 @@
+- `[mempool]` Add new `GetSenders` method to `Mempool` interface, and `Senders` method to `Entry`
+  interface. ([#3297](https://github.com/cometbft/cometbft/issue/3297)).

.changelog/v1.0.0/breaking-changes/3297-proto-mempool-message.md

@@ -0,0 +1,2 @@
+- `[proto]` Mempool proto version upgraded to `v2`. 
+([#3297](https://github.com/cometbft/cometbft/issue/3297)).

.changelog/v1.0.0/breaking-changes/3634-node-handshake-goapi.md

@@ -0,0 +1,2 @@
+- `[consensus/replay]` `Handshake` now takes an additional parameter of type `*abci.InfoResponse` as input
+  ([#3634](https://github.com/cometbft/cometbft/pull/3634))

.changelog/v1.0.0/breaking-changes/3659-mempool-add-contains-method.md

@@ -0,0 +1,2 @@
+- `[mempool]` Add a new `Contains` method to the `Mempool` interface.
+  ([\#3659](https://github.com/cometbft/cometbft/pull/3659))

.changelog/v1.0.0/breaking-changes/3980-kvstore-new-application.md

@@ -0,0 +1,2 @@
+- `[kvstore]` Function `NewApplication` now has an extra `lanes map[string]uint32` parameter
+  ([\#3980](https://github.com/cometbft/cometbft/pull/3980))

.changelog/v1.0.0/bug-fixes/3694-mempool-update-variables-atomically.md

@@ -0,0 +1,3 @@
+- `[mempool]` Fix race condition when accessing entries by updating variables in
+  `CListMempool` atomically.
+  ([\#3694](https://github.com/cometbft/cometbft/issues/3694))

.changelog/v1.0.0/deprecations/3506-mempool-metrics-size.md

@@ -0,0 +1,4 @@
+- `[mempool/metrics]` Mark metrics `mempool_size` and `mempool_size_bytes` as
+  deprecated, as now they can be obtain, respectively, as the sum of
+  `mempool_lane_size` and `mempool_lane_bytes`
+  ([\#3506](https://github.com/cometbft/cometbft/issue/3506)).

.changelog/v1.0.0/features/2803-mempool-lanes.md

@@ -0,0 +1,2 @@
+- `[mempool]` Add Lanes to the mempool for providing Quality of Service guarantees
+([#2803](https://github.com/cometbft/cometbft/issues/2803))

.changelog/v1.0.0/features/2803-proto-mempool-lane.md

@@ -0,0 +1,3 @@
+- `[types/proto]` Extend `CheckTxResponse` with new `lane_id` field and `InfoResponse` with
+  `lane_priorities` and `default_lane` fields
+  ([#2803](https://github.com/cometbft/cometbft/issues/2803))

.changelog/v1.0.0/features/3079-unconfirmed-tx.md

@@ -0,0 +1,2 @@
+- `[rpc]` Add `unconfirmed_tx` to support query mempool transaction by transaction hash.
+  ([\#3079](https://github.com/cometbft/cometbft/pull/3079))

.changelog/v1.0.0/features/3297-config-mempool-dog-protocol.md

@@ -0,0 +1,3 @@
+- `[config]` For the DOG gossip protocol, add `dog_protocol_enabled`, `dog_target_redundancy`,
+  `dog_adjust_interval` to the `mempool` configuration section
+  ([\#3297](https://github.com/cometbft/cometbft/issues/3297)).

.changelog/v1.0.0/features/3297-mempool-dog-protocol.md

@@ -0,0 +1,2 @@
+- `[mempool]` Add new Dynamic Optimal Graph (DOG) gossip protocol
+  ([#3297](https://github.com/cometbft/cometbft/issue/3297)).

.changelog/v1.0.0/features/3297-metrics-dog-protocol.md

@@ -0,0 +1,2 @@
+- `[metrics]` Add new mempool metrics `DisabledRoutes` and `Redundancy` for the DOG protocol
+  ([#3297](https://github.com/cometbft/cometbft/issue/3297)).

.changelog/v1.0.0/features/3297-proto-mempool-dog-protocol.md

@@ -0,0 +1,2 @@
+- `[proto]` Add new proto messages `HaveTx` and `ResetRoute` for the mempool DOG protocol
+  ([#3297](https://github.com/cometbft/cometbft/issue/3297)).

.changelog/v1.0.0/features/3506-mempool-lanes-add-metrics.md

@@ -0,0 +1,2 @@
+- `[metrics]` Add new mempool metrics `lane_size`, `lane_bytes`, and `tx_life_span`
+  ([#3506](https://github.com/cometbft/cometbft/issue/3506)).

.changelog/v1.0.0/features/3622-e2e-mempool-lanes.md

@@ -0,0 +1,2 @@
+- `[e2e]` Added support for mempool lanes in e2e. 
+  ([#3622](https://github.com/cometbft/cometbft/pull/3622))

.changelog/v1.0.0/features/3622-kvstore-mempool-lanes.md

@@ -0,0 +1,2 @@
+- `[kvstore]` Extended `CheckTx` in kvstoreApp to support mempool lanes. 
+  ([#3622](https://github.com/cometbft/cometbft/pull/3622))

.changelog/v1.0.0/features/3634-query-app-for-lanes-info.md

@@ -0,0 +1,2 @@
+- `[node]` Move the ABCI `Info` call from the `Handshake` function to the `NewNodeWithCliParams` function.
+  ([#3634](https://github.com/cometbft/cometbft/pull/3634))

.changelog/v1.0.0/features/3825-e2e-lane-weights.md

@@ -0,0 +1,3 @@
+- `[e2e]` Add `load_lane_weights` option to manifest for generating transactions with
+  lanes picked randomly and proportional to their weight.
+  ([\#3825](https://github.com/cometbft/cometbft/pull/3825)).

.changelog/v1.0.0/features/4005-e2e-lanes-manifest.md

@@ -0,0 +1,2 @@
+- `[e2e]` Add `lanes` and `no_lanes` to manifest to customize the list of lanes the app should use
+([#4005](https://github.com/cometbft/cometbft/issues/4005))

abci/example/kvstore/kvstore.go

@@ -6,6 +6,7 @@ import (
 	"encoding/base64"
 	"encoding/binary"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -26,6 +27,7 @@ var (
 const (
 	ValidatorPrefix        = "val="
 	AppVersion      uint64 = 1
+	defaultLane     string = "default"
 )
 
 var _ types.Application = (*Application)(nil)
@@ -48,31 +50,71 @@ type Application struct {
 	// If true, the app will generate block events in BeginBlock. Used to test the event indexer
 	// Should be false by default to avoid generating too much data.
 	genBlockEvents bool
+
+	lanes          map[string]uint32
+	lanePriorities []uint32
 }
 
-// NewApplication creates an instance of the kvstore from the provided database.
-func NewApplication(db dbm.DB) *Application {
+// NewApplication creates an instance of the kvstore from the provided database,
+// with the given lanes and priorities.
+func NewApplication(db dbm.DB, lanes map[string]uint32) *Application {
+	lanePriorities := make([]uint32, 0, len(lanes))
+	for _, p := range lanes {
+		lanePriorities = append(lanePriorities, p)
+	}
+
 	return &Application{
 		logger:             log.NewNopLogger(),
 		state:              loadState(db),
 		valAddrToPubKeyMap: make(map[string]crypto.PubKey),
+		lanes:              lanes,
+		lanePriorities:     lanePriorities,
 	}
 }
 
-// NewPersistentApplication creates a new application using the goleveldb database engine.
-func NewPersistentApplication(dbDir string) *Application {
+// newDB creates a DB engine for persisting the application state.
+func newDB(dbDir string) *dbm.PebbleDB {
 	name := "kvstore"
 	db, err := dbm.NewPebbleDB(name, dbDir)
 	if err != nil {
 		panic(fmt.Errorf("failed to create persistent app at %s: %w", dbDir, err))
 	}
-	return NewApplication(db)
+	return db
 }
 
-// NewInMemoryApplication creates a new application from an in memory database.
-// Nothing will be persisted.
+// NewPersistentApplication creates a new application using the pebbledb
+// database engine and default lanes.
+func NewPersistentApplication(dbDir string) *Application {
+	return NewApplication(newDB(dbDir), DefaultLanes())
+}
+
+// NewPersistentApplicationWithoutLanes creates a new application using the
+// pebbledb database engine and without lanes.
+func NewPersistentApplicationWithoutLanes(dbDir string) *Application {
+	return NewApplication(newDB(dbDir), nil)
+}
+
+// NewInMemoryApplication creates a new application from an in memory database
+// that uses default lanes. Nothing will be persisted.
 func NewInMemoryApplication() *Application {
-	return NewApplication(dbm.NewMemDB())
+	return NewApplication(dbm.NewMemDB(), DefaultLanes())
+}
+
+// NewInMemoryApplication creates a new application from an in memory database
+// and without lanes. Nothing will be persisted.
+func NewInMemoryApplicationWithoutLanes() *Application {
+	return NewApplication(dbm.NewMemDB(), nil)
+}
+
+// DefaultLanes returns a map from lane names to their priorities. Priority 0 is
+// reserved. The higher the value, the higher the priority.
+func DefaultLanes() map[string]uint32 {
+	return map[string]uint32{
+		"val":       9, // for validator updates
+		"foo":       7,
+		defaultLane: 3,
+		"bar":       1,
+	}
 }
 
 func (app *Application) SetGenBlockEvents() {
@@ -96,12 +138,18 @@ func (app *Application) Info(context.Context, *types.InfoRequest) (*types.InfoRe
 		}
 	}
 
+	defLane := ""
+	if len(app.lanes) != 0 {
+		defLane = defaultLane
+	}
 	return &types.InfoResponse{
 		Data:             fmt.Sprintf("{\"size\":%v}", app.state.Size),
 		Version:          version.ABCIVersion,
 		AppVersion:       AppVersion,
 		LastBlockHeight:  app.state.Height,
 		LastBlockAppHash: app.state.Hash(),
+		LanePriorities:   app.lanes,
+		DefaultLane:      defLane,
 	}, nil
 }
 
@@ -126,18 +174,62 @@ func (app *Application) InitChain(_ context.Context, req *types.InitChainRequest
 // - Contains one and only one `=`
 // - `=` is not the first or last byte.
 // - if key is `val` that the validator update transaction is also valid.
-func (*Application) CheckTx(_ context.Context, req *types.CheckTxRequest) (*types.CheckTxResponse, error) {
+func (app *Application) CheckTx(_ context.Context, req *types.CheckTxRequest) (*types.CheckTxResponse, error) {
 	// If it is a validator update transaction, check that it is correctly formatted
 	if isValidatorTx(req.Tx) {
 		if _, _, _, err := parseValidatorTx(req.Tx); err != nil {
-			//nolint:nilerr
-			return &types.CheckTxResponse{Code: CodeTypeInvalidTxFormat}, nil
+			return &types.CheckTxResponse{Code: CodeTypeInvalidTxFormat}, nil //nolint:nilerr // error is not nil but it returns nil
 		}
 	} else if !isValidTx(req.Tx) {
 		return &types.CheckTxResponse{Code: CodeTypeInvalidTxFormat}, nil
 	}
 
-	return &types.CheckTxResponse{Code: CodeTypeOK, GasWanted: 1}, nil
+	if len(app.lanes) == 0 {
+		return &types.CheckTxResponse{Code: CodeTypeOK, GasWanted: 1}, nil
+	}
+	lane := assignLane(req.Tx)
+	return &types.CheckTxResponse{Code: CodeTypeOK, GasWanted: 1, LaneId: lane}, nil
+}
+
+// assignLane deterministically computes a lane for the given tx.
+func assignLane(tx []byte) string {
+	lane := defaultLane
+	if isValidatorTx(tx) {
+		return "val" // priority 9
+	}
+	key, _, err := parseTx(tx)
+	if err != nil {
+		return lane
+	}
+
+	// If the transaction key is an integer (for example, a transaction of the
+	// form 2=2), we will assign a lane. Any other type of transaction will go
+	// to the default lane.
+	keyInt, err := strconv.Atoi(key)
+	if err != nil {
+		return lane
+	}
+
+	switch {
+	case keyInt%11 == 0:
+		return "foo" // priority 7
+	case keyInt%3 == 0:
+		return "bar" // priority 1
+	default:
+		return lane // priority 3
+	}
+}
+
+// parseTx parses a tx in 'key=value' format into a key and value.
+func parseTx(tx []byte) (key, value string, err error) {
+	parts := bytes.Split(tx, []byte("="))
+	if len(parts) != 2 {
+		return "", "", fmt.Errorf("invalid tx format: %q", string(tx))
+	}
+	if len(parts[0]) == 0 {
+		return "", "", errors.New("key cannot be empty")
+	}
+	return string(parts[0]), string(parts[1]), nil
 }
 
 // Tx must have a format like key:value or key=value. That is:

abci/example/kvstore/kvstore_test.go

@@ -238,6 +238,34 @@ func TestCheckTx(t *testing.T) {
 	}
 }
 
+func TestClientAssignLane(t *testing.T) {
+	val := RandVal()
+
+	testCases := []struct {
+		lane string
+		tx   []byte
+	}{
+		{"foo", NewTx("0", "0")},
+		{defaultLane, NewTx("1", "1")},
+		{defaultLane, NewTx("2", "2")},
+		{"bar", NewTx("3", "3")},
+		{defaultLane, NewTx("4", "4")},
+		{defaultLane, NewTx("5", "5")},
+		{"bar", NewTx("6", "6")},
+		{defaultLane, NewTx("7", "7")},
+		{defaultLane, NewTx("8", "8")},
+		{"bar", NewTx("9", "9")},
+		{defaultLane, NewTx("10", "10")},
+		{"foo", NewTx("11", "11")},
+		{"bar", NewTx("12", "12")},
+		{"val", MakeValSetChangeTx(val)},
+	}
+
+	for idx, tc := range testCases {
+		require.Equal(t, tc.lane, assignLane(tc.tx), idx)
+	}
+}
+
 func TestClientServer(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()