fix(query): add pagination support (#1976)

* Add pagination utility package with encoding/decoding functionality
* Implement pagination support in cert, deployment, and market queries
* Update validation utilities for address handling
* Add comprehensive test coverage for pagination functionality
* chore(mod): bump cometbft, refs akash-network/support#291

Signed-off-by: Artur Troian <[email protected]>
Co-authored-by: Artur Troian <[email protected]>

Author: troian

go.mod

@@ -68,7 +68,7 @@ replace (
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	// use cometBFT system fork of tendermint with akash patches
 
-	github.com/tendermint/tendermint => github.com/akash-network/cometbft v0.34.27-akash.2
+	github.com/tendermint/tendermint => github.com/akash-network/cometbft v0.34.27-akash.3
 
 	github.com/zondax/hid => github.com/troian/hid v0.13.2
 	github.com/zondax/ledger-go => github.com/akash-network/ledger-go v0.14.3

go.sum

@@ -78,8 +78,8 @@ github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/akash-network/akash-api v0.0.75 h1:h9RZemWa7JqMGYb3nVRhRgP4xZnACIy0yN7de60JLyg=
 github.com/akash-network/akash-api v0.0.75/go.mod h1:pvoHHEQbt63+U+HUSTjssZ1nUJ8sJuWtHCu6ztaXcqo=
-github.com/akash-network/cometbft v0.34.27-akash.2 h1:2hKEcX+cIv/OLAJ82gBWdkZlVWn+8JUYs4GrDoPAOhU=
-github.com/akash-network/cometbft v0.34.27-akash.2/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
+github.com/akash-network/cometbft v0.34.27-akash.3 h1:ObmkKrMybIuRLPcwPwUMJ8Pllsr+Gsve443mkJsonMA=
+github.com/akash-network/cometbft v0.34.27-akash.3/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw=
 github.com/akash-network/cosmos-sdk v0.45.16-akash.3 h1:QiHOQ1ACzCvAEXRlzGNQhp9quWLOowE104D0uESGrEk=
 github.com/akash-network/cosmos-sdk v0.45.16-akash.3/go.mod h1:NTnk/GuQdFyfk/iGFxDAgQH9fwcbRW/hREap6qaPg48=
 github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=

tests/upgrade/upgrade_test.go

@@ -254,6 +254,7 @@ type nodeInitParams struct {
 	grpcPort    uint16
 	grpcWebPort uint16
 	pprofPort   uint16
+	apiPort     uint16
 }
 
 var (
@@ -446,6 +447,7 @@ func TestUpgrade(t *testing.T) {
 			grpcPort:    9090 + uint16(idx*3),
 			grpcWebPort: 9091 + uint16(idx*3),
 			pprofPort:   6060 + uint16(idx),
+			apiPort:     1317 + uint16(idx),
 		}
 	}
 
@@ -488,6 +490,7 @@ func TestUpgrade(t *testing.T) {
 				fmt.Sprintf("AKASH_RPC_PPROF_LADDR=%s:%d", listenAddr, params.pprofPort),
 				fmt.Sprintf("AKASH_GRPC_ADDRESS=%s:%d", listenAddr, params.grpcPort),
 				fmt.Sprintf("AKASH_GRPC_WEB_ADDRESS=%s:%d", listenAddr, params.grpcWebPort),
+				fmt.Sprintf("AKASH_API_ADDRESS=tcp://%s:%d", listenAddr, params.apiPort),
 				"DAEMON_NAME=akash",
 				"DAEMON_RESTART_AFTER_UPGRADE=true",
 				"DAEMON_ALLOW_DOWNLOAD_BINARIES=true",
@@ -508,6 +511,7 @@ func TestUpgrade(t *testing.T) {
 				"AKASH_TX_INDEX_INDEXER=null",
 				"AKASH_GRPC_ENABLE=true",
 				"AKASH_GRPC_WEB_ENABLE=true",
+				"AKASH_API_ENABLE=true",
 			},
 		}
 	}

util/query/pagination.go

@@ -0,0 +1,143 @@
+package query
+
+import (
+	"encoding/binary"
+	"fmt"
+	"hash/crc32"
+
+	"github.com/akash-network/node/util/validation"
+)
+
+var (
+	ErrInvalidPaginationKey = fmt.Errorf("pagination: invalid key")
+)
+
+// DecodePaginationKey parses the pagination key and returns the states, prefix and key to be used by the FilteredPaginate
+func DecodePaginationKey(key []byte) ([]byte, []byte, []byte, []byte, error) {
+	if len(key) < 5 {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid key length", ErrInvalidPaginationKey)
+	}
+
+	expectedChecksum := binary.BigEndian.Uint32(key)
+
+	key = key[4:]
+
+	checksum := crc32.ChecksumIEEE(key)
+
+	if expectedChecksum != checksum {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid checksum, 0x%08x != 0x%08x", ErrInvalidPaginationKey, expectedChecksum, checksum)
+	}
+
+	statesC := int(key[0])
+	key = key[1:]
+
+	if len(key) < statesC {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+	}
+
+	states := make([]byte, 0, statesC)
+	for _, state := range key[:statesC] {
+		states = append(states, state)
+	}
+
+	key = key[len(states):]
+
+	if len(key) < 1 {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+	}
+
+	prefixLength := int(key[0])
+
+	key = key[1:]
+
+	if len(key) < prefixLength {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+	}
+
+	prefix := key[:prefixLength]
+
+	key = key[prefixLength:]
+
+	if len(key) == 0 {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+	}
+
+	keyLength := int(key[0])
+	key = key[1:]
+
+	if len(key) < keyLength {
+		return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+	}
+
+	pkey := key[:keyLength]
+
+	key = key[keyLength:]
+	var unsolicited []byte
+
+	if len(key) > 0 {
+		keyLength = int(key[0])
+		key = key[1:]
+
+		if len(key) != keyLength {
+			return nil, nil, nil, nil, fmt.Errorf("%w: invalid state length", ErrInvalidPaginationKey)
+		}
+
+		unsolicited = key
+	}
+
+	return states, prefix, pkey, unsolicited, nil
+}
+
+func EncodePaginationKey(states, prefix, key, unsolicited []byte) ([]byte, error) {
+	if len(states) == 0 {
+		return nil, fmt.Errorf("%w: states cannot be empty", ErrInvalidPaginationKey)
+	}
+
+	if len(prefix) == 0 {
+		return nil, fmt.Errorf("%w: prefix cannot be empty", ErrInvalidPaginationKey)
+	}
+
+	if len(key) == 0 {
+		return nil, fmt.Errorf("%w: key cannot be empty", ErrInvalidPaginationKey)
+	}
+
+	// 4 bytes for checksum
+	// 1 byte for states count
+	// len(states) bytes for states
+	// 1 byte for prefix length
+	// len(prefix) bytes for prefix
+	// 1 byte for key length
+	// len(key) bytes for key
+	encLen := 4 + 1 + len(states) + 1 + len(prefix) + 1 + len(key)
+
+	if len(unsolicited) > 0 {
+		encLen += 1 + len(unsolicited)
+	}
+
+	buf := make([]byte, encLen)
+
+	data := buf[4:]
+
+	tmp := validation.MustEncodeWithLengthPrefix(states)
+	copy(data, tmp)
+
+	offset := len(tmp)
+	tmp = validation.MustEncodeWithLengthPrefix(prefix)
+
+	copy(data[offset:], tmp)
+	offset += len(tmp)
+
+	tmp = validation.MustEncodeWithLengthPrefix(key)
+	copy(data[offset:], tmp)
+
+	if len(unsolicited) > 0 {
+		offset += len(tmp)
+		tmp = validation.MustEncodeWithLengthPrefix(unsolicited)
+		copy(data[offset:], tmp)
+	}
+
+	checksum := crc32.ChecksumIEEE(data)
+	binary.BigEndian.PutUint32(buf, checksum)
+
+	return buf, nil
+}