Skip to content

Commit ae97fc8

Browse files
PaddyMcPaddyMc
authored
fix: query all proposals (#642)
* fix: query all proposals * chore: add CHANGELOG.md * chore: remove unused dep * fix: add another check for missing codec
1 parent 4c4e1d6 commit ae97fc8

File tree

3 files changed

+337
-1
lines changed

3 files changed

+337
-1
lines changed

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,9 @@ This v0.50.x branch was created at the [eb1a8e88a4ddf77bc2fe235fc07c57016b7386f0
6868
* [#640](https://github.com/osmosis-labs/cosmos-sdk/pull/640) fix: bump IAVL to avoid pruning issues
6969
* [#641](https://github.com/osmosis-labs/cosmos-sdk/pull/641) fix: bump IAVL to skip broken legacy state
7070

71+
* Gov proposals fix
72+
* [#642](https://github.com/osmosis-labs/cosmos-sdk/pull/642) fix: gov proposal fix for missing codec value
73+
7174
* Supply offset fix
7275
* [#629](https://github.com/osmosis-labs/cosmos-sdk/pull/629) fix: add old supply offset functions
7376

‎x/gov/keeper/collections_pagination_gov.go‎

Lines changed: 333 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,333 @@
1+
package keeper
2+
3+
import (
4+
"context"
5+
errorsGo "errors"
6+
"fmt"
7+
"strings"
8+
9+
"cosmossdk.io/collections"
10+
storetypes "cosmossdk.io/store/types"
11+
12+
"github.com/cosmos/cosmos-sdk/types/query"
13+
)
14+
15+
// CollectionFilteredPaginate works in the same way as CollectionPaginate but allows to filter
16+
// results using a predicateFunc.
17+
// A nil predicateFunc means no filtering is applied and results are collected as is.
18+
// TransformFunc is applied only to results which are in range of the pagination and allow
19+
// to convert the result to a different type.
20+
// NOTE: do not collect results using the values/keys passed to predicateFunc as they are not
21+
// guaranteed to be in the pagination range requested.
22+
func CollectionFilteredPaginate[K, V any, C query.Collection[K, V], T any](
23+
ctx context.Context,
24+
coll C,
25+
pageReq *query.PageRequest,
26+
predicateFunc func(key K, value V) (include bool, err error),
27+
transformFunc func(key K, value V) (T, error),
28+
opts ...func(opt *query.CollectionsPaginateOptions[K]),
29+
) (results []T, pageRes *query.PageResponse, err error) {
30+
pageReq = initPageRequestDefaults(pageReq)
31+
32+
offset := pageReq.Offset
33+
key := pageReq.Key
34+
limit := pageReq.Limit
35+
countTotal := pageReq.CountTotal
36+
reverse := pageReq.Reverse
37+
38+
if offset > 0 && key != nil {
39+
return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
40+
}
41+
42+
opt := new(query.CollectionsPaginateOptions[K])
43+
for _, o := range opts {
44+
o(opt)
45+
}
46+
47+
var prefix []byte
48+
if opt.Prefix != nil {
49+
prefix, err = encodeCollKey[K, V](coll, *opt.Prefix)
50+
if err != nil {
51+
return nil, nil, err
52+
}
53+
}
54+
55+
if len(key) != 0 {
56+
results, pageRes, err = collFilteredPaginateByKey(ctx, coll, prefix, key, reverse, limit, predicateFunc, transformFunc)
57+
} else {
58+
results, pageRes, err = collFilteredPaginateNoKey(ctx, coll, prefix, reverse, offset, limit, countTotal, predicateFunc, transformFunc)
59+
}
60+
// invalid iter error is ignored to retain Paginate behavior
61+
if errorsGo.Is(err, collections.ErrInvalidIterator) {
62+
return results, new(query.PageResponse), nil
63+
}
64+
// strip the prefix from next key
65+
if len(pageRes.NextKey) != 0 && prefix != nil {
66+
pageRes.NextKey = pageRes.NextKey[len(prefix):]
67+
}
68+
return results, pageRes, err
69+
}
70+
71+
// initPageRequestDefaults initializes a PageRequest's defaults when those are not set.
72+
func initPageRequestDefaults(pageRequest *query.PageRequest) *query.PageRequest {
73+
// if the PageRequest is nil, use default PageRequest
74+
if pageRequest == nil {
75+
pageRequest = &query.PageRequest{}
76+
}
77+
78+
pageRequestCopy := *pageRequest
79+
if len(pageRequestCopy.Key) == 0 {
80+
pageRequestCopy.Key = nil
81+
}
82+
83+
if pageRequestCopy.Limit == 0 {
84+
pageRequestCopy.Limit = query.DefaultLimit
85+
86+
// count total results when the limit is zero/not supplied
87+
pageRequestCopy.CountTotal = true
88+
}
89+
90+
return &pageRequestCopy
91+
}
92+
93+
// todo maybe move to collections?
94+
func encodeCollKey[K, V any, C query.Collection[K, V]](coll C, key K) ([]byte, error) {
95+
buffer := make([]byte, coll.KeyCodec().Size(key))
96+
_, err := coll.KeyCodec().Encode(buffer, key)
97+
return buffer, err
98+
}
99+
100+
func getCollIter[K, V any, C query.Collection[K, V]](ctx context.Context, coll C, prefix, start []byte, reverse bool) (collections.Iterator[K, V], error) {
101+
// TODO: maybe can be simplified
102+
if reverse {
103+
// if we are in reverse mode, we need to increase the start key
104+
// to include the start key in the iteration.
105+
start = storetypes.PrefixEndBytes(append(prefix, start...))
106+
end := prefix
107+
108+
return coll.IterateRaw(ctx, end, start, collections.OrderDescending)
109+
}
110+
var end []byte
111+
if prefix != nil {
112+
start = append(prefix, start...)
113+
end = storetypes.PrefixEndBytes(prefix)
114+
}
115+
return coll.IterateRaw(ctx, start, end, collections.OrderAscending)
116+
}
117+
118+
func advanceIter[I interface {
119+
Next()
120+
Valid() bool
121+
}](iter I, offset uint64,
122+
) bool {
123+
for i := uint64(0); i < offset; i++ {
124+
if !iter.Valid() {
125+
return false
126+
}
127+
iter.Next()
128+
}
129+
return true
130+
}
131+
132+
// collFilteredPaginateNoKey applies the provided pagination on the collection when the starting key is not set.
133+
// If predicateFunc is nil no filtering is applied.
134+
func collFilteredPaginateNoKey[K, V any, C query.Collection[K, V], T any](
135+
ctx context.Context,
136+
coll C,
137+
prefix []byte,
138+
reverse bool,
139+
offset uint64,
140+
limit uint64,
141+
countTotal bool,
142+
predicateFunc func(K, V) (bool, error),
143+
transformFunc func(K, V) (T, error),
144+
) ([]T, *query.PageResponse, error) {
145+
iterator, err := getCollIter[K, V](ctx, coll, prefix, nil, reverse)
146+
if err != nil {
147+
return nil, nil, err
148+
}
149+
defer iterator.Close()
150+
// we advance the iter equal to the provided offset
151+
if !advanceIter(iterator, offset) {
152+
return nil, nil, collections.ErrInvalidIterator
153+
}
154+
155+
var (
156+
count uint64
157+
nextKey []byte
158+
results []T
159+
)
160+
161+
for ; iterator.Valid(); iterator.Next() {
162+
switch {
163+
// first case, we still haven't found all the results up to the limit
164+
case count < limit:
165+
kv, err := iterator.KeyValue()
166+
if err != nil {
167+
if strings.Contains(err.Error(), "no concrete type registered for type URL") {
168+
// URL /osmosis.concentratedliquidity.v1beta1.CreateConcentratedLiquidityPoolsProposal
169+
continue
170+
}
171+
return nil, nil, err
172+
}
173+
// if no predicate function is specified then we just include the result
174+
if predicateFunc == nil {
175+
transformed, err := transformFunc(kv.Key, kv.Value)
176+
if err != nil {
177+
return nil, nil, err
178+
}
179+
results = append(results, transformed)
180+
count++
181+
182+
// if predicate function is defined we check if the result matches the filtering criteria
183+
} else {
184+
include, err := predicateFunc(kv.Key, kv.Value)
185+
if err != nil {
186+
return nil, nil, err
187+
}
188+
if include {
189+
transformed, err := transformFunc(kv.Key, kv.Value)
190+
if err != nil {
191+
return nil, nil, err
192+
}
193+
results = append(results, transformed)
194+
count++
195+
}
196+
}
197+
// second case, we found all the objects specified within the limit
198+
case count == limit:
199+
key, err := iterator.Key()
200+
if err != nil {
201+
if strings.Contains(err.Error(), "no concrete type registered for type URL") {
202+
// URL /osmosis.concentratedliquidity.v1beta1.CreateConcentratedLiquidityPoolsProposal
203+
continue
204+
}
205+
return nil, nil, err
206+
}
207+
nextKey, err = encodeCollKey[K, V](coll, key)
208+
if err != nil {
209+
return nil, nil, err
210+
}
211+
// if count total was not specified, we return the next key only
212+
if !countTotal {
213+
return results, &query.PageResponse{
214+
NextKey: nextKey,
215+
}, nil
216+
}
217+
// otherwise we fallthrough the third case
218+
fallthrough
219+
// this is the case in which we found all the required results
220+
// but we need to count how many possible results exist in total.
221+
// so we keep increasing the count until the iterator is fully consumed.
222+
case count > limit:
223+
if predicateFunc == nil {
224+
count++
225+
226+
// if predicate function is defined we check if the result matches the filtering criteria
227+
} else {
228+
kv, err := iterator.KeyValue()
229+
if err != nil {
230+
if strings.Contains(err.Error(), "no concrete type registered for type URL") {
231+
// URL /osmosis.concentratedliquidity.v1beta1.CreateConcentratedLiquidityPoolsProposal
232+
continue
233+
}
234+
return nil, nil, err
235+
}
236+
237+
include, err := predicateFunc(kv.Key, kv.Value)
238+
if err != nil {
239+
return nil, nil, err
240+
}
241+
if include {
242+
count++
243+
}
244+
}
245+
}
246+
}
247+
248+
resp := &query.PageResponse{
249+
NextKey: nextKey,
250+
}
251+
252+
if countTotal {
253+
resp.Total = count + offset
254+
}
255+
return results, resp, nil
256+
}
257+
258+
// collFilteredPaginateByKey paginates a collection when a starting key
259+
// is provided in the PageRequest. Predicate is applied only if not nil.
260+
func collFilteredPaginateByKey[K, V any, C query.Collection[K, V], T any](
261+
ctx context.Context,
262+
coll C,
263+
prefix []byte,
264+
key []byte,
265+
reverse bool,
266+
limit uint64,
267+
predicateFunc func(key K, value V) (bool, error),
268+
transformFunc func(key K, value V) (transformed T, err error),
269+
) (results []T, pageRes *query.PageResponse, err error) {
270+
iterator, err := getCollIter[K, V](ctx, coll, prefix, key, reverse)
271+
if err != nil {
272+
return nil, nil, err
273+
}
274+
defer iterator.Close()
275+
276+
var (
277+
count uint64
278+
nextKey []byte
279+
)
280+
281+
for ; iterator.Valid(); iterator.Next() {
282+
// if we reached the specified limit
283+
// then we get the next key, and we exit the iteration.
284+
if count == limit {
285+
concreteKey, err := iterator.Key()
286+
if err != nil {
287+
if strings.Contains(err.Error(), "no concrete type registered for type URL") {
288+
// URL /osmosis.concentratedliquidity.v1beta1.CreateConcentratedLiquidityPoolsProposal
289+
continue
290+
}
291+
return nil, nil, err
292+
}
293+
294+
nextKey, err = encodeCollKey[K, V](coll, concreteKey)
295+
if err != nil {
296+
return nil, nil, err
297+
}
298+
break
299+
}
300+
301+
kv, err := iterator.KeyValue()
302+
if err != nil {
303+
return nil, nil, err
304+
}
305+
// if no predicate is specified then we just append the result
306+
if predicateFunc == nil {
307+
transformed, err := transformFunc(kv.Key, kv.Value)
308+
if err != nil {
309+
return nil, nil, err
310+
}
311+
results = append(results, transformed)
312+
// if predicate is applied we execute the predicate function
313+
// and append only if predicateFunc yields true.
314+
} else {
315+
include, err := predicateFunc(kv.Key, kv.Value)
316+
if err != nil {
317+
return nil, nil, err
318+
}
319+
if include {
320+
transformed, err := transformFunc(kv.Key, kv.Value)
321+
if err != nil {
322+
return nil, nil, err
323+
}
324+
results = append(results, transformed)
325+
}
326+
}
327+
count++
328+
}
329+
330+
return results, &query.PageResponse{
331+
NextKey: nextKey,
332+
}, nil
333+
}

‎x/gov/keeper/grpc_query.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ func (q queryServer) Proposal(ctx context.Context, req *v1.QueryProposalRequest)
5656

5757
// Proposals implements the Query/Proposals gRPC method
5858
func (q queryServer) Proposals(ctx context.Context, req *v1.QueryProposalsRequest) (*v1.QueryProposalsResponse, error) {
59-
filteredProposals, pageRes, err := query.CollectionFilteredPaginate(ctx, q.k.Proposals, req.Pagination, func(key uint64, p v1.Proposal) (include bool, err error) {
59+
filteredProposals, pageRes, err := CollectionFilteredPaginate(ctx, q.k.Proposals, req.Pagination, func(key uint64, p v1.Proposal) (include bool, err error) {
6060
matchVoter, matchDepositor, matchStatus := true, true, true
6161

6262
// match status (if supplied/valid)

0 commit comments

Comments
 (0)