Skip to content

Commit 2633bba

Browse files
haarghaarg
authored
Merge pull request #1313 from metacpan/haarg/normalize-limit-pagination
normalize pagination parameters and add size limits
2 parents 9a84ef7 + 453f770 commit 2633bba

File tree

11 files changed

+115
-43
lines changed

11 files changed

+115
-43
lines changed

‎lib/MetaCPAN/API/Controller/Search.pm‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,13 @@ sub web {
1717
return unless $c->openapi->valid_input;
1818
my $args = $c->validation->output;
1919

20-
my @search = ( @{$args}{qw/q from size/} );
21-
push @search, $args->{collapsed} if exists $args->{collapsed};
22-
my $results = $c->model->search->search_web(@search);
20+
my $query = $args->{q};
21+
my $size = $args->{page_size} // $args->{size} // 20;
22+
my $page = $args->{page} // ( 1 + int( ( $args->{from} // 0 ) / $size ) );
23+
my $collapsed = $args->{collapsed};
24+
25+
my $results
26+
= $c->model->search->search_web( $query, $page, $size, $collapsed );
2327

2428
return $c->render( json => $results );
2529
}

‎lib/MetaCPAN/Query/Favorite.pm‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package MetaCPAN::Query::Favorite;
33
use MetaCPAN::Moose;
44

55
use MetaCPAN::ESConfig qw( es_doc_path );
6-
use MetaCPAN::Util qw(hit_total);
6+
use MetaCPAN::Util qw( MAX_RESULT_WINDOW hit_total );
77

88
with 'MetaCPAN::Query::Role::Common';
99

@@ -148,6 +148,14 @@ sub recent {
148148
$page //= 1;
149149
$size //= 100;
150150

151+
if ( $page * $size >= MAX_RESULT_WINDOW ) {
152+
return +{
153+
favorites => [],
154+
took => 0,
155+
total => 0,
156+
};
157+
}
158+
151159
my $favs = $self->es->search(
152160
es_doc_path('favorite'),
153161
body => {

‎lib/MetaCPAN/Query/Release.pm‎

Lines changed: 39 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ use MetaCPAN::Moose;
44

55
use MetaCPAN::ESConfig qw( es_doc_path );
66
use MetaCPAN::Util
7-
qw( hit_total single_valued_arrayref_to_scalar true false );
7+
qw( MAX_RESULT_WINDOW hit_total single_valued_arrayref_to_scalar true false );
88

99
with 'MetaCPAN::Query::Role::Common';
1010

@@ -403,10 +403,18 @@ sub by_author_and_names {
403403
}
404404

405405
sub by_author {
406-
my ( $self, $pauseid, $size, $page ) = @_;
406+
my ( $self, $pauseid, $page, $size ) = @_;
407407
$size //= 1000;
408408
$page //= 1;
409409

410+
if ( $page * $size >= MAX_RESULT_WINDOW ) {
411+
return {
412+
releases => [],
413+
took => 0,
414+
total => 0,
415+
};
416+
}
417+
410418
my $body = {
411419
query => {
412420
bool => {
@@ -495,10 +503,18 @@ sub latest_by_author {
495503
}
496504

497505
sub all_by_author {
498-
my ( $self, $author, $size, $page ) = @_;
506+
my ( $self, $author, $page, $size ) = @_;
499507
$size //= 100;
500508
$page //= 1;
501509

510+
if ( $page * $size >= MAX_RESULT_WINDOW ) {
511+
return {
512+
releases => [],
513+
took => 0,
514+
total => 0,
515+
};
516+
}
517+
502518
my $body = {
503519
query => { term => { author => uc($author) } },
504520
sort => [ { date => 'desc' } ],
@@ -639,11 +655,11 @@ sub top_uploaders {
639655
sub requires {
640656
my ( $self, $module, $page, $page_size, $sort ) = @_;
641657
return $self->_get_depended_releases( [$module], $page, $page_size,
642-
$page_size, $sort );
658+
$sort );
643659
}
644660

645661
sub reverse_dependencies {
646-
my ( $self, $distribution, $page, $page_size, $size, $sort ) = @_;
662+
my ( $self, $distribution, $page, $page_size, $sort ) = @_;
647663

648664
# get the latest release of given distribution
649665
my $release = $self->_get_latest_release($distribution) || return;
@@ -653,7 +669,7 @@ sub reverse_dependencies {
653669

654670
# return releases depended on those modules
655671
return $self->_get_depended_releases( $modules, $page, $page_size,
656-
$size, $sort );
672+
$sort );
657673
}
658674

659675
sub _get_latest_release {
@@ -725,10 +741,18 @@ sub _fix_sort_value {
725741
}
726742

727743
sub _get_depended_releases {
728-
my ( $self, $modules, $page, $page_size, $size, $sort ) = @_;
744+
my ( $self, $modules, $page, $page_size, $sort ) = @_;
729745
$page //= 1;
730746
$page_size //= 50;
731747

748+
if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
749+
return +{
750+
data => [],
751+
took => 0,
752+
total => 0,
753+
};
754+
}
755+
732756
$sort = _fix_sort_value($sort);
733757

734758
my $dependency_filter = {
@@ -770,8 +794,8 @@ sub _get_depended_releases {
770794
],
771795
},
772796
},
773-
size => $size || $page_size,
774-
from => $page * $page_size - $page_size,
797+
size => $page_size,
798+
from => ( $page - 1 ) * $page_size,
775799
sort => $sort,
776800
}
777801
);
@@ -784,22 +808,20 @@ sub _get_depended_releases {
784808
}
785809

786810
sub recent {
787-
my ( $self, $page, $page_size, $type ) = @_;
811+
my ( $self, $type, $page, $page_size ) = @_;
788812
$page //= 1;
789813
$page_size //= 10000;
790814
$type //= '';
791815

792-
my $query;
793-
my $from = ( $page - 1 ) * $page_size;
794-
795-
if ( $from + $page_size > 10000 ) {
796-
return {
816+
if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
817+
return +{
797818
releases => [],
798-
total => 0,
799819
took => 0,
820+
total => 0,
800821
};
801822
}
802823

824+
my $query;
803825
if ( $type eq 'n' ) {
804826
$query = {
805827
constant_score => {
@@ -829,7 +851,7 @@ sub recent {
829851

830852
my $body = {
831853
size => $page_size,
832-
from => $from,
854+
from => ( $page - 1 ) * $page_size,
833855
query => $query,
834856
_source =>
835857
[qw(name author status abstract date distribution maturity)],

‎lib/MetaCPAN/Query/Search.pm‎

Lines changed: 19 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ use List::Util qw( min uniq );
88
use Log::Contextual qw( :log :dlog );
99
use MetaCPAN::ESConfig qw( es_doc_path );
1010
use MetaCPAN::Types::TypeTiny qw( Object Str );
11-
use MetaCPAN::Util qw( hit_total true false );
11+
use MetaCPAN::Util qw( MAX_RESULT_WINDOW hit_total true false );
1212
use MooseX::StrictConstructor;
1313

1414
with 'MetaCPAN::Query::Role::Common';
@@ -54,11 +54,20 @@ sub search_for_first_result {
5454
=cut
5555

5656
sub search_web {
57-
my ( $self, $search_term, $from, $page_size, $collapsed,
57+
my ( $self, $search_term, $page, $page_size, $collapsed,
5858
$max_collapsed_hits )
5959
= @_;
6060
$page_size //= 20;
61-
$from //= 0;
61+
$page //= 1;
62+
63+
if ( $page * $page_size >= MAX_RESULT_WINDOW ) {
64+
return {
65+
results => [],
66+
total => 0,
67+
tool => 0,
68+
colapsed => $collapsed ? true : false,
69+
};
70+
}
6271

6372
$search_term =~ s{([+=><!&|\(\)\{\}[\]\^"~*?\\/])}{\\$1}g;
6473

@@ -78,15 +87,15 @@ sub search_web {
7887

7988
my $results
8089
= $collapsed // $search_term !~ /(distribution|module\.name\S*):/
81-
? $self->_search_collapsed( $search_term, $from, $page_size,
90+
? $self->_search_collapsed( $search_term, $page, $page_size,
8291
$max_collapsed_hits )
83-
: $self->_search_expanded( $search_term, $from, $page_size );
92+
: $self->_search_expanded( $search_term, $page, $page_size );
8493

8594
return $results;
8695
}
8796

8897
sub _search_expanded {
89-
my ( $self, $search_term, $from, $page_size ) = @_;
98+
my ( $self, $search_term, $page, $page_size ) = @_;
9099

91100
# Used for distribution and module searches, the limit is included in
92101
# the query and ES does the right thing (because we are not collapsing
@@ -95,7 +104,7 @@ sub _search_expanded {
95104
$search_term,
96105
{
97106
size => $page_size,
98-
from => $from
107+
from => ( $page - 1 ) * $page_size,
99108
}
100109
);
101110

@@ -122,11 +131,12 @@ sub _search_expanded {
122131
}
123132

124133
sub _search_collapsed {
125-
my ( $self, $search_term, $from, $page_size, $max_collapsed_hits ) = @_;
134+
my ( $self, $search_term, $page, $page_size, $max_collapsed_hits ) = @_;
126135

127136
$max_collapsed_hits ||= 5;
128137

129-
my $total_size = $from + $page_size;
138+
my $from = ( $page - 1 ) * $page_size;
139+
my $total_size = $page * $page_size;
130140

131141
my $es_query_opts = {
132142
size => 0,

‎lib/MetaCPAN/Server/Controller/Favorite.pm‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ sub recent : Path('recent') : Args(0) {
4343
$c->stash_or_detach(
4444
$self->model($c)->recent(
4545
$c->req->param('page') || 1,
46-
$c->req->param('size') || 100
46+
$c->req->param('page_size') || $c->req->param('size') || 100,
4747
)
4848
);
4949
}

‎lib/MetaCPAN/Server/Controller/Release.pm‎

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,14 +45,20 @@ sub modules : Path('modules') : Args(2) {
4545

4646
sub recent : Path('recent') : Args(0) {
4747
my ( $self, $c ) = @_;
48-
my @params = @{ $c->req->params }{qw( page page_size type )};
49-
$c->stash_or_detach( $self->model($c)->recent(@params) );
48+
my $params = $c->req->params;
49+
my $type = $params->{type};
50+
my $page = $params->{page};
51+
my $size = $params->{page_size};
52+
$c->stash_or_detach( $self->model($c)->recent( $type, $page, $size ) );
5053
}
5154

5255
sub by_author : Path('by_author') : Args(1) {
5356
my ( $self, $c, $pauseid ) = @_;
54-
$c->stash_or_detach( $self->model($c)
55-
->by_author( $pauseid, @{ $c->req->params }{qw( size page )} ) );
57+
my $params = $c->req->params;
58+
my $page = $params->{page};
59+
my $size = $params->{page_size} // $params->{size};
60+
$c->stash_or_detach(
61+
$self->model($c)->by_author( $pauseid, $page, $size ) );
5662
}
5763

5864
sub latest_by_distribution : Path('latest_by_distribution') : Args(1) {
@@ -69,9 +75,11 @@ sub latest_by_author : Path('latest_by_author') : Args(1) {
6975

7076
sub all_by_author : Path('all_by_author') : Args(1) {
7177
my ( $self, $c, $pauseid ) = @_;
72-
my @params = @{ $c->req->params }{qw( page_size page )};
78+
my $params = $c->req->params;
79+
my $page = $params->{page};
80+
my $size = $params->{page_size};
7381
$c->stash_or_detach(
74-
$self->model($c)->all_by_author( $pauseid, @params ) );
82+
$self->model($c)->all_by_author( $pauseid, $page, $size ) );
7583
}
7684

7785
sub versions : Path('versions') : Args(1) {

‎lib/MetaCPAN/Server/Controller/ReverseDependencies.pm‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ sub dist : Path('dist') : Args(1) {
1515
my ( $self, $c, $dist ) = @_;
1616
$c->stash_or_detach(
1717
$c->model('ESModel')->doc('release')->reverse_dependencies(
18-
$dist, @{ $c->req->params }{qw< page page_size size sort >}
18+
$dist, @{ $c->req->params }{qw< page page_size sort >}
1919
)
2020
);
2121
}

‎lib/MetaCPAN/Server/Controller/Search/Web.pm‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,13 @@ sub web : Chained('/search/index') : PathPart('web') : Args(0) {
3030
my ( $self, $c ) = @_;
3131
my $args = $c->req->params;
3232

33+
my $query = $args->{q};
34+
my $size = $args->{page_size} // $args->{size} // 20;
35+
my $page = $args->{page} // ( 1 + int( ( $args->{from} // 0 ) / $size ) );
36+
my $collapsed = $args->{collapsed};
37+
3338
my $model = $c->model('Search');
34-
my $results = $model->search_web( @{$args}{qw( q from size collapsed )} );
39+
my $results = $model->search_web( $query, $page, $size, $collapsed );
3540

3641
$c->stash($results);
3742
}

‎lib/MetaCPAN/Util.pm‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,12 @@ use Sub::Exporter -setup => {
3737
true
3838
false
3939
is_bool
40+
MAX_RESULT_WINDOW
4041
) ]
4142
};
4243

44+
use constant MAX_RESULT_WINDOW => 10000;
45+
4346
*true = \&Cpanel::JSON::XS::true;
4447
*false = \&Cpanel::JSON::XS::false;
4548
*is_bool = \&Cpanel::JSON::XS::is_bool;

‎root/static/requests/search.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,26 @@ search_web:
1818
See also `collapsed`
1919
type: string
2020
required: true
21+
- name: page
22+
in: query
23+
description: The page of the results to return
24+
type: integer
25+
- name: page_size
26+
in: query
27+
description: Number of results per page
28+
type: integer
2129
- name: from
2230
in: query
23-
description: The offset to use in the result set
31+
description: |
32+
The offset to use in the result set. Deprecated. Only used if `page`
33+
is not set.
2434
type: integer
2535
default: 0
2636
- name: size
2737
in: query
28-
description: Number of results per page
38+
description: |
39+
Number of results per page. Deprecated. Only used if `page_size` is
40+
not set.
2941
type: integer
3042
default: 20
3143
- name: collapsed

0 commit comments

Comments
 (0)