Skip to content

Commit 708fcdf

Browse files
haarghaarg
committed
convert queries to use _source rather than fields
Newer ES versions don't allow selecting fields. Instead, you need to use stored_fields, but it works rather differently. _source will do basically everything we need, and works on both old and new ES.
1 parent ec0920d commit 708fcdf

File tree

21 files changed

+264
-274
lines changed

21 files changed

+264
-274
lines changed

‎lib/MetaCPAN/Document/File/Set.pm‎

Lines changed: 61 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use List::Util qw( max );
66
use MetaCPAN::Query::Favorite ();
77
use MetaCPAN::Query::File ();
88
use MetaCPAN::Query::Release ();
9-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false );
9+
use MetaCPAN::Util qw( true false );
1010

1111
extends 'ElasticSearchX::Model::Document::Set';
1212

@@ -75,7 +75,8 @@ my @ROGUE_DISTRIBUTIONS = qw(
7575

7676
sub find {
7777
my ( $self, $module ) = @_;
78-
my @candidates = $self->index->type('file')->query( {
78+
79+
my $query = {
7980
bool => {
8081
must => [
8182
{ term => { indexed => true } },
@@ -119,22 +120,38 @@ sub find {
119120
},
120121
],
121122
},
122-
} )->sort( [
123-
'_score',
124-
{ 'version_numified' => { order => 'desc' } },
125-
{ 'date' => { order => 'desc' } },
126-
{ 'mime' => { order => 'asc' } },
127-
{ 'stat.mtime' => { order => 'desc' } }
128-
] )->search_type('dfs_query_then_fetch')->size(100)->all;
123+
};
124+
125+
my $res = $self->es->search(
126+
index => $self->index->name,
127+
type => 'file',
128+
search_type => 'dfs_query_then_fetch',
129+
body => {
130+
query => $query,
131+
sort => [
132+
'_score',
133+
{ 'version_numified' => { order => 'desc' } },
134+
{ 'date' => { order => 'desc' } },
135+
{ 'mime' => { order => 'asc' } },
136+
{ 'stat.mtime' => { order => 'desc' } }
137+
],
138+
_source => [
139+
qw( documentation module.indexed module.authoried module.name )
140+
],
141+
size => 100,
142+
},
143+
);
144+
145+
my @candidates = @{ $res->{hits}{hits} };
129146

130147
my ($file) = grep {
131-
grep { $_->indexed && $_->authorized && $_->name eq $module }
132-
@{ $_->module || [] }
133-
} grep { !$_->documentation || $_->documentation eq $module }
148+
grep { $_->{indexed} && $_->{authorized} && $_->{name} eq $module }
149+
@{ $_->{module} || [] }
150+
} grep { !$_->{documentation} || $_->{documentation} eq $module }
134151
@candidates;
135152

136153
$file ||= shift @candidates;
137-
return $file ? $self->get( $file->id ) : undef;
154+
return $file ? $self->get( $file->{_id} ) : undef;
138155
}
139156

140157
sub find_pod {
@@ -265,15 +282,13 @@ sub history {
265282

266283
sub autocomplete {
267284
my ( $self, @terms ) = @_;
268-
my $query = join( q{ }, @terms );
269-
return $self unless $query;
270285

271-
my $data = $self->search_type('dfs_query_then_fetch')->query( {
286+
my $query = {
272287
bool => {
273288
must => [
274289
{
275290
multi_match => {
276-
query => $query,
291+
query => join( q{ }, @terms ),
277292
type => 'most_fields',
278293
fields => [ 'documentation', 'documentation.*' ],
279294
analyzer => 'camelcase',
@@ -288,15 +303,21 @@ sub autocomplete {
288303
must_not =>
289304
[ { terms => { distribution => \@ROGUE_DISTRIBUTIONS } }, ],
290305
},
291-
} )->sort( [ '_score', 'documentation' ] );
292-
293-
$data = $data->fields( [qw(documentation release author distribution)] )
294-
unless $self->fields;
295-
296-
$data = $data->source(0)->raw->all;
306+
};
307+
308+
my $data = $self->es->search(
309+
search_type => 'dfs_query_then_fetch',
310+
index => $self->index->name,
311+
type => 'file',
312+
body => {
313+
query => $query,
314+
sort => [ '_score', 'documentation' ],
315+
_source => [qw( documentation release author distribution )],
316+
},
317+
);
297318

298-
single_valued_arrayref_to_scalar( $_->{fields} )
299-
for @{ $data->{hits}{hits} };
319+
# this is backcompat. we don't use this end point.
320+
$_->{fields} = delete $_->{_source} for @{ $data->{hits}{hits} };
300321

301322
return $data;
302323
}
@@ -307,8 +328,7 @@ sub autocomplete_suggester {
307328

308329
my $search_size = 100;
309330

310-
my $suggestions
311-
= $self->search_type('dfs_query_then_fetch')->es->suggest( {
331+
my $suggestions = $self->es->suggest( {
312332
index => $self->index->name,
313333
body => {
314334
documentation => {
@@ -319,7 +339,7 @@ sub autocomplete_suggester {
319339
}
320340
}
321341
},
322-
} );
342+
} );
323343

324344
my %docs;
325345

@@ -328,14 +348,6 @@ sub autocomplete_suggester {
328348
( $docs{ $suggest->{text} }, $suggest->{score} );
329349
}
330350

331-
my @fields = qw(
332-
author
333-
date
334-
deprecated
335-
distribution
336-
documentation
337-
release
338-
);
339351
my $data = $self->es->search( {
340352
index => $self->index->name,
341353
type => 'file',
@@ -355,17 +367,22 @@ sub autocomplete_suggester {
355367
],
356368
}
357369
},
370+
_source => [ qw(
371+
author
372+
date
373+
deprecated
374+
distribution
375+
documentation
376+
release
377+
) ],
378+
size => $search_size,
358379
},
359-
fields => \@fields,
360-
size => $search_size,
361380
} );
362381

363382
my %valid = map {
364-
my $got = $_->{fields};
365-
my %record;
366-
@record{@fields} = map { $got->{$_}[0] } @fields;
367-
$record{name} = delete $record{documentation}; # rename
368-
( $_->{fields}{documentation}[0] => \%record );
383+
my %record = %{ $_->{_source} };
384+
$record{name} = delete $record{documentation}; # rename
385+
( $record{name} => \%record );
369386
} @{ $data->{hits}{hits} };
370387

371388
# remove any exact match, it will be added later
@@ -393,7 +410,7 @@ sub autocomplete_suggester {
393410
sub find_changes_files {
394411
my ( $self, $author, $release ) = @_;
395412
my $result = $self->files_by_category( $author, $release, ['changelog'],
396-
{ fields => true } );
413+
{ _source => true } );
397414
my ($file) = @{ $result->{categories}{changelog} || [] };
398415
return $file;
399416
}

‎lib/MetaCPAN/Document/Release/Set.pm‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package MetaCPAN::Document::Release::Set;
22

33
use Moose;
44

5-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar );
6-
75
use MetaCPAN::Query::Release ();
86

97
extends 'ElasticSearchX::Model::Document::Set';

‎lib/MetaCPAN/Query/Author.pm‎

Lines changed: 7 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@ package MetaCPAN::Query::Author;
22

33
use MetaCPAN::Moose;
44

5-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar );
6-
use Ref::Util qw( is_arrayref );
5+
use Ref::Util qw( is_arrayref );
76

87
with 'MetaCPAN::Query::Role::Common';
98

@@ -23,10 +22,7 @@ sub by_ids {
2322
body => $body,
2423
);
2524

26-
my @authors = map {
27-
single_valued_arrayref_to_scalar( $_->{_source} );
28-
$_->{_source}
29-
} @{ $authors->{hits}{hits} };
25+
my @authors = map $_->{_source}, @{ $authors->{hits}{hits} };
3026

3127
return {
3228
authors => \@authors,
@@ -48,10 +44,7 @@ sub by_user {
4844
}
4945
);
5046

51-
my @authors = map {
52-
single_valued_arrayref_to_scalar( $_->{_source} );
53-
$_->{_source}
54-
} @{ $authors->{hits}{hits} };
47+
my @authors = map $_->{_source}, @{ $authors->{hits}{hits} };
5548

5649
return {
5750
authors => \@authors,
@@ -94,10 +87,8 @@ sub search {
9487
body => $body,
9588
);
9689

97-
my @authors = map {
98-
single_valued_arrayref_to_scalar( $_->{_source} );
99-
+{ %{ $_->{_source} }, id => $_->{_id} }
100-
} @{ $ret->{hits}{hits} };
90+
my @authors = map { +{ %{ $_->{_source} }, id => $_->{_id} } }
91+
@{ $ret->{hits}{hits} };
10192

10293
return +{
10394
authors => \@authors,
@@ -127,10 +118,8 @@ sub prefix_search {
127118
body => $body,
128119
);
129120

130-
my @authors = map {
131-
single_valued_arrayref_to_scalar( $_->{_source} );
132-
+{ %{ $_->{_source} }, id => $_->{_id} }
133-
} @{ $ret->{hits}{hits} };
121+
my @authors = map { +{ %{ $_->{_source} }, id => $_->{_id} } }
122+
@{ $ret->{hits}{hits} };
134123

135124
return +{
136125
authors => \@authors,

‎lib/MetaCPAN/Query/Favorite.pm‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package MetaCPAN::Query::Favorite;
22

33
use MetaCPAN::Moose;
44

5-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar );
6-
75
with 'MetaCPAN::Query::Role::Common';
86

97
sub agg_by_distributions {
@@ -72,18 +70,16 @@ sub by_user {
7270
index => $self->index_name,
7371
type => 'favorite',
7472
body => {
75-
query => { term => { user => $user } },
76-
fields => [qw( author date distribution )],
77-
sort => ['distribution'],
78-
size => $size,
73+
query => { term => { user => $user } },
74+
_source => [qw( author date distribution )],
75+
sort => ['distribution'],
76+
size => $size,
7977
}
8078
);
8179
return {} unless $favs->{hits}{total};
8280
my $took = $favs->{took};
8381

84-
my @favs = map { $_->{fields} } @{ $favs->{hits}{hits} };
85-
86-
single_valued_arrayref_to_scalar( \@favs );
82+
my @favs = map { $_->{_source} } @{ $favs->{hits}{hits} };
8783

8884
# filter out backpan only distributions
8985

@@ -104,14 +100,14 @@ sub by_user {
104100
]
105101
}
106102
},
107-
fields => ['distribution'],
108-
size => scalar(@favs),
103+
_source => ['distribution'],
104+
size => scalar(@favs),
109105
}
110106
);
111107
$took += $no_backpan->{took};
112108

113109
if ( $no_backpan->{hits}{total} ) {
114-
my %has_no_backpan = map { $_->{fields}{distribution}[0] => 1 }
110+
my %has_no_backpan = map { $_->{_source}{distribution} => 1 }
115111
@{ $no_backpan->{hits}{hits} };
116112

117113
@favs = grep { exists $has_no_backpan{ $_->{distribution} } } @favs;
@@ -195,8 +191,6 @@ sub users_by_distribution {
195191

196192
my @plusser_users = map { $_->{_source}{user} } @{ $favs->{hits}{hits} };
197193

198-
single_valued_arrayref_to_scalar( \@plusser_users );
199-
200194
return { users => \@plusser_users };
201195
}
202196

‎lib/MetaCPAN/Query/File.pm‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package MetaCPAN::Query::File;
22

33
use MetaCPAN::Moose;
44

5-
use MetaCPAN::Util qw( single_valued_arrayref_to_scalar true false );
5+
use MetaCPAN::Util qw( true false );
66

77
with 'MetaCPAN::Query::Role::Common';
88

@@ -24,8 +24,8 @@ sub dir {
2424
]
2525
},
2626
},
27-
size => 999,
28-
fields => [
27+
size => 999,
28+
_source => [
2929
qw(name stat.mtime path stat.size directory slop documentation mime)
3030
],
3131
};
@@ -36,8 +36,7 @@ sub dir {
3636
body => $body,
3737
} );
3838

39-
my $dir = [ map { $_->{fields} } @{ $data->{hits}{hits} } ];
40-
single_valued_arrayref_to_scalar($dir);
39+
my $dir = [ map { $_->{_source} } @{ $data->{hits}{hits} } ];
4140

4241
return { dir => $dir };
4342
}
@@ -259,7 +258,7 @@ sub interesting_files {
259258
return $return
260259
unless @clauses;
261260

262-
my $source = $options->{fields} || [ qw(
261+
$options->{_source} //= [ qw(
263262
author
264263
distribution
265264
documentation
@@ -269,6 +268,7 @@ sub interesting_files {
269268
release
270269
status
271270
) ];
271+
$options->{size} //= 250;
272272

273273
my $body = {
274274
query => {
@@ -291,9 +291,7 @@ sub interesting_files {
291291
],
292292
},
293293
},
294-
_source => $source,
295-
296-
size => $options->{size} || 250,
294+
%$options,
297295
};
298296

299297
my $data = $self->es->search( {

0 commit comments

Comments
 (0)