|
6 | 6 | use MetaCPAN::Logger qw< :log :dlog >; |
7 | 7 |
|
8 | 8 | use MetaCPAN::ES; |
9 | | -use MetaCPAN::Ingest qw< minion >; |
| 9 | +use MetaCPAN::Ingest qw< false minion >; |
10 | 10 |
|
11 | 11 | # args |
12 | 12 | my ( $age, $check_missing, $count, $distribution, $limit, $queue ); |
|
39 | 39 | ### |
40 | 40 |
|
41 | 41 | sub index_favorites () { |
42 | | - my $body; |
| 42 | + my $query = { match_all => {} }; |
43 | 43 | my $age_filter; |
44 | 44 |
|
45 | 45 | if ($age) { |
|
48 | 48 | } |
49 | 49 |
|
50 | 50 | if ($distribution) { |
51 | | - $body = { |
52 | | - query => { |
53 | | - term => { distribution => $distribution } |
54 | | - } |
55 | | - }; |
| 51 | + $query = { term => { distribution => $distribution } }; |
56 | 52 | } |
57 | 53 | elsif ($age) { |
58 | 54 | my $es = MetaCPAN::ES->new( type => "favorite" ); |
59 | 55 | my $favs = $es->scroll( |
60 | 56 | scroll => '5m', |
61 | | - fields => [qw< distribution >], |
62 | 57 | body => { |
63 | 58 | query => $age_filter, |
64 | | - ( $limit ? ( size => $limit ) : () ) |
| 59 | + _source => [qw< distribution >], |
| 60 | + size => $limit || 500, |
| 61 | + sort => '_doc', |
65 | 62 | } |
66 | 63 | ); |
67 | 64 |
|
68 | 65 | my %recent_dists; |
69 | 66 |
|
70 | 67 | while ( my $fav = $favs->next ) { |
71 | | - my $dist = $fav->{fields}{distribution}[0]; |
| 68 | + my $dist = $fav->{_source}{distribution}; |
72 | 69 | $recent_dists{$dist}++ if $dist; |
73 | 70 | } |
74 | 71 |
|
75 | 72 | my @keys = keys %recent_dists; |
76 | 73 | if (@keys) { |
77 | | - $body = { |
78 | | - query => { |
79 | | - terms => { distribution => \@keys } |
80 | | - } |
81 | | - }; |
| 74 | + $query = { terms => { distribution => \@keys } }; |
82 | 75 | } |
83 | 76 | $es->index_refresh; |
84 | 77 | } |
|
94 | 87 | my $es = MetaCPAN::ES->new( type => "favorite" ); |
95 | 88 | my $favs = $es->scroll( |
96 | 89 | scroll => '30s', |
97 | | - fields => [qw< distribution >], |
98 | | - ( $body ? ( body => $body ) : () ), |
| 90 | + body => { |
| 91 | + query => $query, |
| 92 | + _source => [qw< distribution >], |
| 93 | + size => 500, |
| 94 | + sort => '_doc', |
| 95 | + }, |
99 | 96 | ); |
100 | 97 |
|
101 | 98 | while ( my $fav = $favs->next ) { |
102 | | - my $dist = $fav->{fields}{distribution}[0]; |
| 99 | + my $dist = $fav->{_source}{distribution}; |
103 | 100 | $dist_fav_count{$dist}++ if $dist; |
104 | 101 | } |
105 | 102 |
|
|
119 | 116 | my $es = MetaCPAN::ES->new( type => "file" ); |
120 | 117 | my $files = $es->scroll( |
121 | 118 | scroll => '15m', |
122 | | - fields => [qw< id distribution >], |
123 | | - size => 500, |
124 | 119 | body => { |
125 | 120 | query => { |
126 | 121 | bool => { |
127 | 122 | must_not => [ |
128 | 123 | { range => { dist_fav_count => { gte => 1 } } } |
129 | 124 | ], |
130 | 125 | @age_filter, |
131 | | - } |
132 | | - } |
| 126 | + }, |
| 127 | + }, |
| 128 | + _source => [qw< id distribution >], |
| 129 | + size => 500, |
| 130 | + sort => '_doc', |
| 131 | + |
133 | 132 | }, |
134 | 133 | ); |
135 | 134 |
|
136 | 135 | while ( my $file = $files->next ) { |
137 | | - my $dist = $file->{fields}{distribution}[0]; |
| 136 | + my $dist = $file->{_source}{distribution}; |
138 | 137 | next unless $dist; |
139 | 138 | next if exists $missing{$dist} or exists $dist_fav_count{$dist}; |
140 | 139 |
|
|
189 | 188 | my $bulk = $es->bulk( timeout => '120m' ); |
190 | 189 | my $files = $es->scroll( |
191 | 190 | scroll => '15s', |
192 | | - fields => [qw< id >], |
193 | 191 | body => { |
194 | 192 | query => { term => { distribution => $dist } } |
| 193 | + _source => false, |
| 194 | + size => 500, |
| 195 | + sort => '_doc', |
195 | 196 | }, |
196 | 197 | ); |
197 | 198 |
|
198 | 199 | while ( my $file = $files->next ) { |
199 | | - my $id = $file->{fields}{id}[0]; |
| 200 | + my $id = $file->{_id}; |
200 | 201 | my $cnt = $dist_fav_count{$dist}; |
201 | 202 |
|
202 | 203 | log_debug {"Updating file id $id with fav_count $cnt"}; |
203 | 204 |
|
204 | 205 | $bulk->update( { |
205 | | - id => $file->{fields}{id}[0], |
| 206 | + id => $file->{_id}; |
206 | 207 | doc => { dist_fav_count => $cnt }, |
207 | 208 | } ); |
208 | 209 | } |
|
0 commit comments