Merge pull request #1351 from metacpan/haarg/mirror-script-no-model

rewrite mirror script to avoid ESXM

Author: haarg

lib/MetaCPAN/Script/Mirrors.pm

@@ -6,6 +6,8 @@ use warnings;
 use Cpanel::JSON::XS ();
 use Log::Contextual  qw( :log :dlog );
 use Moose;
+use MetaCPAN::ESConfig qw( es_doc_path );
+use MetaCPAN::Util     qw( true false );
 
 with 'MetaCPAN::Role::Script', 'MooseX::Getopt';
 
@@ -19,26 +21,60 @@ sub index_mirrors {
     my $self = shift;
     log_info { 'Getting mirrors.json file from ' . $self->cpan };
 
-    my $json = $self->cpan->child( 'indices', 'mirrors.json' )->slurp;
-    my $type = $self->model->doc('mirror');
-
-    # Clear out everything in the index
-    # so don't end up with old mirrors
-    $type->delete;
+    my $es = $self->es;
 
+    my $json    = $self->cpan->child( 'indices', 'mirrors.json' )->slurp;
     my $mirrors = Cpanel::JSON::XS::decode_json($json);
-    foreach my $mirror (@$mirrors) {
-        $mirror->{location}
-            = { lon => $mirror->{longitude}, lat => $mirror->{latitude} };
+    my %mirrors = map +( $_->{name} => $_ ), @$mirrors;
+
+    my $need_purge;
+
+    my $scroll = $es->scroll_helper( es_doc_path('mirror'), size => 500, );
+    my $bulk   = $es->bulk_helper(
+        es_doc_path('mirror'),
+        on_success => sub {
+            my ( $method, $res ) = @_;
+            if ( $method eq 'update' ) {
+
+                # result is not supported until 5, but this will work when we
+                # update
+                if ( exists $res->{result} ) {
+                    return
+                        if $res->{result} eq 'noop';
+                }
+            }
+            $need_purge++;
+        },
+    );
+    while ( my $doc = $scroll->next ) {
+        if ( !$mirrors{ $doc->{_id} } ) {
+            Dlog_trace {"Deleting $doc->{_id}"};
+            $bulk->delete_ids( $doc->{_id} );
+        }
+    }
+
+    for my $mirror (@$mirrors) {
+        my $data = {%$mirror};
+        delete $data->{$_} for grep !defined $data->{$_}, keys %$data;
+        $data->{location} = {
+            lon => delete $mirror->{longitude},
+            lat => delete $mirror->{latitude},
+        };
+
         Dlog_trace {"Indexing $_"} $mirror;
-        $type->put( {
-            map  { $_ => $mirror->{$_} }
-            grep { defined $mirror->{$_} } keys %$mirror
+        $bulk->update( {
+            id            => $mirror->{name},
+            doc           => $data,
+            doc_as_upsert => true,
         } );
     }
+
+    $bulk->flush;
+
     log_info {'done'};
 
-    $self->cdn_purge_now( { keys => ['MIRRORS'], } );
+    $self->cdn_purge_now( { keys => ['MIRRORS'] } )
+        if $need_purge;
 
 }