rewrite mirror script to avoid ESXM

The mirror network is gone, but we still index the list of the one
mirror. While we could consider removing this index in the future, for
now just fix the script to use Elasticsearch directly rather than
ElasticSearchX::Model.

Author: haarg

lib/MetaCPAN/Script/Mirrors.pm

@@ -6,6 +6,8 @@ use warnings;
 use Cpanel::JSON::XS ();
 use Log::Contextual  qw( :log :dlog );
 use Moose;
+use MetaCPAN::ESConfig qw( es_doc_path );
+use MetaCPAN::Util     qw( true false );
 
 with 'MetaCPAN::Role::Script', 'MooseX::Getopt';
 
@@ -19,26 +21,60 @@ sub index_mirrors {
     my $self = shift;
     log_info { 'Getting mirrors.json file from ' . $self->cpan };
 
-    my $json = $self->cpan->child( 'indices', 'mirrors.json' )->slurp;
-    my $type = $self->model->doc('mirror');
-
-    # Clear out everything in the index
-    # so don't end up with old mirrors
-    $type->delete;
+    my $es = $self->es;
 
+    my $json    = $self->cpan->child( 'indices', 'mirrors.json' )->slurp;
     my $mirrors = Cpanel::JSON::XS::decode_json($json);
-    foreach my $mirror (@$mirrors) {
-        $mirror->{location}
-            = { lon => $mirror->{longitude}, lat => $mirror->{latitude} };
+    my %mirrors = map +( $_->{name} => $_ ), @$mirrors;
+
+    my $need_purge;
+
+    my $scroll = $es->scroll_helper( es_doc_path('mirror'), size => 500, );
+    my $bulk   = $es->bulk_helper(
+        es_doc_path('mirror'),
+        on_success => sub {
+            my ( $method, $res ) = @_;
+            if ( $method eq 'update' ) {
+
+                # result is not supported until 5, but this will work when we
+                # update
+                if ( exists $res->{result} ) {
+                    return
+                        if $res->{result} eq 'noop';
+                }
+            }
+            $need_purge++;
+        },
+    );
+    while ( my $doc = $scroll->next ) {
+        if ( !$mirrors{ $doc->{_id} } ) {
+            Dlog_trace {"Deleting $doc->{_id}"};
+            $bulk->delete_ids( $doc->{_id} );
+        }
+    }
+
+    for my $mirror (@$mirrors) {
+        my $data = {%$mirror};
+        delete $data->{$_} for grep !defined $data->{$_}, keys %$data;
+        $data->{location} = {
+            lon => delete $mirror->{longitude},
+            lat => delete $mirror->{latitude},
+        };
+
         Dlog_trace {"Indexing $_"} $mirror;
-        $type->put( {
-            map  { $_ => $mirror->{$_} }
-            grep { defined $mirror->{$_} } keys %$mirror
+        $bulk->update( {
+            id            => $mirror->{name},
+            doc           => $data,
+            doc_as_upsert => true,
         } );
     }
+
+    $bulk->flush;
+
     log_info {'done'};
 
-    $self->cdn_purge_now( { keys => ['MIRRORS'], } );
+    $self->cdn_purge_now( { keys => ['MIRRORS'] } )
+        if $need_purge;
 
 }