Tests - setup + basic tests for all scripts

.circleci/config.yml

@@ -0,0 +1,80 @@
+---
+# Use the latest 2.1 version of CircleCI pipeline process engine. See:
+# https://circleci.com/docs/2.0/configuration-reference
+version: 2.1
+orbs:
+  codecov: codecov/[email protected]
+# Orchestrate or schedule a set of jobs
+workflows:
+  docker-compose:
+    jobs:
+      - build-and-test
+jobs:
+  build-and-test:
+    machine: true
+    resource_class: large
+    steps:
+      - run:
+          name: docker-compose version
+          command: docker-compose --version
+      - run:
+          command: |
+            git clone https://github.com/metacpan/metacpan-docker.git
+            cd metacpan-docker
+          name: metacpan-docker checkout
+      - checkout:
+          path: metacpan-docker/src/metacpan-ingest
+      - run:
+          command: |
+            pushd metacpan-docker
+            ./bin/metacpan-docker init
+          name: clone missing repositories
+      - run:
+          command: |
+            pushd metacpan-docker
+            docker-compose build --build-arg CPM_ARGS='--with-test' insgest_test
+          name: compose build
+      - run:
+          command: |
+            pushd metacpan-docker
+            ./bin/metacpan-docker init
+            #docker-compose --verbose up -d ingest_test
+            docker-compose --profile ingest-test up --build
+          name: compose up
+      - run:
+          command: |
+            pushd metacpan-docker
+            docker-compose exec -T ingest_test cpm install -g Devel::Cover
+          name: install Devel::Cover
+      # Since we're running docker-compose -d, we don't actually know if
+      # Elasticsearch is available at the time this build step begins. We
+      # probably need to wait for it here, so we'll add our own check.
+      - run:
+          command: |
+            pushd metacpan-docker
+            ./src/metacpan-ingest/wait-for-es.sh http://localhost:9200 elasticsearch_test
+          name: wait for ES
+      - run:
+          command: |
+            pushd metacpan-docker
+            docker build . -t metacpan/metacpan-ingest && docker compose run ingest env PLACK_ENV=dev HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lrv --jobs 4 t
+#            docker-compose exec -T ingest_test env PLACK_ENV=dev HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t
+          name: run tests with coverage
+      # We are relying on environment variables from the host to be available when
+      # we publish the report, so we publish from the host rather than trying
+      # to propagate env variables to the container.
+      - run:
+          command: |
+            pushd metacpan-docker
+            docker-compose exec -T ingest_test cover -report json
+          name: create coverage report
+      - codecov/upload:
+          file: metacpan-docker/src/metacpan-ingest/cover_db/cover.json
+      - run:
+          command: |
+            pushd metacpan-docker
+            docker-compose logs
+            docker stats --no-stream
+            docker ps -a | head
+          name: docker-compose logs
+          when: on_fail

.github/workflows/build-container.yml

@@ -38,7 +38,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
       - name: Run Perl tests
-        run: docker run -i ${{ steps.docker-build-test.outputs.imageid }}
+        run: docker compose build && docker compose run ingest env PLACK_ENV=dev prove -lv t
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5

Dockerfile

@@ -26,6 +26,10 @@ COPY lib lib
 COPY conf conf
 COPY *.conf .
 
+# move the following to test only
+COPY t t
+COPY test_data test_data
+
 ENV PERL5LIB="/app/local/lib/perl5:/app/lib" PATH="/app/local/bin:${PATH}"
 
 USER metacpan

README.md

@@ -0,0 +1 @@
+[![Build container](https://github.com/metacpan/metacpan-ingest/actions/workflows/build-container.yml/badge.svg)](https://github.com/metacpan/metacpan-ingest/actions/workflows/build-container.yml)

bin/author.pl

@@ -51,18 +51,18 @@
 };
 
 # args
-my ( $mode, $pauseid, $whois_file );
+my ( $findls_file, $pauseid, $whois_file );
 GetOptions(
-    "mode=s"       => \$mode,
-    "pauseid=s"    => \$pauseid,
-    "whois_file=s" => \$whois_file,
+    "findls_file=s" => \$findls_file,
+    "pauseid=s"     => \$pauseid,
+    "whois_file=s"  => \$whois_file,
 );
 
 # setup
-my $es = MetaCPAN::ES->new( index => "author", ( $mode ? ( mode => $mode ) : () ) );
+my $es = MetaCPAN::ES->new( index => "author" );
 
 log_info {'Reading 00whois'};
-my $authors_data = read_00whois( $whois_file );
+my $authors_data = read_00whois($whois_file);
 
 if ($pauseid) {
     log_info {"Indexing 1 author"};
@@ -199,7 +199,7 @@ ($id)
     return undef
         unless $dir->is_dir;
 
-    my $cpan_file_map     = cpan_file_map();
+    my $cpan_file_map     = cpan_file_map($findls_file);
     my $author_cpan_files = $cpan_file_map->{$id}
         or return undef;
 

bin/backpan.pl

@@ -9,17 +9,18 @@
 use MetaCPAN::Ingest qw< cpan_file_map >;
 
 # args
-my ( $distribution, $files_only, $undo );
+my ( $distribution, $files_only, $findls_file, $undo );
 GetOptions(
     "distribution=s" => \$distribution,
     "files_only"     => \$files_only,
+    "findls_file=s"  => \$findls_file,
     "undo"           => \$undo,
 );
 
 # setup
-my $cpan_file_map = cpan_file_map();
-my $es_release    = MetaCPAN::ES->new( type => "release" );
-my $es_file       = MetaCPAN::ES->new( type => "file" );
+my $cpan_file_map = cpan_file_map($findls_file);
+my $es_release    = MetaCPAN::ES->new( index => "release" );
+my $es_file       = MetaCPAN::ES->new( index => "file" );
 
 my %bulk;
 my %release_status;
@@ -55,7 +56,7 @@ ()
             (
                 $undo
                     or exists $cpan_file_map->{$author}{$archive}
-                )
+            )
             ? 'cpan'
             : 'backpan',
             $release->{_id}
@@ -169,6 +170,6 @@ =head1 DESCRIPTION
 Sets "backpan" status on all BackPAN releases.
 
 --undo will set distributions' status back as 'cpan'
---file_only will only fix the 'file' type
+--file_only will only fix the 'file' index
 
 =cut

bin/backup.pl

@@ -21,19 +21,15 @@
 my $size       = 1000;
 my $index      = "cpan";
 
-my ( $dry_run, $mode, $purge, $restore );
+my ( $dry_run, $purge, $restore );
 GetOptions(
     "batch_size=i" => \$batch_size,
     "dry_run"      => \$dry_run,
     "index=s"      => \$index,
-    "mode=s"       => \$mode,
     "purge"        => \$purge,
     "restore=s"    => \$restore,
     "size=i"       => \$size,
 );
-# TODO: find a better way
-my @es_mode = ( $mode ? mode => $mode : () );
-$mode eq 'test' and Log::Log4perl::init('log4perl_test.conf');
 
 # setup
 my $home = home();
@@ -76,7 +72,7 @@ ()
         # create a new bulk helper for each
         my $key = $raw->{_index};
 
-        $es_store{$key} ||= MetaCPAN::ES->new( index => $key, @es_mode );
+        $es_store{$key} ||= MetaCPAN::ES->new( index => $key );
         my $es = $es_store{$key};
 
         $bulk_store{$key} ||= $es->bulk( max_count => $batch_size );
@@ -156,15 +152,14 @@ ()
 }
 
 sub run_backup {
-    my $filename = join( '-',
-        DateTime->now->strftime('%F'),
-        grep {defined} $index );
+    my $filename
+        = join( '-', DateTime->now->strftime('%F'), grep {defined} $index );
 
     my $file = $home->child( qw< var backup >, "$filename.json.gz" );
     $file->parent->mkpath unless ( -e $file->parent );
     my $fh = IO::Zlib->new( "$file", 'wb4' );
 
-    my $es = MetaCPAN::ES->new( index => $index, @es_mode );
+    my $es     = MetaCPAN::ES->new( index => $index );
     my $scroll = $es->scroll(
         scroll => '1m',
         body   => {

bin/check.pl

@@ -9,19 +9,18 @@
 use MetaCPAN::Ingest qw< read_02packages_fh >;
 
 # args
-my ( $module, $max_errors, $errors_only );
-
+my ( $errors_only, $max_errors, $module );
 GetOptions(
-    "module=s"    => \$module,
-    "max_errors"  => \$max_errors,
     "errors_only" => \$errors_only,
+    "max_errors"  => \$max_errors,
+    "module=s"    => \$module,
 );
 
 # setup
 my $error_count = 0;
 my $packages_fh = read_02packages_fh();
-my $es_file     = MetaCPAN::ES->new( type => "file" );
-my $es_release  = MetaCPAN::ES->new( type => "release" );
+my $es_file     = MetaCPAN::ES->new( index => "file" );
+my $es_release  = MetaCPAN::ES->new( index => "release" );
 
 my $modules_start = 0;
 while ( my $line = <$packages_fh> ) {

bin/checksum.pl

@@ -11,14 +11,14 @@
 
 # args
 my $limit = 1000;
-my $dry_run;
+my ($dry_run);
 GetOptions(
-    "limit=i" => \$limit,
     "dry_run" => \$dry_run,
+    "limit=i" => \$limit,
 );
 
 # setup
-my $es = MetaCPAN::ES->new( type => "release" );
+my $es = MetaCPAN::ES->new( index => "release" );
 my $bulk;
 $bulk = $es->bulk() unless $dry_run;
 

bin/contributor.pl

@@ -17,19 +17,20 @@
     "all"            => \$all,
     "distribution=s" => \$distribution,
     "release=s"      => \$release,
-
 );
 
 # setup
 my $author_mapping = {};
 my $email_mapping  = {};
 
-my $es_author      = MetaCPAN::ES->new( type => 'author' );
-my $es_release     = MetaCPAN::ES->new( type => "release" );
-my $es_contributor = MetaCPAN::ES->new( type => "contributor" );
+my $es_author      = MetaCPAN::ES->new( index => 'author' );
+my $es_release     = MetaCPAN::ES->new( index => "release" );
+my $es_contributor = MetaCPAN::ES->new( index => "contributor" );
 
 run();
 
+$es_contributor->index_refresh;
+
 log_info {"done"};
 
 ###
@@ -186,7 +187,7 @@ ($release)
     $authors = [ grep { $_ ne 'unknown' } @$authors ];
 
     my $author_email = $author_mapping->{$author_name}
-        //= eval { $es_author->get_source( id => $author_name )->{email}; }
+        //= eval { $es_author->get_source($author_name)->{email}; }
         or return [];
 
     my $author_info = {
@@ -296,6 +297,6 @@ =head1 SYNOPSIS
 =head1 DESCRIPTION
 
 Update the list of contributors (CPAN authors only) of all/matching
-releases in the 'contributor' type (index).
+releases in the 'contributor' index.
 
 =cut

bin/cover.pl

@@ -15,7 +15,7 @@
 my ( $json_file, $test );
 GetOptions(
     "json=s" => \$json_file,
-    "test"   => \$test
+    "test"   => \$test,
 );
 my $cover_url     //= 'http://cpancover.com/latest/cpancover.json';
 my $cover_dev_url //= 'http://cpancover.com/latest/cpancover_dev.json';
@@ -24,8 +24,9 @@
 my %valid_keys
     = map { $_ => 1 } qw< branch condition statement subroutine total >;
 
-my $es   = MetaCPAN::ES->new( index => "cover", type => "cover" );
-my $bulk = $es->bulk();
+my $es_release = MetaCPAN::ES->new( index => "release" );
+my $es_cover   = MetaCPAN::ES->new( index => "cover" );
+my $bulk_cover = $es_cover->bulk();
 
 my $data = retrieve_cover_data();
 
@@ -34,11 +35,9 @@
 for my $dist ( sort keys %{$data} ) {
     for my $version ( keys %{ $data->{$dist} } ) {
         my $release   = $dist . '-' . $version;
-        my $rel_check = $es->search(
-            index => 'cpan',
-            type  => 'release',
-            size  => 0,
-            body  => {
+        my $rel_check = $es_release->search(
+            size => 0,
+            body => {
                 query => { term => { name => $release } },
             },
         );
@@ -56,7 +55,7 @@
             delete $doc_data{$k} unless exists $valid_keys{$k};
         }
 
-        $bulk->update( {
+        $bulk_cover->update( {
             id  => $release,
             doc => {
                 distribution => $dist,
@@ -69,7 +68,7 @@
     }
 }
 
-$bulk->flush;
+$bulk_cover->flush;
 
 ###
 

bin/cpan_testers.pl

@@ -36,7 +36,7 @@
 
 my $ua = ua();
 
-my $es   = MetaCPAN::ES->new( type => "release" );
+my $es   = MetaCPAN::ES->new( index => "release" );
 my $bulk = $es->bulk();
 
 log_info { 'Mirroring ' . $db };