Avoid opening too many jobs in the collector. Only parse so many at one time.

Most unix systems limit the number of file handles any one process can
have open at a time to 1024. When the collector gets behind, it may try
to open too many files for completed jobs at once. Limit this with a
new collector setting which defaults to 300. They'll be picked up once
the ones polled are processed.

This commit adds App::Yath::Options::Collector

Author: toddr

lib/App/Yath/Command/collector.pm

@@ -6,6 +6,7 @@ our $VERSION = '1.000039';
 
 use File::Spec;
 
+use App::Yath::Options;
 use App::Yath::Util qw/isolate_stdout/;
 
 use Test2::Harness::Util::JSON qw/decode_json/;
@@ -16,6 +17,12 @@ use Test2::Harness::Run;
 use parent 'App::Yath::Command';
 use Test2::Harness::Util::HashBase;
 
+include_options(
+    'App::Yath::Options::Debug',
+    'App::Yath::Options::PreCommand',
+    'App::Yath::Options::Collector',
+);
+
 sub internal_only   { 1 }
 sub summary         { "For internal use only" }
 sub name            { 'collector' }
@@ -28,7 +35,7 @@ sub run {
 
     my $fh = isolate_stdout();
 
-    my $settings = Test2::Harness::Settings->new(File::Spec->catfile($dir, 'settings.json'));
+    my $settings = $self->settings;
 
     require(mod2file($collector_class));
 

lib/App/Yath/Options/Collector.pm

@@ -0,0 +1,65 @@
+package App::Yath::Options::Collector;
+use strict;
+use warnings;
+
+our $VERSION = '1.000039';
+
+use App::Yath::Options;
+
+option_group {prefix => 'collector', category => "Collector Options"} => sub {
+    option max_jobs_to_process => (
+        description => 'The maximum number of jobs that the collector can process each loop (Default: 300)',
+        default => 300,
+    );
+};
+
+1;
+
+__END__
+
+
+=pod
+
+=encoding UTF-8
+
+=head1 NAME
+
+App::Yath::Options::Collector - collector options for Yath.
+
+=head1 DESCRIPTION
+
+This is where the command line options for the collector are defined.
+
+=head1 PROVIDED OPTIONS POD IS AUTO-GENERATED
+
+=head1 SOURCE
+
+The source code repository for Test2-Harness can be found at
+F<http://github.com/Test-More/Test2-Harness/>.
+
+=head1 MAINTAINERS
+
+=over 4
+
+=item Chad Granum E<lt>[email protected]E<gt>
+
+=back
+
+=head1 AUTHORS
+
+=over 4
+
+=item Chad Granum E<lt>[email protected]E<gt>
+
+=back
+
+=head1 COPYRIGHT
+
+Copyright 2020 Chad Granum E<lt>[email protected]E<gt>.
+
+This program is free software; you can redistribute it and/or
+modify it under the same terms as Perl itself.
+
+See F<http://dev.perl.org/licenses/>
+
+=cut

lib/Test2/Harness/Collector.pm

@@ -221,9 +221,19 @@ sub jobs {
 
     return $jobs if $self->{+JOBS_DONE};
 
+    # Don't monitor more than 'max_jobs_to_process' or we might have too many open file handles and crash
+    # Max open files handles on a process applies. Usually this is 1024 so we can't have everything open at once when we're behind.
+    my $max_jobs_to_process = $self->settings->collector->max_jobs_to_process // 1024;
+    my $additional_jobs_to_parse = $max_jobs_to_process - scalar keys %$jobs;
+    if($additional_jobs_to_parse <= 0) {
+        # This is an unlikely code path. If we're here, it means the last loop couldn't process any results.
+        print "WARN: max jobs still exceded. Continuing to process jobs\n";
+        return $jobs;
+    }
+
     my $queue = $self->jobs_queue or return $jobs;
 
-    for my $item ($queue->poll) {
+    for my $item ($queue->poll($additional_jobs_to_parse)) {
         my ($spos, $epos, $job) = @$item;
 
         unless ($job) {
@@ -271,6 +281,11 @@ sub jobs {
         );
     }
 
+    # The collector didn't read in all the jobs because it'd run out of file handles. We need to let the stream know we're behind.
+    if( scalar keys %$jobs >= $max_jobs_to_process ) {
+        print STDERR "WARN: The Yath Collector is running behind. More than $max_jobs_to_process test results have not been processed.\n";
+    }
+
     return $jobs;
 }
 

lib/Test2/Harness/Util/Queue.pm

@@ -43,11 +43,12 @@ sub reset {
 
 sub poll {
     my $self = shift;
+    my $max = shift;
 
     return $self->{+ENDED} if $self->{+ENDED};
 
     $self->{+QH} ||= Test2::Harness::Util::File::JSONL->new(name => $self->{+FILE});
-    my @out = $self->{+QH}->poll_with_index();
+    my @out = $self->{+QH}->poll_with_index( $max ? (max => $max) : () );
 
     $self->{+ENDED} = $out[-1] if @out && !defined($out[-1]->[-1]);