Improved spaceout handling. (#401)

See '--large-files' entry in man geninfo(1), lcov(1).

This feature is equivalent to finding all the GCDA files which match the regexp(s), processing them serially, then finding the remining GCDA files and processing them in parallel - and then aggregating the result.
The feature just simplifies the Jenkins regression script.
More sophisticated scheduling is possible - but seems unnecessary at present.

Signed-off-by: Henry Cox <[email protected]>

Author: henry2cox

bin/genhtml

@@ -3794,7 +3794,6 @@ sub check_path_consistency
             ++$diffBaseMap{$b}->[1]->{$alias};
         } elsif (!exists($self->[UNCHANGED]->{$self->findName($curr)})) {
             $missed{$curr} = 1;       # in current but not in diff
-            #lcovutil::info("missed curr: $curr\n");
         }
     }
     # for each file in 'baseline' info:
@@ -3810,7 +3809,6 @@ sub check_path_consistency
             ++$diffBaseMap{$b}->[1]->{$alias};
         } elsif (!exists($self->[UNCHANGED]->{$self->findName($base)})) {
             # in baseline but not in diff
-            #lcovutil::info("missed base: $base\n");
             if (exists($missed{$base})) {
                 $missed{$base} |= 2;
             } else {

bin/geninfo

@@ -222,6 +222,7 @@ our $intervalChildCpuTime = 0;    # since last updata
 our $defaultChunkSize;
 our $defaultInterval;
 our %childRetryCounts;
+our @large_files;
 
 our $cwd = getcwd();
 chomp($cwd);
@@ -255,7 +256,8 @@ my %geninfo_opts = ("test-name|t=s"       => \$test_name,
                     "derive-func-data"    => \$opt_derive_func_data,
                     "external|e"          => \$lcovutil::opt_external,
                     "no-external"         => \$lcovutil::opt_no_external,
-                    "compat=s"            => \$lcovutil::geninfo_opt_compat,);
+                    "compat=s"            => \$lcovutil::geninfo_opt_compat,
+                    'large-file=s'        => \@large_files);
 
 # Parse command line options
 if (!lcovutil::parseOptions(\%lcovutil::geninfo_rc_opts, \%geninfo_opts,
@@ -268,6 +270,12 @@ $buildDirSearchPath =
     SearchPath->new('build directory', @lcovutil::build_directory);
 @gcov_tool = @lcovutil::rc_gcov_tool unless @gcov_tool;
 
+eval {
+    map { qr($_) } @large_files;
+};
+die("invalid 'large-file' regexp: $@")
+    if ($@);
+
 # Check regexp
 if (defined($lcovutil::rc_adjust_src_path)) {
     my ($pattern, $replace) = split(/\s*=>\s*/, $lcovutil::rc_adjust_src_path);
@@ -1065,7 +1073,7 @@ sub _process_one_chunk($$$$)
 
         # "name" will be .gcno if "$initial" else will be $gcda
         my $name = defined($gcda_file) ? $gcda_file : $gcno_file;
-        info(1, "Processing $name%s\n", defined($pid) ? " in child $pid" : "");
+        info(1, "Processing $name%s\n", defined($pid) ? " in child $pid" : "" . "\n");
         my $context = MessageContext->new("capturing from $name");
 
         # multiple gcda files may refer to the same source - so generate the
@@ -1181,6 +1189,19 @@ sub _merge_one_child($$$)
              $lcovutil::max_fork_fails != 0) ||
             $lcovutil::verbose);
     print(STDERR $childErr);
+    # look for spaceout message in the gcov log
+    if (0 == $signal                                &&
+        0 != $childstatus                           &&
+        0 != $lcovutil::max_fork_fails              &&
+        lcovutil::is_ignored($lcovutil::ERROR_FORK) &&
+        grep(
+            { /(std::bad_alloc|annot allocate memory|out of memory|integretity check failed for compressed file)/
+            } ($childLog, $childErr))
+    ) {
+
+        # pretend it was killed so we retry
+        $signal = POSIX::SIGKILL;
+    }
     my $data = Storable::retrieve($dumped)
         if (-f $dumped && 0 == $childstatus);
     # note that $data will not be defined (no data dumped) if there was
@@ -1298,15 +1319,26 @@ sub gen_info(@)
         $chunkSize = 1;
     }
     my @worklist;
-    my $chunk = [];
+    my $serialChunk = [1, []];
+    my $chunk       = [0, []];    # [isSerial, [fileList]]
     foreach my $j (@$filelist) {
-        push(@$chunk, $j);
-        if (scalar(@$chunk) == $chunkSize) {
+        my $filename = $j->[0] . $lcovutil::dirseparator . $j->[1];
+        if (grep({ $filename =~ $_ } @main::large_files)) {
+            lcovutil::info(1, "large file: $filename\n");
+            push(@{$serialChunk->[1]}, $j);
+            next;
+        }
+        push(@{$chunk->[1]}, $j);
+        if (scalar(@{$chunk->[1]}) == $chunkSize) {
             push(@worklist, $chunk);
-            $chunk = [];
+            $chunk = [0, []];
         }
     }    #foreach DATA_FILE
-    push(@worklist, $chunk) if @$chunk;
+    push(@worklist, $chunk) if @{$chunk->[1]};
+    # serial chunk is at the top of the stack - so serial processing
+    #  happens before we fork multiple processes
+    push(@worklist, $serialChunk)
+        if (@{$serialChunk->[1]});
 
     # Process all files in list
     my $currentParallel = 0;
@@ -1342,7 +1374,8 @@ sub gen_info(@)
             my $chunk = pop(@worklist);
             ++$processedChunks;
 
-            if (1 < $lcovutil::maxParallelism) {
+            if (1 < $lcovutil::maxParallelism &&
+                1 != $chunk->[0]) {
 
                 my $currentSize = 0;
                 if (0 != $lcovutil::maxMemory) {
@@ -1410,8 +1443,8 @@ sub gen_info(@)
                     my ($stdout, $stderr, $code) = Capture::Tiny::capture {
                         eval {
                             $childInfo =
-                                _process_one_chunk($chunk, $processedChunks,
-                                                   $childInfo, $$);
+                                _process_one_chunk($chunk->[1],
+                                              $processedChunks, $childInfo, $$);
                         };
                         if ($@) {
                             $status = 1;         # error
@@ -1458,12 +1491,14 @@ sub gen_info(@)
                         #  there is no childInfo data
                         $data =
                             Storable::store(
-                                      [$single_file ? $childInfo : undef,
-                                       $buildDirCounts,
-                                       [$files_created, scalar(@$chunk), $then],
-                                       lcovutil::compute_update($currentState)
-                                      ],
-                                      $dumpf) if defined($childInfo);
+                                       [$single_file ? $childInfo : undef,
+                                        $buildDirCounts,
+                                        [$files_created, scalar(@{$chunk->[1]}),
+                                         $then
+                                        ],
+                                        lcovutil::compute_update($currentState)
+                                       ],
+                                       $dumpf) if defined($childInfo);
                     };
                     if ($@ || (defined($childInfo) && !defined($data))) {
                         lcovutil::ignorable_error($lcovutil::ERROR_PARALLEL,
@@ -1477,12 +1512,24 @@ sub gen_info(@)
                 }
             } else {
                 # not parallel..
+                my $saveParallel = $lcovutil::maxParallelism;
+                $lcovutil::maxParallelism = 1;
+                if ($chunk->[0]) {
+                    my $num = scalar(@{$chunk->[1]});
+                    lcovutil::info("Processing $num file" .
+                                   ($num == 1 ? '' : 's') .
+                                   " from chunk 0 serially\n");
+                }
                 my $now = Time::HiRes::gettimeofday();
                 $trace_data =
-                    _process_one_chunk($chunk, $processedChunks, $trace_data,
-                                       undef);
-                $processedFiles += scalar(@$chunk);
+                    _process_one_chunk($chunk->[1], $processedChunks,
+                                       $trace_data, undef);
+                $processedFiles += scalar(@{$chunk->[1]});
+                if ($chunk->[0]) {
+                    lcovutil::info("Finished processing chunk 0\n");
+                }
                 my $then = Time::HiRes::gettimeofday();
+                $lcovutil::maxParallelism = $saveParallel;
                 $lcovutil::profileData{process}{$processedChunks} =
                     $then - $now;
             }
@@ -1975,7 +2022,8 @@ sub compute_internal_directories(@)
             my $t = Cwd::realpath($top);
             die("expected directory found '$t'") unless -d $t;
             unless (exists($visited{$t})) {
-                lcovutil::info("internal: target '$t' of link '$top'\n");
+                lcovutil::info(1,
+                            "internal directory: target '$t' of link '$top'\n");
                 $visited{$t} = $top;
                 push(@dirstack, $t);
                 push(@lcovutil::internal_dirs, $t)
@@ -2836,11 +2884,22 @@ sub process_intermediate($$$$)
             (0 != $rc ||
              $lcovutil::verbose));
 
+    my $gcovOutGlobPattern =
+        "$tempdir/*.gcov $tempdir/.*.gcov $tempdir/*.gcov.json.gz $tempdir/.*gcov.json.gz";
+
     if (0 != $rc) {
         if (check_gcov_fail($err, $file)) {
             return;
         }
         $errmsg = "GCOV failed for $file";
+        # can parse the error log to see if it spaced out - then return
+        # code so parent can catch it
+        if ($err =~ /out of memory allocating/) {
+            lcovutil::info("spaceout calling gcov for '$data_file'\n");
+            $errmsg .= ' out of memory';
+            $errorType = $lcovutil::ERROR_CHILD
+                if 1 != $lcovutil::maxParallelism;
+        }
         goto err;
     }
 
@@ -2853,22 +2912,32 @@ sub process_intermediate($$$$)
     # 'meson' build system likes to use "." as leading character in generated
     # files.  Seems an unfortunate decision.
     my $start = Time::HiRes::gettimeofday();
-    for my $gcov_filename (
-        glob(
-            "$tempdir/*.gcov $tempdir/.*.gcov $tempdir/*.gcov.json.gz $tempdir/.*gcov.json.gz"
-        )
-    ) {
-        if ($gcov_filename =~ /\.gcov\.json/) {
-            read_intermediate_json($gcov_filename, \%data, \$json_basedir);
-            $json_format = 1;
-        } else {
-            read_intermediate_text($gcov_filename, \%data);
-        }
-        if ($lcovutil::preserve_intermediates) {
-            File::Copy::move($gcov_filename, $fdir) or
-                die("cannot rename $gcov_filename: $!");
-        } else {
-            unlink($gcov_filename);
+    for my $gcov_filename (glob($gcovOutGlobPattern)) {
+        eval {
+            if ($gcov_filename =~ /\.gcov\.json/) {
+                read_intermediate_json($gcov_filename, \%data, \$json_basedir);
+                $json_format = 1;
+            } else {
+                read_intermediate_text($gcov_filename, \%data);
+            }
+            if ($lcovutil::preserve_intermediates) {
+                File::Copy::move($gcov_filename, $fdir) or
+                    die("cannot rename $gcov_filename: $!");
+            } else {
+                unlink($gcov_filename);
+            }
+        };
+        if ($@) {
+            if (1 != $lcovutil::maxParallelism &&
+                $@ =~ /(integrity check failed|cannot start)/) {
+                # looks like we ran out of memory..
+                # maybe need new error type ERROR_MEMORY
+                #$errorType = $lcovutil::ERROR_GCOV;
+                $errmsg = $@;
+                goto err;
+            } else {
+                die("read_intermediate failed: $@");
+            }
         }
     }
     my $end = Time::HiRes::gettimeofday();
@@ -2915,6 +2984,7 @@ sub process_intermediate($$$$)
     return $trace;
 
     err:
+    unlink(glob($gcovOutGlobPattern));    # clean up - in case gcov died
     ignorable_error($errorType, "$errmsg!");
     return undef;
 }

bin/lcov

@@ -162,6 +162,7 @@ our $compat_libtool;     # If set, indicates that libtool mode is to be enabled
 our $no_compat_libtool;  # If set, indicates that libtool mode is to be disabled
 
 our @gcov_tool;
+our @large_files;        # handled sequentially in geninfo
 our $initial;
 our $captureAll;
 our $no_recursion = 0;
@@ -214,6 +215,7 @@ my %lcov_options = ("directory|d|di=s"     => \@directory,
                     "compat-libtool"       => \$compat_libtool,
                     "no-compat-libtool"    => \$no_compat_libtool,
                     "gcov-tool=s"          => \@gcov_tool,
+                    'large-file=s'         => \@large_files,
 
                     "initial|i"         => \$initial,
                     "all"               => \$captureAll,
@@ -938,6 +940,7 @@ sub lcov_geninfo(@)
                           \@lcovutil::exclude_function_patterns
                          ],
                          ['--filter', \@lcovutil::opt_filter],
+                         ['--large-file', \@large_files],
     ) {
         my ($opt, $l) = @$listOpt;
         foreach my $v (@$l) {

man/geninfo.1

@@ -51,6 +51,9 @@ geninfo \- Generate tracefiles from GCOV coverage data files
 .RB [ \-\-parallel | -j
 .IR [integer] ]
 .br
+.br [ \-\-large\-file
+.IR regexp ]
+.br
 .RB [ \-\-memory
 .IR integer_num_Mb ]
 .br
@@ -1299,6 +1302,13 @@ generated:  host, date, environment,
 .RS
 Specify parallelism to use during processing (maximum number of forked child processes).  If the optional integer parallelism parameter is zero or is missing, then use to use up the number of cores on the machine.  Default is not to use a single process (no parallelism).
 
+The
+.I \-\large\-file
+option described below may be necessary to enable parallelism to succeed
+in the presence of data files which consume excessive memory in
+.B gcov.
+
+
 Also see the
 .I memory, memory_percentage, max_fork_fails, fork_fail_timeout, geninfo_chunk_size
 and
@@ -1307,6 +1317,24 @@ entries in man
 .B lcovrc(5)
 for a description of some options which may aid in parameter tuning and performance optimization.
 
+.RE
+.BI "\-\-large\-file "
+.I regexp
+.RS
+
+GCDA files whose name matches a
+.I \-\-large\-file
+regexp are processed serially - not in parallel with other files - so that
+their
+.B gcov
+process can use all available system memory.
+.br
+Use this option is you see errors related to memory allocation from gcov.
+.br
+This feature is exactly as if you had moved the matching GCDA files to another location and processed them serially, then processed remaining GDCA files in parallel and merged the results.
+
+This option may be used multiple times to specify more than one regexp.
+
 .RE
 .BI "\-\-memory "
 .I integer

man/lcov.1

@@ -91,6 +91,9 @@ Capture coverage data tracefile (from compiler-generated data):
 .RB [ \-\-comment
 .IR comment_string ]
 .br
+.RB [ \-\-large\-file
+.IR regexp ]
+.br
 .RE
 .RE
 
@@ -1178,6 +1181,16 @@ specified at a time.
 Follow links when searching for .da files.
 .RE
 
+.BI "\-\-large\-file "
+.I regexp
+.RS
+See the
+.I \-\-large\-file
+section of man
+.B geninfo(1)
+for details.
+.RE
+
 .B \-\-from\-package
 .I package
 .br