Don't allow SourceHandler ties. Harness behavior must be deterministic.

test.tap scores as 0.9 both as a TAP file and as a Perl executable. Currently
one will be randomly chosen adding a heisenbug to the test suite.

This patch makes confidence ties an error.

See http://stackoverflow.com/questions/44101948/prove-returns-inconsistent-test-results-with-testmore-and-tap-extension/44102160 for an example in the wild.

Author: schwern

MANIFEST

@@ -209,6 +209,7 @@ t/source_tests/source.sh
 t/source_tests/source.t
 t/source_tests/source.tap
 t/source_tests/source_args.sh
+t/source_tests/test.tap
 t/spool.t
 t/state.t
 t/state_results.t

lib/TAP/Parser/IteratorFactory.pm

@@ -269,6 +269,15 @@ sub detect_source {
           keys %handlers
     );
 
+    # Check for a tie.
+    if( @handlers > 1 &&
+        $handlers{$handlers[0]} == $handlers{$handlers[1]}
+    ) {
+        my $filename = $source->meta->{file}{basename};
+        die("There is a tie between $handlers[0] and $handlers[1].\n".
+            "Both voted $handlers{$handlers[0]} on $filename.\n");
+    }
+
     # this is really useful for debugging handlers:
     if ( $ENV{TAP_HARNESS_SOURCE_FACTORY_VOTES} ) {
         warn(

t/iterator_factory.t

@@ -10,7 +10,7 @@ BEGIN {
 use strict;
 use warnings;
 
-use Test::More tests => 42;
+use Test::More tests => 44;
 
 use IO::File;
 use File::Spec;
@@ -124,6 +124,9 @@ my @sources = (
         handler  => 'TAP::Parser::SourceHandler::Handle',
         iterator => 'TAP::Parser::Iterator::Stream',
     },
+    {   file     => 'test.tap',
+        tie      => 1,
+    },
 );
 
 for my $test (@sources) {
@@ -141,10 +144,18 @@ for my $test (@sources) {
     my $source   = TAP::Parser::Source->new->raw( ref($raw) ? $raw : \$raw );
     my $iterator = eval { $sf->make_iterator($source) };
     my $error    = $@;
-    ok( !$error, "$name: no error on make_iterator" );
-    diag($error) if $error;
 
-    #    isa_ok( $iterator, $test->{iterator}, $name );
+    if( $test->{tie} ) {
+        like(
+            $error, qr{^There is a tie.*Both voted .* on $test->{file}}ms,
+            "$name: votes tied"
+        )
+    }
+    else {
+        ok( !$error, "$name: no error on make_iterator" );
+        diag($error) if $error;
+    }
+
     is( $sf->_last_handler, $test->{handler}, $name );
 }
 

t/source_tests/test.tap

@@ -0,0 +1,8 @@
+#!/usr/bin/perl
+
+# This looks equally like a TAP file and a Perl executable.
+
+print <<'END_TESTS';
+1..1
+ok 1 - source.pl
+END_TESTS