autopkgtest: rework "expected failures" mechanism

terceiro · sorah · commit 9aaa59160f3a · 2020-04-13T21:02:18.000+09:00
Instead of running tests that we know will fail, just exclude them from
them run, and drop the 'known-failures.txt' file.

Now, since some test files will fail to even load, add a skiplist file.
Files listed in there won't even be attempted and will be counted as
skipped.
diff --git a/debian/tests/excludes/autopkgtest/TestFiber.rb b/debian/tests/excludes/autopkgtest/TestFiber.rb
@@ -0,0 +1 @@
+exclude :test_fatal_in_fiber, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemCommandsEnvironmentCommand.rb b/debian/tests/excludes/autopkgtest/TestGemCommandsEnvironmentCommand.rb
@@ -0,0 +1 @@
+exclude :test_execute, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemCommandsUninstallCommand.rb b/debian/tests/excludes/autopkgtest/TestGemCommandsUninstallCommand.rb
@@ -0,0 +1,2 @@
+exclude :test_execute_removes_executable, 'depends on source tree'
+exclude :test_execute_prerelease, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemCommandsUpdateCommand.rb b/debian/tests/excludes/autopkgtest/TestGemCommandsUpdateCommand.rb
@@ -0,0 +1,2 @@
+exclude :test_update_rubygems_arguments_1_8_x, 'depends on source tree'
+exclude :test_update_rubygems_arguments, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemExtRakeBuilder.rb b/debian/tests/excludes/autopkgtest/TestGemExtRakeBuilder.rb
@@ -0,0 +1,2 @@
+exclude :test_class_build, 'depends on source tree'
+exclude :test_class_build_with_args, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemPackage.rb b/debian/tests/excludes/autopkgtest/TestGemPackage.rb
@@ -0,0 +1,6 @@
+exclude :test_verify_security_policy_low_security, 'depends on source tree'
+exclude :test_build_auto_signed_encrypted_key, 'depends on source tree'
+exclude :test_verify_security_policy_checksum_missing, 'depends on source tree'
+exclude :test_build_auto_signed, 'depends on source tree'
+exclude :test_build_signed_encrypted_key, 'depends on source tree'
+exclude :test_build_signed, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemPackageTarWriter.rb b/debian/tests/excludes/autopkgtest/TestGemPackageTarWriter.rb
@@ -0,0 +1 @@
+exclude :test_add_file_signer, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestGemSpecification.rb b/debian/tests/excludes/autopkgtest/TestGemSpecification.rb
@@ -0,0 +1 @@
+exclude :test_base_dir_default, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestIO.rb b/debian/tests/excludes/autopkgtest/TestIO.rb
@@ -0,0 +1,2 @@
+exclude :test_seek, 'depends on source tree'
+exclude :test_seek_symwhence, 'depends on source tree'
diff --git a/debian/tests/excludes/autopkgtest/TestRipper/Generic.rb b/debian/tests/excludes/autopkgtest/TestRipper/Generic.rb
@@ -0,0 +1,3 @@
+exclude :'test_parse_files:ext', 'depends on source tree'
+exclude :'test_parse_files:lib', 'depends on source tree'
+exclude :'test_parse_files:sample', 'depends on source tree'
diff --git a/debian/tests/run-all b/debian/tests/run-all
@@ -4,7 +4,6 @@ exec 2>&1
 set -e
 #set -x
 
-KNOW_FAILURES=$(dirname $(readlink -f $0))/known-failures.txt
 # test against the default OpenSSL settings and not the Debian-specific ones
 export OPENSSL_CONF=`pwd`/debian/openssl.cnf
 
@@ -18,6 +17,7 @@ if [ -z "$AUTOPKGTEST_TMP" ]; then
   trap cleanup INT TERM EXIT
 fi
 
+skiplist=$(readlink -f $(dirname $0))/skiplist
 excludedir=$(readlink -f $(dirname $0))/excludes
 cp -r 'test/' $AUTOPKGTEST_TMP
 cp -r 'tool/' $AUTOPKGTEST_TMP
@@ -29,45 +29,16 @@ if [ -z "$tests" ]; then
   tests=$(find 'test/' -name 'test_*.rb' -and -not -path '*-ext-*' | sort)
 fi
 
-pass=0
-fail=0
-fail_expected=0
-total=0
-
 excludes="--excludes-dir=test/excludes/"
 excludes="$excludes --excludes-dir=${excludedir}/any/"
 excludes="$excludes --excludes-dir=${excludedir}/$(dpkg-architecture -qDEB_HOST_ARCH)/"
+excludes="$excludes --excludes-dir=${excludedir}/autopkgtest/"
 
+run_tests=''
 for t in $tests; do
-  if ruby2.7 test/runner.rb -v $excludes --name='!/memory_leak/' $t >log 2>&1; then
-    echo "PASS $t"
-    pass=$(($pass + 1))
-  else
-    if grep "^$t$" $KNOW_FAILURES; then
-      fail_expected=$(($fail_expected + 1))
-      echo "FAIL (EXPECTED) $t"
-      echo "FAIL (EXPECTED) $t" | sed -e 's/./-/g'
-    else
-      fail=$(($fail + 1))
-      echo "FAIL $t"
-      echo "FAIL $t" | sed -e 's/./-/g'
-    fi
-    echo
-    cat log
+  if ! grep -q "^$t$" "$skiplist"; then
+    run_tests="$run_tests $t"
   fi
-  total=$(($total + 1))
 done
-rm -f log
-
-echo
-echo "Finished"
-echo '--------'
-echo "   Tests executed: $(($total))"
-echo "             PASS: $pass"
-echo "             FAIL: $fail"
-echo "EXPECTED FAILURES: $fail_expected"
-echo
 
-if [ $fail -gt 0 ]; then
-  exit 1
-fi
+ruby2.7 test/runner.rb -v $excludes --name='!/memory_leak/' $run_tests
diff --git a/debian/tests/skiplist b/debian/tests/skiplist
@@ -1,30 +1,19 @@
-test/erb/test_erb_command.rb
-test/mkmf/test_config.rb
-test/mkmf/test_constant.rb
-test/mkmf/test_convertible.rb
-test/mkmf/test_find_executable.rb
+test/mkmf/test_libs.rb
+test/mkmf/test_have_macro.rb
 test/mkmf/test_flags.rb
-test/mkmf/test_framework.rb
+test/mkmf/test_sizeof.rb
+test/mkmf/test_convertible.rb
 test/mkmf/test_have_func.rb
+test/mkmf/test_framework.rb
+test/mkmf/test_find_executable.rb
+test/mkmf/test_config.rb
 test/mkmf/test_have_library.rb
-test/mkmf/test_have_macro.rb
-test/mkmf/test_libs.rb
+test/mkmf/base.rb
 test/mkmf/test_signedness.rb
-test/mkmf/test_sizeof.rb
+test/mkmf/test_constant.rb
 test/rubygems/test_gem_commands_cert_command.rb
-test/rubygems/test_gem_commands_environment_command.rb
-test/rubygems/test_gem_commands_uninstall_command.rb
-test/rubygems/test_gem_commands_update_command.rb
-test/rubygems/test_gem_ext_rake_builder.rb
-test/rubygems/test_gem_installer.rb
-test/rubygems/test_gem_package.rb
-test/rubygems/test_gem_package_tar_writer.rb
-test/rubygems/test_gem.rb
 test/rubygems/test_gem_request.rb
-test/rubygems/test_gem_security_policy.rb
 test/rubygems/test_gem_security.rb
+test/rubygems/test_gem_security_policy.rb
 test/rubygems/test_gem_security_signer.rb
 test/rubygems/test_gem_security_trust_dir.rb
-test/rubygems/test_gem_specification.rb
-test/ruby/test_fiber.rb
-test/ruby/test_io.rb

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+exclude :test_fatal_in_fiber, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+exclude :test_execute, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+exclude :test_execute_removes_executable, 'depends on source tree'`
	`2`	`+exclude :test_execute_prerelease, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+exclude :test_update_rubygems_arguments_1_8_x, 'depends on source tree'`
	`2`	`+exclude :test_update_rubygems_arguments, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+exclude :test_class_build, 'depends on source tree'`
	`2`	`+exclude :test_class_build_with_args, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+exclude :test_add_file_signer, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+exclude :test_base_dir_default, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+exclude :test_seek, 'depends on source tree'`
	`2`	`+exclude :test_seek_symwhence, 'depends on source tree'`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+exclude :'test_parse_files:ext', 'depends on source tree'`
	`2`	`+exclude :'test_parse_files:lib', 'depends on source tree'`
	`3`	`+exclude :'test_parse_files:sample', 'depends on source tree'`