testrunner: automatically allow parent testcase failure (#3737)

## Changes
- Include logs when test is passed on retry so that it's clear it does
not cause the failure.
- Automatically match parent tests cases.

## Why
If you have a subtest failure, parent test fails as well. Rather than
requiring to specify all parent tests, we automatically allow them.
Inspired by this output:

```
acceptance TestAccept/bundle/templates/default-python/combinations/classic/DATABRICKS_BUNDLE_ENGINE=direct-exp/DLT=no/NBOOK=no/PY=no failure is not allowed
acceptance TestAccept/bundle/templates/default-python/combinations/classic/DATABRICKS_BUNDLE_ENGINE=direct-exp/DLT=no/NBOOK=no/PY=yes failure is not allowed
acceptance TestAccept/bundle/templates/default-python/combinations/classic failure is not allowed
acceptance TestAccept failure is not allowed
```

Here parent tests are
'TestAccept/bundle/templates/default-python/combinations/classic' and
'TestAccept'. So if known_failures.txt specifies "acceptance
TestAccept/bundle/templates/default-python/combinations/classic/" then
TestAccept would not match without this PR.

If users specify "acceptance
TestAccept/bundle/templates/default-python/combinations/classic/DATABRICKS_BUNDLE_ENGINE=direct-exp/DLT=no/NBOOK=no/PY=no"
then neither "TestAccept" nor
"TestAccept/bundle/templates/default-python/combinations/classic" would
match without this PR.

## Tests
Unit tests.

Author: denik

tools/testrunner/main.go

@@ -188,7 +188,8 @@ func checkFailures(config *Config, jsonFile string, originalExitCode int) int {
 				fmt.Printf("%s %s failure is not allowed\n", result.Package, result.Test)
 				unexpectedFailures[key] = true
 			}
-		} else if result.Action == "pass" {
+		} else if result.Action == "pass" && unexpectedFailures[key] {
+			fmt.Printf("%s %s passed on retry\n", result.Package, result.Test)
 			// We run gotestsum with --rerun-fails, so we need to account for intermittent failures
 			delete(unexpectedFailures, key)
 		}
@@ -328,20 +329,20 @@ func (r ConfigRule) matches(packageName, testName string) bool {
 
 	// Check test pattern
 	if r.TestPrefix {
-		return matchesPathPrefix(testName, r.TestPattern)
+		return matchesPathPrefix(testName, r.TestPattern) || matchesPathPrefix(r.TestPattern, testName)
 	} else {
-		return testName == r.TestPattern
+		return testName == r.TestPattern || matchesPathPrefix(r.TestPattern, testName)
 	}
 }
 
 // matchesPathPrefix returns true if s matches pattern or starts with pattern + "/"
 // If pattern is empty (wildcard "*"), it matches any string
-func matchesPathPrefix(s, pattern string) bool {
-	if pattern == "" {
+func matchesPathPrefix(s, prefix string) bool {
+	if prefix == "" {
 		return true
 	}
-	if s == pattern {
+	if s == prefix {
 		return true
 	}
-	return strings.HasPrefix(s, pattern+"/")
+	return strings.HasPrefix(s, prefix+"/")
 }

tools/testrunner/main_test.go

@@ -42,6 +42,18 @@ func TestConfigRuleMatches(t *testing.T) {
 		// Empty values cases
 		{"* TestDeploy", "", "TestDeploy", true},
 		{"bundle *", "bundle", "", true},
+
+		// Subtest failure results in parent test failure as well. So we allow strict prefixes to fail as well
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic", "acceptance", "TestAccept", true},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic", "acceptance", "TestAnother", false},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic", "acceptance", "TestAccept/bundle/templates/default-python/combinations/classic/x", false},
+
+		// pattern version
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic/", "acceptance", "TestAccept", true},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic/", "acceptance", "TestAnother", false},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic/", "acceptance", "TestAccept/bundle/templates/default-python/combinations/classic/x", true},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic/", "acceptance", "TestAccept/bundle/templates/default-python/combinations/classic", true},
+		{"acceptance TestAccept/bundle/templates/default-python/combinations/classic/", "acceptance", "TestAccept/bundle/templates/default-python/combinations", true},
 	}
 
 	for _, tt := range tests {