Fail on any test timeout (#2929)

Author: MarcoPolo

scripts/test_analysis/main.go

@@ -51,6 +51,17 @@ func (t *tester) runTests(passThruFlags []string) error {
 	}
 	log.Printf("Not all tests passed: %v", err)
 
+	timedOutPackages, err := t.findTimedoutTests(context.Background())
+	if err != nil {
+		return err
+	}
+	if len(timedOutPackages) > 0 {
+		// Fail immediately if we find any timeouts. We'd have to run all tests
+		// in the package, and this could take a long time.
+		log.Printf("Found %d timed out packages. Failing", len(timedOutPackages))
+		return errors.New("one or more tests timed out")
+	}
+
 	failedTests, err := t.findFailedTests(context.Background())
 	if err != nil {
 		return err
@@ -129,6 +140,11 @@ type failedTest struct {
 	Test    string
 }
 
+type timedOutPackage struct {
+	Package string
+	Outputs string
+}
+
 func (t *tester) findFailedTests(ctx context.Context) ([]failedTest, error) {
 	db, err := sql.Open("sqlite", t.Dir+dbPath)
 	if err != nil {
@@ -151,6 +167,49 @@ func (t *tester) findFailedTests(ctx context.Context) ([]failedTest, error) {
 	return out, nil
 }
 
+func (t *tester) findTimedoutTests(ctx context.Context) ([]timedOutPackage, error) {
+	db, err := sql.Open("sqlite", t.Dir+dbPath)
+	if err != nil {
+		return nil, err
+	}
+	defer db.Close()
+
+	rows, err := db.QueryContext(ctx, `WITH failed_packages AS (
+    SELECT
+        Package
+    FROM
+        test_results
+    WHERE
+        Action = 'fail'
+        AND Elapsed > 300
+)
+SELECT
+	test_results.Package, GROUP_CONCAT(Output, "") as Outputs
+FROM
+    test_results
+INNER JOIN
+    failed_packages
+ON
+    test_results.Package = failed_packages.Package
+GROUP BY
+    test_results.Package
+HAVING
+    Outputs LIKE '%timed out%'
+ORDER BY Time;`)
+	if err != nil {
+		return nil, err
+	}
+	var out []timedOutPackage
+	for rows.Next() {
+		var pkg, outputs string
+		if err := rows.Scan(&pkg, &outputs); err != nil {
+			return nil, err
+		}
+		out = append(out, timedOutPackage{pkg, outputs})
+	}
+	return out, nil
+}
+
 func filterOutFlags(flags []string, exclude *regexp.Regexp) []string {
 	out := make([]string, 0, len(flags))
 	for _, f := range flags {
@@ -170,20 +229,45 @@ func (t *tester) summarize() (string, error) {
 	if err != nil {
 		return "", err
 	}
+	timeouts, err := t.findTimedoutTests(ctx)
+	if err != nil {
+		return "", err
+	}
+
+	testFailureCount := len(testFailures) + len(timeouts)
 
 	plural := "s"
-	if len(testFailures) == 1 {
+	if testFailureCount == 1 {
 		plural = ""
 	}
-	out.WriteString(fmt.Sprintf("## %d Test Failure%s\n\n", len(testFailures), plural))
+	out.WriteString(fmt.Sprintf("## %d Test Failure%s\n\n", testFailureCount, plural))
 
-	db, err := sql.Open("sqlite", t.Dir+dbPath)
-	if err != nil {
-		return "", err
+	if len(timeouts) > 0 {
+		out.WriteString("### Timed Out Tests\n\n")
+		for _, timeout := range timeouts {
+			_, err = out.WriteString(fmt.Sprintf(`<details>
+<summary>%s</summary>
+<pre>
+%s
+</pre>
+</details>`, timeout.Package, timeout.Outputs))
+			if err != nil {
+				return "", err
+			}
+		}
+		out.WriteString("\n")
 	}
-	defer db.Close()
 
-	rows, err := db.QueryContext(ctx, `SELECT
+	if len(testFailures) > 0 {
+		out.WriteString("### Failed Tests\n\n")
+
+		db, err := sql.Open("sqlite", t.Dir+dbPath)
+		if err != nil {
+			return "", err
+		}
+		defer db.Close()
+
+		rows, err := db.QueryContext(ctx, `SELECT
     tr_output.Package,
     tr_output.Test,
     GROUP_CONCAT(tr_output.Output,  "") AS Outputs
@@ -204,22 +288,23 @@ GROUP BY
     tr_output.Test
 ORDER BY
     MIN(tr_output.Time);`)
-	if err != nil {
-		return "", err
-	}
-	for rows.Next() {
-		var pkg, test, outputs string
-		if err := rows.Scan(&pkg, &test, &outputs); err != nil {
+		if err != nil {
 			return "", err
 		}
-		_, err = out.WriteString(fmt.Sprintf(`<details>
+		for rows.Next() {
+			var pkg, test, outputs string
+			if err := rows.Scan(&pkg, &test, &outputs); err != nil {
+				return "", err
+			}
+			_, err = out.WriteString(fmt.Sprintf(`<details>
 <summary>%s.%s</summary>
 <pre>
 %s
 </pre>
 </details>`, pkg, test, outputs))
-		if err != nil {
-			return "", err
+			if err != nil {
+				return "", err
+			}
 		}
 	}
 	return out.String(), nil