Add an option to only rerun tests that have failed (#16263)

- testCompilation supports running only failed tests with --failed
argument

fixes #3798

Author: prolativ

compiler/test/dotc/comptest.scala

@@ -12,6 +12,7 @@ object comptest extends ParallelTesting {
   def isInteractive = true
   def testFilter = Nil
   def updateCheckFiles: Boolean = false
+  def failedTests = None
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"

compiler/test/dotty/Properties.scala

@@ -13,6 +13,10 @@ object Properties {
     prop == null || prop == "TRUE"
   }
 
+  /** If property is unset or FALSE we consider it `false` */
+  private def propIsTrue(name: String): Boolean =
+    sys.props.getOrElse(name, "FALSE") == "TRUE"
+
   /** Are we running on the CI? */
   val isRunByCI: Boolean = sys.env.isDefinedAt("DOTTY_CI_RUN")
   || sys.env.isDefinedAt("DRONE")  // TODO remove this when we drop Drone
@@ -30,9 +34,11 @@ object Properties {
    */
   val testsFilter: List[String] = sys.props.get("dotty.tests.filter").fold(Nil)(_.split(',').toList)
 
+  /** Run only failed tests */
+  val rerunFailed: Boolean = propIsTrue("dotty.tests.rerunFailed")
+
   /** Tests should override the checkfiles with the current output */
-  val testsUpdateCheckfile: Boolean =
-    sys.props.getOrElse("dotty.tests.updateCheckfiles", "FALSE") == "TRUE"
+  val testsUpdateCheckfile: Boolean = propIsTrue("dotty.tests.updateCheckfiles")
 
   /** When set, the run tests are only compiled - not run, a warning will be
    *  issued

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -10,6 +10,7 @@ import org.junit.Assume._
 import org.junit.experimental.categories.Category
 
 import scala.concurrent.duration._
+import reporting.TestReporter
 import vulpix._
 
 import java.nio.file._
@@ -214,6 +215,7 @@ object BootstrappedOnlyCompilationTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -16,6 +16,7 @@ import scala.jdk.CollectionConverters._
 import scala.util.matching.Regex
 import scala.concurrent.duration._
 import TestSources.sources
+import reporting.TestReporter
 import vulpix._
 
 class CompilationTests {
@@ -317,6 +318,7 @@ object CompilationTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -5,6 +5,7 @@ package dotc
 import scala.language.unsafeNulls
 
 import org.junit.{AfterClass, Test}
+import reporting.TestReporter
 import vulpix._
 
 import java.io.{File => JFile}
@@ -48,6 +49,7 @@ object FromTastyTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -12,6 +12,7 @@ import org.junit.{AfterClass, Test}
 import org.junit.experimental.categories.Category
 
 import scala.concurrent.duration._
+import reporting.TestReporter
 import vulpix._
 
 
@@ -76,6 +77,7 @@ object IdempotencyTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {

compiler/test/dotty/tools/dotc/TastyBootstrapTests.scala

@@ -17,6 +17,7 @@ import scala.util.matching.Regex
 import scala.concurrent.duration._
 import TestSources.sources
 import vulpix._
+import reporting.TestReporter
 
 class TastyBootstrapTests {
   import ParallelTesting._
@@ -114,6 +115,7 @@ object TastyBootstrapTests extends ParallelTesting {
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
   def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   @AfterClass def tearDown(): Unit = {

compiler/test/dotty/tools/dotc/coverage/CoverageTests.scala

@@ -4,13 +4,13 @@ import org.junit.Test
 import org.junit.AfterClass
 import org.junit.Assert.*
 import org.junit.experimental.categories.Category
-
 import dotty.{BootstrappedOnlyTests, Properties}
 import dotty.tools.vulpix.*
 import dotty.tools.vulpix.TestConfiguration.*
 import dotty.tools.dotc.Main
+import dotty.tools.dotc.reporting.TestReporter
 
-import java.nio.file.{Files, FileSystems, Path, Paths, StandardCopyOption}
+import java.nio.file.{FileSystems, Files, Path, Paths, StandardCopyOption}
 import scala.jdk.CollectionConverters.*
 import scala.util.Properties.userDir
 import scala.language.unsafeNulls
@@ -85,6 +85,7 @@ object CoverageTests extends ParallelTesting:
   def testFilter = Properties.testsFilter
   def isInteractive = SummaryReport.isInteractive
   def updateCheckFiles = Properties.testsUpdateCheckfile
+  def failedTests = TestReporter.lastRunFailedTests
 
   given summaryReport: SummaryReporting = SummaryReport()
   @AfterClass def tearDown(): Unit =

compiler/test/dotty/tools/dotc/reporting/TestReporter.scala

@@ -3,18 +3,20 @@ package dotc
 package reporting
 
 import scala.language.unsafeNulls
-
-import java.io.{ PrintStream, PrintWriter, File => JFile, FileOutputStream, StringWriter }
+import java.io.{BufferedReader, FileInputStream, FileOutputStream, FileReader, PrintStream, PrintWriter, StringReader, StringWriter, File as JFile}
 import java.text.SimpleDateFormat
 import java.util.Date
-import core.Decorators._
+import core.Decorators.*
 
 import scala.collection.mutable
-
+import scala.jdk.CollectionConverters.*
 import util.SourcePosition
-import core.Contexts._
-import Diagnostic._
-import interfaces.Diagnostic.{ ERROR, WARNING }
+import core.Contexts.*
+import Diagnostic.*
+import dotty.Properties
+import interfaces.Diagnostic.{ERROR, WARNING}
+
+import scala.io.Codec
 
 class TestReporter protected (outWriter: PrintWriter, filePrintln: String => Unit, logLevel: Int)
 extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with MessageRendering {
@@ -84,17 +86,23 @@ extends Reporter with UniqueMessagePositions with HideNonSensicalMessages with M
 }
 
 object TestReporter {
+  private val testLogsDirName: String = "testlogs"
+  private val failedTestsFileName: String = "last-failed.log"
+  private val failedTestsFile: JFile = new JFile(s"$testLogsDirName/$failedTestsFileName")
+
   private var outFile: JFile = _
   private var logWriter: PrintWriter = _
+  private var failedTestsWriter: PrintWriter = _
 
   private def initLog() = if (logWriter eq null) {
     val date = new Date
     val df0 = new SimpleDateFormat("yyyy-MM-dd")
     val df1 = new SimpleDateFormat("yyyy-MM-dd-'T'HH-mm-ss")
-    val folder = s"testlogs/tests-${df0.format(date)}"
+    val folder = s"$testLogsDirName/tests-${df0.format(date)}"
     new JFile(folder).mkdirs()
     outFile = new JFile(s"$folder/tests-${df1.format(date)}.log")
     logWriter = new PrintWriter(new FileOutputStream(outFile, true))
+    failedTestsWriter = new PrintWriter(new FileOutputStream(failedTestsFile, false))
   }
 
   def logPrintln(str: String) = {
@@ -144,4 +152,16 @@ object TestReporter {
     }
     rep
   }
+
+  def lastRunFailedTests: Option[List[String]] =
+    Option.when(
+      Properties.rerunFailed &&
+        failedTestsFile.exists() &&
+        failedTestsFile.isFile
+    )(java.nio.file.Files.readAllLines(failedTestsFile.toPath).asScala.toList)
+
+  def writeFailedTests(tests: List[String]): Unit =
+    initLog()
+    tests.foreach(failed => failedTestsWriter.println(failed))
+    failedTestsWriter.flush()
 }

compiler/test/dotty/tools/vulpix/FailedTestInfo.scala

@@ -0,0 +1,3 @@
+package dotty.tools.vulpix
+
+case class FailedTestInfo(title: String, extra: String)