Skip to content

Commit c73f5b2

Browse files
GedochaoGedochao
authored
Allow to preconfigure multiple test frameworks (#3653)
1 parent e600737 commit c73f5b2

File tree

14 files changed

+155
-66
lines changed

14 files changed

+155
-66
lines changed

‎modules/build/src/main/scala/scala/build/internal/Runner.scala

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -407,7 +407,7 @@ object Runner {
407407
entrypoint: File,
408408
requireTests: Boolean,
409409
args: Seq[String],
410-
testFrameworkOpt: Option[String],
410+
predefinedTestFrameworks: Seq[String],
411411
logger: Logger,
412412
jsDom: Boolean,
413413
esModule: Boolean
@@ -446,9 +446,9 @@ object Runner {
446446
logger.debug(s"JS tests class path: $classPath")
447447

448448
val parentInspector = new AsmTestRunner.ParentInspector(classPath)
449-
val foundFrameworkNames: List[String] = testFrameworkOpt match {
450-
case some @ Some(_) => some.toList
451-
case None => value(frameworkNames(classPath, parentInspector, logger)).toList
449+
val foundFrameworkNames: List[String] = predefinedTestFrameworks match {
450+
case f if f.nonEmpty => f.toList
451+
case Nil => value(frameworkNames(classPath, parentInspector, logger)).toList
452452
}
453453

454454
val res =
@@ -485,7 +485,7 @@ object Runner {
485485
def testNative(
486486
classPath: Seq[Path],
487487
launcher: File,
488-
frameworkNameOpt: Option[String],
488+
predefinedTestFrameworks: Seq[String],
489489
requireTests: Boolean,
490490
args: Seq[String],
491491
logger: Logger
@@ -494,9 +494,9 @@ object Runner {
494494
logger.debug(s"Native tests class path: $classPath")
495495

496496
val parentInspector = new AsmTestRunner.ParentInspector(classPath)
497-
val foundFrameworkNames: List[String] = frameworkNameOpt match {
498-
case Some(fw) => List(fw)
499-
case None => value(frameworkNames(classPath, parentInspector, logger)).toList
497+
val foundFrameworkNames: List[String] = predefinedTestFrameworks match {
498+
case f if f.nonEmpty => f.toList
499+
case Nil => value(frameworkNames(classPath, parentInspector, logger)).toList
500500
}
501501

502502
val config = ScalaNativeTestAdapter.Config()

‎modules/cli/src/main/scala/scala/cli/commands/test/Test.scala

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ object Test extends ScalaCommand[TestOptions] {
4646
sharedJava.allJavaOpts.map(JavaOpt(_)).map(Positioned.commandLine)
4747
),
4848
testOptions = baseOptions.testOptions.copy(
49-
frameworkOpt = testFramework.map(_.trim).filter(_.nonEmpty),
49+
frameworks = testFrameworks.map(_.trim).filter(_.nonEmpty).map(Positioned.commandLine),
5050
testOnly = testOnly.map(_.trim).filter(_.nonEmpty)
5151
),
5252
internalDependencies = baseOptions.internalDependencies.copy(
@@ -192,7 +192,7 @@ object Test extends ScalaCommand[TestOptions] {
192192
allowExecve: Boolean
193193
): Either[BuildException, Int] = either {
194194

195-
val testFrameworkOpt = build.options.testOptions.frameworkOpt
195+
val predefinedTestFrameworks = build.options.testOptions.frameworks
196196

197197
build.options.platform.value match {
198198
case Platform.JS =>
@@ -215,7 +215,7 @@ object Test extends ScalaCommand[TestOptions] {
215215
js.toIO,
216216
requireTests,
217217
args,
218-
testFrameworkOpt,
218+
predefinedTestFrameworks.map(_.value),
219219
logger,
220220
build.options.scalaJsOptions.dom.getOrElse(false),
221221
esModule
@@ -232,7 +232,7 @@ object Test extends ScalaCommand[TestOptions] {
232232
Runner.testNative(
233233
build.fullClassPath.map(_.toNIO),
234234
launcher.toIO,
235-
testFrameworkOpt,
235+
predefinedTestFrameworks.map(_.value),
236236
requireTests,
237237
args,
238238
logger
@@ -242,15 +242,18 @@ object Test extends ScalaCommand[TestOptions] {
242242
case Platform.JVM =>
243243
val classPath = build.fullClassPathMaybeAsJar(asJar)
244244

245-
val testFrameworkOpt0 = testFrameworkOpt.orElse {
246-
findTestFramework(classPath.map(_.toNIO), logger)
247-
}
245+
val predefinedTestFrameworks0 =
246+
predefinedTestFrameworks match {
247+
case f if f.nonEmpty => f
248+
case Nil =>
249+
findTestFramework(classPath.map(_.toNIO), logger).map(Positioned.none).toList
250+
}
248251
val testOnly = build.options.testOptions.testOnly
249252

250253
val extraArgs =
251254
(if requireTests then Seq("--require-tests") else Nil) ++
252255
build.options.internal.verbosity.map(v => s"--verbosity=$v") ++
253-
testFrameworkOpt0.map(fw => s"--test-framework=$fw").toSeq ++
256+
predefinedTestFrameworks0.map(_.value).map(fw => s"--test-framework=$fw") ++
254257
testOnly.map(to => s"--test-only=$to").toSeq ++
255258
Seq("--") ++ args
256259

‎modules/cli/src/main/scala/scala/cli/commands/test/TestOptions.scala

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,37 +7,36 @@ import scala.cli.commands.shared._
77
import scala.cli.commands.tags
88

99
@HelpMessage(TestOptions.helpMessage, "", TestOptions.detailedHelpMessage)
10-
// format: off
1110
final case class TestOptions(
1211
@Recurse
13-
shared: SharedOptions = SharedOptions(),
12+
shared: SharedOptions = SharedOptions(),
1413
@Recurse
15-
sharedJava: SharedJavaOptions = SharedJavaOptions(),
14+
sharedJava: SharedJavaOptions = SharedJavaOptions(),
1615
@Recurse
17-
watch: SharedWatchOptions = SharedWatchOptions(),
16+
watch: SharedWatchOptions = SharedWatchOptions(),
1817
@Recurse
19-
compileCross: CrossOptions = CrossOptions(),
20-
18+
compileCross: CrossOptions = CrossOptions(),
2119
@Group(HelpGroup.Test.toString)
22-
@HelpMessage("Name of the test framework's runner class to use while running tests")
20+
@HelpMessage(
21+
"""Names of the test frameworks' runner classes to use while running tests.
22+
|Skips framework lookup and only runs passed frameworks.""".stripMargin
23+
)
2324
@ValueDescription("class-name")
2425
@Tag(tags.should)
2526
@Tag(tags.inShortHelp)
26-
testFramework: Option[String] = None,
27-
27+
@Name("testFramework")
28+
testFrameworks: List[String] = Nil,
2829
@Group(HelpGroup.Test.toString)
2930
@Tag(tags.should)
3031
@Tag(tags.inShortHelp)
3132
@HelpMessage("Fail if no test suites were run")
32-
requireTests: Boolean = false,
33+
requireTests: Boolean = false,
3334
@Group(HelpGroup.Test.toString)
3435
@Tag(tags.should)
3536
@Tag(tags.inShortHelp)
3637
@HelpMessage("Specify a glob pattern to filter the tests suite to be run.")
37-
testOnly: Option[String] = None
38-
38+
testOnly: Option[String] = None
3939
) extends HasSharedOptions
40-
// format: on
4140

4241
object TestOptions {
4342
implicit lazy val parser: Parser[TestOptions] = Parser.derive

‎modules/cli/src/main/scala/scala/cli/exportCmd/MillProjectDescriptor.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ final case class MillProjectDescriptor(
148148
Seq.empty
149149
}
150150
val parentInspector = new AsmTestRunner.ParentInspector(testClassPath)
151-
val frameworkName0 = options.testOptions.frameworkOpt.orElse {
151+
val frameworkName0 = options.testOptions.frameworks.headOption.orElse {
152152
frameworkNames(testClassPath, parentInspector, logger).toOption
153153
.flatMap(_.headOption) // TODO: handle multiple frameworks here
154154
}

‎modules/cli/src/main/scala/scala/cli/exportCmd/SbtProjectDescriptor.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,7 @@ final case class SbtProjectDescriptor(
261261
}
262262

263263
val parentInspector = new AsmTestRunner.ParentInspector(testClassPath)
264-
val frameworkName0 = options.testOptions.frameworkOpt.orElse {
264+
val frameworkName0 = options.testOptions.frameworks.headOption.orElse {
265265
frameworkNames(testClassPath, parentInspector, logger).toOption
266266
.flatMap(_.headOption) // TODO: handle multiple frameworks here
267267
}

‎modules/directives/src/main/scala/scala/build/preprocessing/directives/Tests.scala

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,29 @@ import scala.cli.commands.SpecificationLevel
88

99
@DirectiveGroupName("Test framework")
1010
@DirectiveExamples("//> using testFramework utest.runner.Framework")
11+
@DirectiveExamples("//> using test.frameworks utest.runner.Framework munit.Framework")
1112
@DirectiveUsage(
12-
"using testFramework _class_name_",
13+
"""using testFramework _class_name_
14+
|
15+
|using testFrameworks _class_name_ _another_class_name_
16+
|
17+
|using test.framework _class_name_
18+
|
19+
|using test.frameworks _class_name_ _another_class_name_""".stripMargin,
1320
"`//> using testFramework` _class-name_"
1421
)
1522
@DirectiveDescription("Set the test framework")
1623
@DirectiveLevel(SpecificationLevel.SHOULD)
1724
final case class Tests(
25+
@DirectiveName("testFramework")
1826
@DirectiveName("test.framework")
19-
testFramework: Option[String] = None
27+
@DirectiveName("test.frameworks")
28+
testFrameworks: Seq[Positioned[String]] = Nil
2029
) extends HasBuildOptions {
2130
def buildOptions: Either[BuildException, BuildOptions] = {
2231
val buildOpt = BuildOptions(
2332
testOptions = TestOptions(
24-
frameworkOpt = testFramework
33+
frameworks = testFrameworks
2534
)
2635
)
2736
Right(buildOpt)

‎modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -949,5 +949,62 @@ abstract class TestTestDefinitions extends ScalaCliSuite with TestScalaVersionAr
949949
}
950950
}
951951
}
952+
953+
test(s"multiple explicitly preconfigured test frameworks$platformDescription") {
954+
TestUtil.retryOnCi() {
955+
val expectedMessages @ Seq(scalatestMessage, munitMessage, customMessage) =
956+
Seq("Hello from scalatest", "Hello from Munit", "Hello from custom framework")
957+
TestInputs(
958+
os.rel / "project.scala" ->
959+
s"""//> using test.dep org.scalatest::scalatest::3.2.19
960+
|//> using dep com.lihaoyi::utest::$utestVersion
961+
|//> using test.dep org.scalameta::munit::$munitVersion
962+
|//> using test.frameworks org.scalatest.tools.Framework munit.Framework custom.CustomFramework
963+
|""".stripMargin,
964+
os.rel / "scalatest.test.scala" ->
965+
s"""import org.scalatest.flatspec.AnyFlatSpec
966+
|
967+
|class ScalaTestSpec extends AnyFlatSpec {
968+
| "example" should "work" in {
969+
| assertResult(1)(1)
970+
| println("$scalatestMessage")
971+
| }
972+
|}
973+
|""".stripMargin,
974+
os.rel / "munit.test.scala" ->
975+
s"""import munit.FunSuite
976+
|
977+
|class Munit extends FunSuite {
978+
| test("foo") {
979+
| assert(2 + 2 == 4)
980+
| println("$munitMessage")
981+
| }
982+
|}
983+
|""".stripMargin,
984+
os.rel / "custom.test.scala" ->
985+
s"""package custom
986+
|
987+
|class CustomFramework extends utest.runner.Framework {
988+
| override def setup(): Unit =
989+
| println("$customMessage")
990+
|}""".stripMargin
991+
).fromRoot { root =>
992+
val r =
993+
os.proc(
994+
TestUtil.cli,
995+
"test",
996+
extraOptions,
997+
".",
998+
platformOptions
999+
).call(cwd = root)
1000+
val output = r.out.trim()
1001+
expect(output.nonEmpty)
1002+
expectedMessages.foreach { expectedMessage =>
1003+
expect(output.contains(expectedMessage))
1004+
expect(countSubStrings(output, expectedMessage) == 1)
1005+
}
1006+
}
1007+
}
1008+
}
9521009
}
9531010
}

‎modules/options/src/main/scala/scala/build/options/TestOptions.scala

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
package scala.build.options
22

3+
import scala.build.Positioned
4+
35
final case class TestOptions(
4-
frameworkOpt: Option[String] = None,
6+
frameworks: Seq[Positioned[String]] = Nil,
57
testOnly: Option[String] = None
68
)
79

‎modules/test-runner/src/main/scala/scala/build/testrunner/DynamicTestRunner.scala

Lines changed: 31 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -28,23 +28,23 @@ object DynamicTestRunner {
2828

2929
def main(args: Array[String]): Unit = {
3030

31-
val (testFrameworkOpt, requireTests, verbosity, testOnly, args0) = {
31+
val (testFrameworks, requireTests, verbosity, testOnly, args0) = {
3232
@tailrec
3333
def parse(
34-
testFrameworkOpt: Option[String],
34+
testFrameworks: List[String],
3535
reverseTestArgs: List[String],
3636
requireTests: Boolean,
3737
verbosity: Int,
3838
testOnly: Option[String],
3939
args: List[String]
40-
): (Option[String], Boolean, Int, Option[String], List[String]) =
40+
): (List[String], Boolean, Int, Option[String], List[String]) =
4141
args match {
42-
case Nil => (testFrameworkOpt, requireTests, verbosity, testOnly, reverseTestArgs.reverse)
42+
case Nil => (testFrameworks, requireTests, verbosity, testOnly, reverseTestArgs.reverse)
4343
case "--" :: t =>
44-
(testFrameworkOpt, requireTests, verbosity, testOnly, reverseTestArgs.reverse ::: t)
44+
(testFrameworks, requireTests, verbosity, testOnly, reverseTestArgs.reverse ::: t)
4545
case h :: t if h.startsWith("--test-framework=") =>
4646
parse(
47-
Some(h.stripPrefix("--test-framework=")),
47+
testFrameworks ++ List(h.stripPrefix("--test-framework=")),
4848
reverseTestArgs,
4949
requireTests,
5050
verbosity,
@@ -53,7 +53,7 @@ object DynamicTestRunner {
5353
)
5454
case h :: t if h.startsWith("--test-only=") =>
5555
parse(
56-
testFrameworkOpt,
56+
testFrameworks,
5757
reverseTestArgs,
5858
requireTests,
5959
verbosity,
@@ -62,41 +62,47 @@ object DynamicTestRunner {
6262
)
6363
case h :: t if h.startsWith("--verbosity=") =>
6464
parse(
65-
testFrameworkOpt,
65+
testFrameworks,
6666
reverseTestArgs,
6767
requireTests,
6868
h.stripPrefix("--verbosity=").toInt,
6969
testOnly,
7070
t
7171
)
7272
case "--require-tests" :: t =>
73-
parse(testFrameworkOpt, reverseTestArgs, true, verbosity, testOnly, t)
73+
parse(testFrameworks, reverseTestArgs, true, verbosity, testOnly, t)
7474
case h :: t =>
75-
parse(testFrameworkOpt, h :: reverseTestArgs, requireTests, verbosity, testOnly, t)
75+
parse(testFrameworks, h :: reverseTestArgs, requireTests, verbosity, testOnly, t)
7676
}
7777

78-
parse(None, Nil, false, 0, None, args.toList)
78+
parse(Nil, Nil, false, 0, None, args.toList)
7979
}
8080

8181
val logger = Logger(verbosity)
8282

83+
if (testFrameworks.nonEmpty) logger.debug(
84+
s"""Directly passed ${testFrameworks.length} test frameworks:
85+
| - ${testFrameworks.mkString("\n - ")}""".stripMargin
86+
)
87+
8388
val classLoader = Thread.currentThread().getContextClassLoader
8489
val classPath0 = TestRunner.classPath(classLoader)
85-
val frameworks = testFrameworkOpt
86-
.map(loadFramework(classLoader, _))
87-
.map(Seq(_))
88-
.getOrElse {
89-
getFrameworksToRun(
90-
frameworkServices = findFrameworkServices(classLoader),
91-
frameworks = findFrameworks(classPath0, classLoader, TestRunner.commonTestFrameworks)
92-
)(logger) match {
93-
case f if f.nonEmpty => f
94-
case _ if verbosity >= 2 => sys.error("No test framework found")
95-
case _ =>
96-
System.err.println("No test framework found")
97-
sys.exit(1)
90+
val frameworks =
91+
Option(testFrameworks)
92+
.filter(_.nonEmpty)
93+
.map(_.map(loadFramework(classLoader, _)).toSeq)
94+
.getOrElse {
95+
getFrameworksToRun(
96+
frameworkServices = findFrameworkServices(classLoader),
97+
frameworks = findFrameworks(classPath0, classLoader, TestRunner.commonTestFrameworks)
98+
)(logger) match {
99+
case f if f.nonEmpty => f
100+
case _ if verbosity >= 2 => sys.error("No test framework found")
101+
case _ =>
102+
System.err.println("No test framework found")
103+
sys.exit(1)
104+
}
98105
}
99-
}
100106
def classes = {
101107
val keepJars = false // look into dependencies, much slower
102108
listClasses(classPath0, keepJars).map(name => classLoader.loadClass(name))

‎website/docs/reference/cli-options.md

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1762,9 +1762,12 @@ Available in commands:
17621762

17631763
<!-- Automatically generated, DO NOT EDIT MANUALLY -->
17641764

1765-
### `--test-framework`
1765+
### `--test-frameworks`
17661766

1767-
Name of the test framework's runner class to use while running tests
1767+
Aliases: `--test-framework`
1768+
1769+
Names of the test frameworks' runner classes to use while running tests.
1770+
Skips framework lookup and only runs passed frameworks.
17681771

17691772
### `--require-tests`
17701773

0 commit comments

Comments
 (0)