Skip to content

Commit 4ab3227

Browse files
GedochaoGedochao
authored
Retry some more flaky tests on the CI (#3382)
1 parent e86e87f commit 4ab3227

File tree

5 files changed

+89
-76
lines changed

5 files changed

+89
-76
lines changed

‎modules/integration/src/test/scala/scala/cli/integration/GitHubTests.scala

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,20 +51,24 @@ class GitHubTests extends ScalaCliSuite {
5151
}
5252
}
5353

54-
override def munitFlakyOK: Boolean = TestUtil.isCI && Properties.isMac
54+
override def munitFlakyOK: Boolean = TestUtil.isCI
55+
56+
def createSecret(): Unit = {
57+
try
58+
createSecretTest()
59+
catch {
60+
case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") =>
61+
fail("libsodium, couldn't be loaded")
62+
}
63+
64+
}
5565

5666
// currently having issues loading libsodium from the static launcher
5767
// that launcher is mainly meant to be used on CIs or from docker, missing
5868
// that feature shouldn't be a big deal there
5969
if (TestUtil.cliKind != "native-static")
60-
test("create secret".flaky) {
61-
try
62-
createSecretTest()
63-
catch {
64-
case e: UnsatisfiedLinkError if e.getMessage.contains("libsodium") =>
65-
fail("libsodium, couldn't be loaded")
66-
}
67-
}
70+
if (Properties.isMac) test("create secret".flaky)(createSecret())
71+
else test("create secret")(TestUtil.retryOnCi()(createSecret()))
6872

6973
}
7074

‎modules/integration/src/test/scala/scala/cli/integration/RunGistTestDefinitions.scala

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -46,14 +46,16 @@ trait RunGistTestDefinitions { _: RunTestDefinitions =>
4646
}
4747

4848
test("Github Gists Script URL") {
49-
val url =
50-
"https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec"
51-
val message = "Hello"
52-
emptyInputs.fromRoot { root =>
53-
val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url))
54-
.call(cwd = root)
55-
.out.trim()
56-
expect(output == message)
49+
TestUtil.retryOnCi() {
50+
val url =
51+
"https://gist.github.com/alexarchambault/7b4ec20c4033690dd750ffd601e540ec"
52+
val message = "Hello"
53+
emptyInputs.fromRoot { root =>
54+
val output = os.proc(TestUtil.cli, extraOptions, escapedUrls(url))
55+
.call(cwd = root)
56+
.out.trim()
57+
expect(output == message)
58+
}
5759
}
5860
}
5961
}

‎modules/integration/src/test/scala/scala/cli/integration/RunTestDefinitions.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1905,7 +1905,7 @@ abstract class RunTestDefinitions
19051905
test(
19061906
s"offline mode should fail on missing artifacts (with Scala $actualAnnouncedScalaVersion)"
19071907
) {
1908-
TestUtil.retryOnCi() {
1908+
TestUtil.retryOnCi(maxAttempts = 5) {
19091909
// Kill bloop deamon to test scalac fallback
19101910
os.proc(TestUtil.cli, "--power", "bloop", "exit")
19111911
.call(cwd = os.pwd)

‎modules/integration/src/test/scala/scala/cli/integration/TestTestDefinitions.scala

Lines changed: 64 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -556,70 +556,77 @@ abstract class TestTestDefinitions extends ScalaCliSuite with TestScalaVersionAr
556556

557557
for ((platformName, platformArgs) <- platforms)
558558
test(s"custom test framework $platformName") {
559-
val inputs = TestInputs(
560-
os.rel / "MyTests.test.scala" ->
561-
s"""//> using dep com.lihaoyi::utest::$utestVersion
562-
|
563-
|package mytests
564-
|import utest._
565-
|
566-
|object MyTests extends TestSuite {
567-
| val tests = Tests {
568-
| test("foo") {
569-
| assert(2 + 2 == 4)
570-
| println("Hello from " + "tests")
571-
| }
572-
| }
573-
|}
574-
|""".stripMargin,
575-
os.rel / "CustomFramework.test.scala" ->
576-
"""package custom
577-
|
578-
|class CustomFramework extends utest.runner.Framework {
579-
| override def setup(): Unit =
580-
| println("Hello from CustomFramework")
581-
|}
582-
|""".stripMargin
583-
)
584-
inputs.fromRoot { root =>
585-
val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".")
586-
.call(cwd = root)
587-
val baseOutput = baseRes.out.text()
588-
expect(baseOutput.contains("Hello from tests"))
589-
expect(!baseOutput.contains("Hello from CustomFramework"))
590-
591-
val cmd = Seq[os.Shellable](
592-
TestUtil.cli,
593-
"test",
594-
extraOptions,
595-
platformArgs,
596-
".",
597-
"--test-framework",
598-
"custom.CustomFramework"
559+
TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) {
560+
val inputs = TestInputs(
561+
os.rel / "MyTests.test.scala" ->
562+
s"""//> using dep com.lihaoyi::utest::$utestVersion
563+
|
564+
|package mytests
565+
|import utest._
566+
|
567+
|object MyTests extends TestSuite {
568+
| val tests = Tests {
569+
| test("foo") {
570+
| assert(2 + 2 == 4)
571+
| println("Hello from " + "tests")
572+
| }
573+
| }
574+
|}
575+
|""".stripMargin,
576+
os.rel / "CustomFramework.test.scala" ->
577+
"""package custom
578+
|
579+
|class CustomFramework extends utest.runner.Framework {
580+
| override def setup(): Unit =
581+
| println("Hello from CustomFramework")
582+
|}
583+
|""".stripMargin
599584
)
600-
val res = os.proc(cmd).call(cwd = root)
601-
val output = res.out.text()
602-
expect(output.contains("Hello from tests"))
603-
expect(output.contains("Hello from CustomFramework"))
585+
inputs.fromRoot { root =>
586+
val baseRes = os.proc(TestUtil.cli, "test", extraOptions, platformArgs, ".")
587+
.call(cwd = root)
588+
val baseOutput = baseRes.out.text()
589+
expect(baseOutput.contains("Hello from tests"))
590+
expect(!baseOutput.contains("Hello from CustomFramework"))
591+
592+
val cmd = Seq[os.Shellable](
593+
TestUtil.cli,
594+
"test",
595+
extraOptions,
596+
platformArgs,
597+
".",
598+
"--test-framework",
599+
"custom.CustomFramework"
600+
)
601+
val res = os.proc(cmd).call(cwd = root)
602+
val output = res.out.text()
603+
expect(output.contains("Hello from tests"))
604+
expect(output.contains("Hello from CustomFramework"))
605+
}
604606
}
605607
}
606608

607609
for ((platformName, platformArgs) <- platforms)
608610
test(s"Fail if no tests were run $platformName") {
609-
val inputs = TestInputs(
610-
os.rel / "MyTests.test.scala" ->
611-
s"""//> using dep org.scalameta::munit::$munitVersion
612-
|
613-
|object MyTests
614-
|""".stripMargin
615-
)
611+
TestUtil.retryOnCi(maxAttempts = if (platformName.contains("native")) 3 else 1) {
612+
val inputs = TestInputs(
613+
os.rel / "MyTests.test.scala" ->
614+
s"""//> using dep org.scalameta::munit::$munitVersion
615+
|
616+
|object MyTests
617+
|""".stripMargin
618+
)
616619

617-
inputs.fromRoot { root =>
618-
val res = os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".")
619-
.call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false)
620-
expect(res.exitCode != 0)
621-
val output = res.out.text()
622-
expect(output.contains("Error: no tests were run") || output.contains("No tests were run"))
620+
inputs.fromRoot { root =>
621+
val res =
622+
os.proc(TestUtil.cli, "test", extraOptions, "--require-tests", platformArgs, ".")
623+
.call(cwd = root, stderr = os.Pipe, mergeErrIntoOut = true, check = false)
624+
expect(res.exitCode != 0)
625+
val output = res.out.text()
626+
expect(
627+
output.contains("Error: no tests were run") || output.contains("No tests were run")
628+
)
629+
}
623630
}
624631
}
625632

‎modules/integration/src/test/scala/scala/cli/integration/UpdateTests.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ class UpdateTests extends ScalaCliSuite {
108108

109109
if (!Properties.isWin && Constants.ghOrg == "VirtusLab" && Constants.ghName == "scala-cli")
110110
test("updating dummy scala-cli using update command") {
111-
runUpdate()
111+
TestUtil.retryOnCi()(runUpdate())
112112
}
113113

114114
test("run update before run/test/compile should not return exit code") {

0 commit comments

Comments
 (0)