Retry some of the occasionally flaky tests when failing on the CI (#3320)

Author: Gedochao

modules/integration/src/test/scala/scala/cli/integration/BspTestDefinitions.scala

@@ -1907,74 +1907,78 @@ abstract class BspTestDefinitions extends ScalaCliSuite with TestScalaVersionArg
     }
 
   test("BSP respects JAVA_HOME") {
-    val javaVersion = "22"
-    val inputs = TestInputs(os.rel / "check-java.sc" ->
-      s"""assert(System.getProperty("java.version").startsWith("$javaVersion"))
-         |println(System.getProperty("java.home"))""".stripMargin)
-    inputs.fromRoot { root =>
-      os.proc(TestUtil.cli, "bloop", "exit", "--power").call(cwd = root)
-      val java22Home =
-        os.Path(
-          os.proc(TestUtil.cs, "java-home", "--jvm", s"zulu:$javaVersion").call().out.trim(),
-          os.pwd
-        )
-      os.proc(TestUtil.cli, "setup-ide", "check-java.sc")
-        .call(cwd = root, env = Map("JAVA_HOME" -> java22Home.toString()))
-      val ideOptionsPath = root / Constants.workspaceDirName / "ide-options-v2.json"
-      expect(ideOptionsPath.toNIO.toFile.exists())
-      val ideEnvsPath = root / Constants.workspaceDirName / "ide-envs.json"
-      expect(ideEnvsPath.toNIO.toFile.exists())
-      val jsonOptions = List("--json-options", ideOptionsPath.toString)
-      val envOptions  = List("--envs-file", ideEnvsPath.toString)
-      withBsp(inputs, Seq("."), bspOptions = jsonOptions ++ envOptions, reuseRoot = Some(root)) {
-        (_, _, remoteServer) =>
-          async {
-            val targets = await(remoteServer.workspaceBuildTargets().asScala)
-              .getTargets.asScala
-              .filter(!_.getId.getUri.contains("-test"))
-              .map(_.getId())
-            val compileResult =
-              await(remoteServer.buildTargetCompile(new b.CompileParams(targets.asJava)).asScala)
-            expect(compileResult.getStatusCode == b.StatusCode.OK)
-            val runResult =
-              await(remoteServer.buildTargetRun(new b.RunParams(targets.head)).asScala)
-            expect(runResult.getStatusCode == b.StatusCode.OK)
-          }
+    TestUtil.retryOnCi() {
+      val javaVersion = "22"
+      val inputs = TestInputs(os.rel / "check-java.sc" ->
+        s"""assert(System.getProperty("java.version").startsWith("$javaVersion"))
+           |println(System.getProperty("java.home"))""".stripMargin)
+      inputs.fromRoot { root =>
+        os.proc(TestUtil.cli, "bloop", "exit", "--power").call(cwd = root)
+        val java22Home =
+          os.Path(
+            os.proc(TestUtil.cs, "java-home", "--jvm", s"zulu:$javaVersion").call().out.trim(),
+            os.pwd
+          )
+        os.proc(TestUtil.cli, "setup-ide", "check-java.sc")
+          .call(cwd = root, env = Map("JAVA_HOME" -> java22Home.toString()))
+        val ideOptionsPath = root / Constants.workspaceDirName / "ide-options-v2.json"
+        expect(ideOptionsPath.toNIO.toFile.exists())
+        val ideEnvsPath = root / Constants.workspaceDirName / "ide-envs.json"
+        expect(ideEnvsPath.toNIO.toFile.exists())
+        val jsonOptions = List("--json-options", ideOptionsPath.toString)
+        val envOptions  = List("--envs-file", ideEnvsPath.toString)
+        withBsp(inputs, Seq("."), bspOptions = jsonOptions ++ envOptions, reuseRoot = Some(root)) {
+          (_, _, remoteServer) =>
+            async {
+              val targets = await(remoteServer.workspaceBuildTargets().asScala)
+                .getTargets.asScala
+                .filter(!_.getId.getUri.contains("-test"))
+                .map(_.getId())
+              val compileResult =
+                await(remoteServer.buildTargetCompile(new b.CompileParams(targets.asJava)).asScala)
+              expect(compileResult.getStatusCode == b.StatusCode.OK)
+              val runResult =
+                await(remoteServer.buildTargetRun(new b.RunParams(targets.head)).asScala)
+              expect(runResult.getStatusCode == b.StatusCode.OK)
+            }
+        }
       }
     }
   }
 
   test("BSP respects --java-home") {
-    val javaVersion = "22"
-    val inputs = TestInputs(os.rel / "check-java.sc" ->
-      s"""assert(System.getProperty("java.version").startsWith("$javaVersion"))
-         |println(System.getProperty("java.home"))""".stripMargin)
-    inputs.fromRoot { root =>
-      os.proc(TestUtil.cli, "bloop", "exit", "--power").call(cwd = root)
-      val java22Home =
-        os.Path(
-          os.proc(TestUtil.cs, "java-home", "--jvm", s"zulu:$javaVersion").call().out.trim(),
-          os.pwd
-        )
-      os.proc(TestUtil.cli, "setup-ide", "check-java.sc", "--java-home", java22Home.toString())
-        .call(cwd = root)
-      val ideOptionsPath = root / Constants.workspaceDirName / "ide-options-v2.json"
-      expect(ideOptionsPath.toNIO.toFile.exists())
-      val jsonOptions = List("--json-options", ideOptionsPath.toString)
-      withBsp(inputs, Seq("."), bspOptions = jsonOptions, reuseRoot = Some(root)) {
-        (_, _, remoteServer) =>
-          async {
-            val targets = await(remoteServer.workspaceBuildTargets().asScala)
-              .getTargets.asScala
-              .filter(!_.getId.getUri.contains("-test"))
-              .map(_.getId())
-            val compileResult =
-              await(remoteServer.buildTargetCompile(new b.CompileParams(targets.asJava)).asScala)
-            expect(compileResult.getStatusCode == b.StatusCode.OK)
-            val runResult =
-              await(remoteServer.buildTargetRun(new b.RunParams(targets.head)).asScala)
-            expect(runResult.getStatusCode == b.StatusCode.OK)
-          }
+    TestUtil.retryOnCi() {
+      val javaVersion = "22"
+      val inputs = TestInputs(os.rel / "check-java.sc" ->
+        s"""assert(System.getProperty("java.version").startsWith("$javaVersion"))
+           |println(System.getProperty("java.home"))""".stripMargin)
+      inputs.fromRoot { root =>
+        os.proc(TestUtil.cli, "bloop", "exit", "--power").call(cwd = root)
+        val java22Home =
+          os.Path(
+            os.proc(TestUtil.cs, "java-home", "--jvm", s"zulu:$javaVersion").call().out.trim(),
+            os.pwd
+          )
+        os.proc(TestUtil.cli, "setup-ide", "check-java.sc", "--java-home", java22Home.toString())
+          .call(cwd = root)
+        val ideOptionsPath = root / Constants.workspaceDirName / "ide-options-v2.json"
+        expect(ideOptionsPath.toNIO.toFile.exists())
+        val jsonOptions = List("--json-options", ideOptionsPath.toString)
+        withBsp(inputs, Seq("."), bspOptions = jsonOptions, reuseRoot = Some(root)) {
+          (_, _, remoteServer) =>
+            async {
+              val targets = await(remoteServer.workspaceBuildTargets().asScala)
+                .getTargets.asScala
+                .filter(!_.getId.getUri.contains("-test"))
+                .map(_.getId())
+              val compileResult =
+                await(remoteServer.buildTargetCompile(new b.CompileParams(targets.asJava)).asScala)
+              expect(compileResult.getStatusCode == b.StatusCode.OK)
+              val runResult =
+                await(remoteServer.buildTargetRun(new b.RunParams(targets.head)).asScala)
+              expect(runResult.getStatusCode == b.StatusCode.OK)
+            }
+        }
       }
     }
   }

modules/integration/src/test/scala/scala/cli/integration/ConfigTests.scala

@@ -175,87 +175,89 @@ class ConfigTests extends ScalaCliSuite {
 
   if (TestUtil.isNativeCli)
     test(s"Create a PGP key with external JVM process, java version too low") {
-      TestInputs().fromRoot { root =>
-        val configFile = {
-          val dir = root / "config"
-          os.makeDir.all(dir, perms = if (Properties.isWin) null else "rwx------")
-          dir / "config.json"
-        }
-
-        val java8Home =
-          os.Path(os.proc(TestUtil.cs, "java-home", "--jvm", "zulu:8").call().out.trim(), os.pwd)
-
-        val extraEnv = Map(
-          "JAVA_HOME" -> java8Home.toString,
-          "PATH" -> ((java8Home / "bin").toString + File.pathSeparator + System.getenv("PATH")),
-          "SCALA_CLI_CONFIG" -> configFile.toString
-        )
-
-        val pgpCreated = os.proc(
-          TestUtil.cli,
-          "--power",
-          "config",
-          "--create-pgp-key",
-          "--email",
-          "[email protected]",
-          "--pgp-password",
-          "none",
-          "--force-jvm-signing-cli",
-          "-v",
-          "-v",
-          "-v"
-        )
-          .call(cwd = root, env = extraEnv, mergeErrIntoOut = true)
+      TestUtil.retryOnCi() {
+        TestInputs().fromRoot { root =>
+          val configFile = {
+            val dir = root / "config"
+            os.makeDir.all(dir, perms = if (Properties.isWin) null else "rwx------")
+            dir / "config.json"
+          }
+
+          val java8Home =
+            os.Path(os.proc(TestUtil.cs, "java-home", "--jvm", "zulu:8").call().out.trim(), os.pwd)
+
+          val extraEnv = Map(
+            "JAVA_HOME" -> java8Home.toString,
+            "PATH" -> ((java8Home / "bin").toString + File.pathSeparator + System.getenv("PATH")),
+            "SCALA_CLI_CONFIG" -> configFile.toString
+          )
+
+          val pgpCreated = os.proc(
+            TestUtil.cli,
+            "--power",
+            "config",
+            "--create-pgp-key",
+            "--email",
+            "[email protected]",
+            "--pgp-password",
+            "none",
+            "--force-jvm-signing-cli",
+            "-v",
+            "-v",
+            "-v"
+          )
+            .call(cwd = root, env = extraEnv, mergeErrIntoOut = true)
 
-        val javaCommandLine = pgpCreated.out.text()
-          .linesIterator
-          .dropWhile(!_.equals("  Running")).slice(1, 2)
-          .toSeq
+          val javaCommandLine = pgpCreated.out.text()
+            .linesIterator
+            .dropWhile(!_.equals("  Running")).slice(1, 2)
+            .toSeq
 
-        expect(javaCommandLine.nonEmpty)
-        expect(javaCommandLine.head.contains("17"))
+          expect(javaCommandLine.nonEmpty)
+          expect(javaCommandLine.head.contains("17"))
 
-        val passwordInConfig = os.proc(TestUtil.cli, "--power", "config", "pgp.secret-key-password")
-          .call(cwd = root, env = extraEnv, stderr = os.Pipe)
-        expect(passwordInConfig.out.text().isEmpty())
+          val passwordInConfig =
+            os.proc(TestUtil.cli, "--power", "config", "pgp.secret-key-password")
+              .call(cwd = root, env = extraEnv, stderr = os.Pipe)
+          expect(passwordInConfig.out.text().isEmpty())
 
-        val secretKey = os.proc(TestUtil.cli, "--power", "config", "pgp.secret-key")
-          .call(cwd = root, env = extraEnv, stderr = os.Pipe)
-          .out.trim()
-        val rawPublicKey =
-          os.proc(TestUtil.cli, "--power", "config", "pgp.public-key", "--password-value")
+          val secretKey = os.proc(TestUtil.cli, "--power", "config", "pgp.secret-key")
             .call(cwd = root, env = extraEnv, stderr = os.Pipe)
             .out.trim()
-
-        val tmpFile    = root / "test-file"
-        val tmpFileAsc = root / "test-file.asc"
-        os.write(tmpFile, "Hello")
-
-        val q = "\""
-
-        def maybeEscape(arg: String): String =
-          if (Properties.isWin) q + arg + q
-          else arg
-
-        os.proc(
-          TestUtil.cli,
-          "--power",
-          "pgp",
-          "sign",
-          "--secret-key",
-          maybeEscape(secretKey),
-          tmpFile
-        ).call(cwd = root, stdin = os.Inherit, stdout = os.Inherit, env = extraEnv)
-
-        val pubKeyFile = root / "key.pub"
-        os.write(pubKeyFile, rawPublicKey)
-        val verifyResult =
-          os.proc(TestUtil.cli, "--power", "pgp", "verify", "--key", pubKeyFile, tmpFileAsc)
-            .call(cwd = root, env = extraEnv, mergeErrIntoOut = true)
-
-        expect(verifyResult.out.text().contains("valid signature"))
+          val rawPublicKey =
+            os.proc(TestUtil.cli, "--power", "config", "pgp.public-key", "--password-value")
+              .call(cwd = root, env = extraEnv, stderr = os.Pipe)
+              .out.trim()
+
+          val tmpFile    = root / "test-file"
+          val tmpFileAsc = root / "test-file.asc"
+          os.write(tmpFile, "Hello")
+
+          val q = "\""
+
+          def maybeEscape(arg: String): String =
+            if (Properties.isWin) q + arg + q
+            else arg
+
+          os.proc(
+            TestUtil.cli,
+            "--power",
+            "pgp",
+            "sign",
+            "--secret-key",
+            maybeEscape(secretKey),
+            tmpFile
+          ).call(cwd = root, stdin = os.Inherit, stdout = os.Inherit, env = extraEnv)
+
+          val pubKeyFile = root / "key.pub"
+          os.write(pubKeyFile, rawPublicKey)
+          val verifyResult =
+            os.proc(TestUtil.cli, "--power", "pgp", "verify", "--key", pubKeyFile, tmpFileAsc)
+              .call(cwd = root, env = extraEnv, mergeErrIntoOut = true)
+
+          expect(verifyResult.out.text().contains("valid signature"))
+        }
       }
-
     }
 
   def createDefaultPgpKeyTest(pgpPasswordOption: String): Unit = {