Skip to content

Commit 04d6239

Browse files
lihaoyiautofix-ci[bot]
lihaoyi
and
autofix-ci[bot]
authored
Speed up CI (#4442)
* Increase `-j` parallelism to 0.75C to try and saturate the CPU * Move android parallelism limit out of `action.yml`, where the `if` condition was for some reason not evaluating properly * Split up some big integration suites to take advantage of `testForkGrouping` Fixes #4440. Manually looked at the CI logs to see that the test command is now running without `-j1` and the status ticker is showing multiple tasks running at once --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
1 parent a383b9c commit 04d6239

File tree

9 files changed

+141
-92
lines changed

9 files changed

+141
-92
lines changed

‎.github/actions/post-build-selective/action.yml

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,6 @@ inputs:
1212
default: ''
1313
type: string
1414

15-
install-android-sdk:
16-
default: false
17-
type: boolean
1815
runs:
1916
using: "composite"
2017
steps:
@@ -40,14 +37,7 @@ runs:
4037
#- run: ./mill -i -k selective.resolveChanged ${{ inputs.millargs }}
4138
# shell: ${{ inputs.shell }}
4239

43-
- run: ./mill -i -j1 -k selective.run ${{ inputs.millargs }}
44-
if: ${{ inputs.install-android-sdk }}
45-
env:
46-
COURSIER_ARCHIVE_CACHE: ${{ inputs.coursierarchive }}
47-
shell: ${{ inputs.shell }}
48-
4940
- run: ./mill -i -k selective.run ${{ inputs.millargs }}
50-
if: ${{ !inputs.install-android-sdk }}
5141
env:
5242
COURSIER_ARCHIVE_CACHE: ${{ inputs.coursierarchive }}
5343
shell: ${{ inputs.shell }}

‎.mill-opts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
--jobs=0.5C
1+
--jobs=0.75C

‎example/package.mill

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,9 @@ object `package` extends RootModule with Module {
134134
}
135135
}
136136

137+
trait ExampleCrossModuleAndroid extends ExampleCrossModule {
138+
def testExclusive = true
139+
}
137140
trait ExampleCrossModuleJava extends ExampleCrossModule {
138141

139142
def upstreamCross(s: String) = s match {

‎integration/invalidation/codesig-nested/src/CodeSigNestedTests.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,11 @@ object CodeSigNestedTests extends UtestIntegrationTestSuite {
188188
val addedNewlinesInsideCurlies = eval("outer.inner.qux")
189189
assert(addedNewlinesInsideCurlies.out == "")
190190
}
191+
}
192+
}
191193

194+
object CodeSigNestedTraitTests extends UtestIntegrationTestSuite {
195+
val tests: Tests = Tests {
192196
test("trait") - integrationTest { tester =>
193197
import tester._
194198
val initial = eval("traitOuter.traitInner.inner")

‎integration/invalidation/codesig-scalamodule/src/CodeSigScalaModuleTests.scala

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,14 @@ object CodeSigScalaModuleTests extends UtestIntegrationTestSuite {
120120
)
121121
}
122122

123+
}
124+
}
125+
126+
object CodeSigScalaModuleMultipleTests extends UtestIntegrationTestSuite {
127+
val tests: Tests = Tests {
128+
def filterLines(out: String) = {
129+
out.linesIterator.filter(!_.contains("[info]")).toSet
130+
}
123131
test("multiple") - integrationTest { tester =>
124132
import tester._
125133
// Tests for fine-grained method-based invalidation between multiple ScalaModules,

‎integration/invalidation/codesig-subfolder/src/CodeSigSubfolderTests.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,11 @@ object CodeSigSubfolderTests extends UtestIntegrationTestSuite {
8484

8585
assert(mangledValFooUsedInBar.err.contains("compiling 1 Scala source"))
8686
}
87+
}
88+
}
89+
90+
object CodeSigSubfolderRenamedSameOrderTests extends UtestIntegrationTestSuite {
91+
val tests: Tests = Tests {
8792

8893
test("subfolder-renames-same-order") - integrationTest { tester =>
8994
import tester._
@@ -100,6 +105,11 @@ object CodeSigSubfolderTests extends UtestIntegrationTestSuite {
100105
val cached5 = eval("foo")
101106
assert(!cached5.out.contains("running foo"))
102107
}
108+
}
109+
}
110+
111+
object CodeSigSubfolderRenamedReorderTests extends UtestIntegrationTestSuite {
112+
val tests: Tests = Tests {
103113
test("subfolder-renames-reorder") - integrationTest { tester =>
104114
import tester._
105115
val cached4 = eval("foo")

‎integration/invalidation/selective-execution/src/SelectiveExecutionTests.scala

Lines changed: 66 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,58 @@ import utest._
88
import utest.asserts.{RetryMax, RetryInterval}
99

1010
object SelectiveExecutionTests extends UtestIntegrationTestSuite {
11+
implicit val retryMax: RetryMax = RetryMax(120.seconds)
12+
implicit val retryInterval: RetryInterval = RetryInterval(1.seconds)
13+
val tests: Tests = Tests {
14+
test("default-command") - integrationTest { tester =>
15+
import tester._
16+
17+
eval(("selective.prepare", "bar"), check = true)
18+
19+
val resolve = eval(("selective.resolve", "bar"), check = true)
20+
assert(resolve.out == "")
21+
22+
modifyFile(workspacePath / "bar/bar.txt", _ + "!")
23+
val resolve2 = eval(("selective.resolve", "bar"), check = true)
24+
assert(resolve2.out != "")
25+
26+
val cached = eval(("selective.run", "bar"), check = true, stderr = os.Inherit)
27+
28+
assert(!cached.out.contains("Computing fooCommand"))
29+
assert(cached.out.contains("Computing barCommand"))
30+
}
31+
32+
test("failures") {
33+
test("missing-prepare") - integrationTest { tester =>
34+
import tester._
35+
36+
val cached = eval(
37+
("selective.run", "{foo.fooCommand,bar.barCommand}"),
38+
check = false,
39+
stderr = os.Pipe
40+
)
41+
42+
assert(cached.err.contains("`selective.run` can only be run after `selective.prepare`"))
43+
}
44+
}
45+
test("renamed-tasks") - integrationTest { tester =>
46+
import tester._
47+
eval(("selective.prepare", "{foo,bar}._"), check = true)
48+
49+
modifyFile(workspacePath / "build.mill", _.replace("fooTask", "fooTaskRenamed"))
50+
modifyFile(workspacePath / "build.mill", _.replace("barCommand", "barCommandRenamed"))
51+
52+
val cached = eval(("selective.resolve", "{foo,bar}._"), stderr = os.Pipe)
53+
54+
assert(
55+
cached.out.linesIterator.toList ==
56+
Seq("bar.barCommandRenamed", "foo.fooCommand", "foo.fooTaskRenamed")
57+
)
58+
}
59+
}
60+
}
61+
62+
object SelectiveExecutionChangedInputsTests extends UtestIntegrationTestSuite {
1163
implicit val retryMax: RetryMax = RetryMax(120.seconds)
1264
implicit val retryInterval: RetryInterval = RetryInterval(1.seconds)
1365
val tests: Tests = Tests {
@@ -80,23 +132,13 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite {
80132
assert(!cached.out.contains("Computing fooCommand"))
81133
assert(cached.out.contains("Computing barCommand"))
82134
}
83-
test("default-command") - integrationTest { tester =>
84-
import tester._
85-
86-
eval(("selective.prepare", "bar"), check = true)
87-
88-
val resolve = eval(("selective.resolve", "bar"), check = true)
89-
assert(resolve.out == "")
90-
91-
modifyFile(workspacePath / "bar/bar.txt", _ + "!")
92-
val resolve2 = eval(("selective.resolve", "bar"), check = true)
93-
assert(resolve2.out != "")
94-
95-
val cached = eval(("selective.run", "bar"), check = true, stderr = os.Inherit)
135+
}
136+
}
96137

97-
assert(!cached.out.contains("Computing fooCommand"))
98-
assert(cached.out.contains("Computing barCommand"))
99-
}
138+
object SelectiveExecutionChangedCodeTests extends UtestIntegrationTestSuite {
139+
implicit val retryMax: RetryMax = RetryMax(120.seconds)
140+
implicit val retryInterval: RetryInterval = RetryInterval(1.seconds)
141+
val tests: Tests = Tests {
100142
test("changed-code") - integrationTest { tester =>
101143
import tester._
102144

@@ -138,6 +180,14 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite {
138180
assert(!cached2.out.contains("Computing barCommand"))
139181
}
140182

183+
}
184+
}
185+
186+
object SelectiveExecutionWatchTests extends UtestIntegrationTestSuite {
187+
implicit val retryMax: RetryMax = RetryMax(120.seconds)
188+
implicit val retryInterval: RetryInterval = RetryInterval(1.seconds)
189+
val tests: Tests = Tests {
190+
141191
test("watch") {
142192
test("changed-inputs") - integrationTest { tester =>
143193
import tester._
@@ -239,32 +289,5 @@ object SelectiveExecutionTests extends UtestIntegrationTestSuite {
239289
}
240290
}
241291
}
242-
test("failures") {
243-
test("missing-prepare") - integrationTest { tester =>
244-
import tester._
245-
246-
val cached = eval(
247-
("selective.run", "{foo.fooCommand,bar.barCommand}"),
248-
check = false,
249-
stderr = os.Pipe
250-
)
251-
252-
assert(cached.err.contains("`selective.run` can only be run after `selective.prepare`"))
253-
}
254-
}
255-
test("renamed-tasks") - integrationTest { tester =>
256-
import tester._
257-
eval(("selective.prepare", "{foo,bar}._"), check = true)
258-
259-
modifyFile(workspacePath / "build.mill", _.replace("fooTask", "fooTaskRenamed"))
260-
modifyFile(workspacePath / "build.mill", _.replace("barCommand", "barCommandRenamed"))
261-
262-
val cached = eval(("selective.resolve", "{foo,bar}._"), stderr = os.Pipe)
263-
264-
assert(
265-
cached.out.linesIterator.toList ==
266-
Seq("bar.barCommandRenamed", "foo.fooCommand", "foo.fooTaskRenamed")
267-
)
268-
}
269292
}
270293
}

‎integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala

Lines changed: 44 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -20,49 +20,51 @@ import scala.concurrent.ExecutionContext.Implicits.global
2020
* 4. `interp.watchValue`
2121
* 5. Implicitly watched files, like `build.mill`
2222
*/
23-
object WatchSourceInputTests extends UtestIntegrationTestSuite {
23+
trait WatchTests extends UtestIntegrationTestSuite {
2424

2525
val maxDuration = 120000
26-
val tests: Tests = Tests {
27-
def awaitCompletionMarker(tester: IntegrationTester, name: String) = {
28-
val maxTime = System.currentTimeMillis() + maxDuration
29-
while (!os.exists(tester.workspacePath / "out" / name)) {
30-
if (System.currentTimeMillis() > maxTime) {
31-
sys.error(s"awaitCompletionMarker($name) timed out")
32-
}
33-
Thread.sleep(100)
26+
def awaitCompletionMarker(tester: IntegrationTester, name: String): Unit = {
27+
val maxTime = System.currentTimeMillis() + maxDuration
28+
while (!os.exists(tester.workspacePath / "out" / name)) {
29+
if (System.currentTimeMillis() > maxTime) {
30+
sys.error(s"awaitCompletionMarker($name) timed out")
3431
}
32+
Thread.sleep(100)
3533
}
34+
}
3635

37-
def testBase(show: Boolean)(f: (
38-
mutable.Buffer[String],
39-
mutable.Buffer[String],
40-
mutable.Buffer[String]
41-
) => IntegrationTester.EvalResult): Unit = {
42-
val expectedOut = mutable.Buffer.empty[String]
43-
// Most of these are normal `println`s, so they go to `stdout` by
44-
// default unless you use `show` in which case they go to `stderr`.
45-
val expectedErr = if (show) mutable.Buffer.empty[String] else expectedOut
46-
val expectedShows0 = mutable.Buffer.empty[String]
47-
val res = f(expectedOut, expectedErr, expectedShows0)
48-
val (shows, out) = res.out.linesIterator.toVector.partition(_.startsWith("\""))
49-
val err = res.err.linesIterator.toVector
50-
.filter(!_.contains("Compiling compiler interface..."))
51-
.filter(!_.contains("Watching for changes"))
52-
.filter(!_.contains("[info] compiling"))
53-
.filter(!_.contains("[info] done compiling"))
54-
.filter(!_.contains("mill-server/ exitCode file not found"))
55-
56-
assert(out == expectedOut)
57-
58-
// If show is not enabled, we don't expect any of our custom prints to go to stderr
59-
if (show) assert(err == expectedErr)
60-
else assert(err.isEmpty)
61-
62-
val expectedShows = expectedShows0.map('"' + _ + '"')
63-
if (show) assert(shows == expectedShows)
64-
}
36+
def testBase(show: Boolean)(f: (
37+
mutable.Buffer[String],
38+
mutable.Buffer[String],
39+
mutable.Buffer[String]
40+
) => IntegrationTester.EvalResult): Unit = {
41+
val expectedOut = mutable.Buffer.empty[String]
42+
// Most of these are normal `println`s, so they go to `stdout` by
43+
// default unless you use `show` in which case they go to `stderr`.
44+
val expectedErr = if (show) mutable.Buffer.empty[String] else expectedOut
45+
val expectedShows0 = mutable.Buffer.empty[String]
46+
val res = f(expectedOut, expectedErr, expectedShows0)
47+
val (shows, out) = res.out.linesIterator.toVector.partition(_.startsWith("\""))
48+
val err = res.err.linesIterator.toVector
49+
.filter(!_.contains("Compiling compiler interface..."))
50+
.filter(!_.contains("Watching for changes"))
51+
.filter(!_.contains("[info] compiling"))
52+
.filter(!_.contains("[info] done compiling"))
53+
.filter(!_.contains("mill-server/ exitCode file not found"))
54+
55+
assert(out == expectedOut)
56+
57+
// If show is not enabled, we don't expect any of our custom prints to go to stderr
58+
if (show) assert(err == expectedErr)
59+
else assert(err.isEmpty)
60+
61+
val expectedShows = expectedShows0.map('"' + _ + '"')
62+
if (show) assert(shows == expectedShows)
63+
}
64+
}
6565

66+
object WatchSourceTests extends WatchTests {
67+
val tests: Tests = Tests {
6668
def testWatchSource(tester: IntegrationTester, show: Boolean) =
6769
testBase(show) { (expectedOut, expectedErr, expectedShows) =>
6870
val showArgs = if (show) Seq("show") else Nil
@@ -129,7 +131,6 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite {
129131

130132
Await.result(evalResult, Duration.apply(maxDuration, SECONDS))
131133
}
132-
133134
test("sources") {
134135

135136
// Make sure we clean up the workspace between retries
@@ -148,6 +149,11 @@ object WatchSourceInputTests extends UtestIntegrationTestSuite {
148149
}
149150
}
150151
}
152+
}
153+
}
154+
155+
object WatchInputTests extends WatchTests {
156+
val tests: Tests = Tests {
151157

152158
def testWatchInput(tester: IntegrationTester, show: Boolean) =
153159
testBase(show) { (expectedOut, expectedErr, expectedShows) =>

‎integration/package.mill

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import mill.resolve.SelectMode
1616
import mill.contrib.buildinfo.BuildInfo
1717
import mill.T
1818
import mill.define.Cross
19+
import mill.testrunner.TestResult
1920

2021
// plugins and dependencies
2122
import $meta._
@@ -34,7 +35,11 @@ object `package` extends RootModule {
3435
def runClasspath: T[Seq[PathRef]]
3536
def localRunClasspath: T[Seq[PathRef]]
3637
def forkEnv: T[Map[String, String]]
38+
def testExclusive = false
3739
trait ModeModule extends build.MillBaseTestsModule {
40+
override def test(args: String*) = Task.Command(exclusive = testExclusive) {
41+
testTask(Task.Anon { args }, Task.Anon { Seq.empty[String] })()
42+
}
3843

3944
def mode: String = millModuleSegments.parts.last
4045
def scalaVersion = build.Deps.scalaVersion

0 commit comments

Comments
 (0)