diff --git a/core/api/src/mill/api/JsonFormatters.scala b/core/api/src/mill/api/JsonFormatters.scala index 2ebb0bdf127a..3caa447db858 100644 --- a/core/api/src/mill/api/JsonFormatters.scala +++ b/core/api/src/mill/api/JsonFormatters.scala @@ -23,8 +23,8 @@ trait JsonFormatters { implicit val pathReadWrite: RW[os.Path] = upickle.readwriter[String] .bimap[os.Path]( - _.toString, - os.Path(_) + p => MappedRoots.encodeKnownRootsInPath(p), + s => os.Path(MappedRoots.decodeKnownRootsInPath(s)) ) implicit val relPathRW: RW[os.RelPath] = upickle.readwriter[String] diff --git a/core/api/src/mill/api/MappedRoots.scala b/core/api/src/mill/api/MappedRoots.scala new file mode 100644 index 000000000000..96c23ae89ce3 --- /dev/null +++ b/core/api/src/mill/api/MappedRoots.scala @@ -0,0 +1,95 @@ +package mill.api + +import mill.api.internal.NamedParameterOnlyDummy +import mill.constants.PathVars + +import scala.annotation.unused +import scala.util.DynamicVariable + +type MappedRoots = Seq[(key: String, path: os.Path)] + +object MappedRoots extends MappedRootsImpl + +trait MappedRootsImpl { + + private val rootMapping: DynamicVariable[MappedRoots] = DynamicVariable(Seq()) + + def get: MappedRoots = rootMapping.value + + def toMap: Map[String, os.Path] = get.map(m => (m.key, m.path)).toMap + + def withMillDefaults[T]( + @unused t: NamedParameterOnlyDummy = new NamedParameterOnlyDummy, + outPath: os.Path, + workspacePath: os.Path = BuildCtx.workspaceRoot, + homePath: os.Path = os.home + )(thunk: => T): T = withMapping( + Seq( + ("MILL_OUT", outPath), + ("WORKSPACE", workspacePath), + // TODO: add coursier here + ("HOME", homePath) + ) + )(thunk) + + def withMapping[T](mapping: MappedRoots)(thunk: => T): T = withMapping(_ => mapping)(thunk) + + def withMapping[T](mapping: MappedRoots => MappedRoots)(thunk: => T): T = { + val newMapping = mapping(rootMapping.value) + var seenKeys = Set[String]() + var seenPaths = Set[os.Path]() + newMapping.foreach { case m => + require(!m.key.startsWith("$"), "Key must not start with a `$`.") + require(m.key != PathVars.ROOT, s"Invalid key, '${PathVars.ROOT}' is a reserved key name.") + require( + !seenKeys.contains(m.key), + s"Key must be unique, but '${m.key}' was given multiple times." + ) + require( + !seenPaths.contains(m.path), + s"Paths must be unique, but '${m.path}' was given multiple times." + ) + seenKeys += m.key + seenPaths += m.path + } + rootMapping.withValue(newMapping)(thunk) + } + + def encodeKnownRootsInPath(p: os.Path): String = { + MappedRoots.get.collectFirst { + case rep if p.startsWith(rep.path) => + s"$$${rep.key}${ + if (p != rep.path) { + s"/${p.subRelativeTo(rep.path).toString()}" + } else "" + }" + }.getOrElse(p.toString) + } + + def decodeKnownRootsInPath(encoded: String): String = { + if (encoded.startsWith("$")) { + val offset = 1 // "$".length + MappedRoots.get.collectFirst { + case mapping if encoded.startsWith(mapping.key, offset) => + s"${mapping.path.toString}${encoded.substring(mapping.key.length + offset)}" + }.getOrElse(encoded) + } else { + encoded + } + } + + /** + * Use this to assert at runtime, that a root path with the given `key` is defined. + * @throws NoSuchElementException when no path was mapped under the given `key`. + */ + def requireMappedPaths(key: String*): Unit = { + val map = toMap + for { + singleKey <- key + } { + if (!map.contains(singleKey)) + throw new NoSuchElementException(s"No root path mapping defined for '${key}'") + } + } + +} diff --git a/core/api/src/mill/api/PathRef.scala b/core/api/src/mill/api/PathRef.scala index 83bbc3b14cfe..2b1007a1901d 100644 --- a/core/api/src/mill/api/PathRef.scala +++ b/core/api/src/mill/api/PathRef.scala @@ -10,11 +10,12 @@ import java.util.concurrent.ConcurrentHashMap import scala.annotation.nowarn import scala.language.implicitConversions import scala.util.DynamicVariable +import scala.util.hashing.MurmurHash3 /** - * A wrapper around `os.Path` that calculates it's hashcode based - * on the contents of the filesystem underneath it. Used to ensure filesystem - * changes can bust caches which are keyed off hashcodes. + * A wrapper around `os.Path` that calculates a `sig` (which ends up in the [[hashCode]]) + * based on the contents of the filesystem underneath it. + * Used to ensure filesystem changes can bust caches which are keyed off hashcodes. */ case class PathRef private[mill] ( path: os.Path, @@ -24,6 +25,18 @@ case class PathRef private[mill] ( ) extends PathRefApi { private[mill] def javaPath = path.toNIO + /** + * The path with common mapped path roots replaced, to make it relocatable. + * See [[MappedRoots]]. + */ + private val mappedPath: String = MappedRoots.encodeKnownRootsInPath(path) + + /** + * Apply the current contextual path mapping to this PathRef. + * Updates [[mappedPath]] but does not recalculate the [[sig]]. + */ + def remap: PathRef = PathRef(path, quick, sig, revalidate) + def recomputeSig(): Int = PathRef.apply(path, quick).sig def validate(): Boolean = recomputeSig() == sig @@ -38,16 +51,33 @@ case class PathRef private[mill] ( def withRevalidate(revalidate: PathRef.Revalidate): PathRef = copy(revalidate = revalidate) def withRevalidateOnce: PathRef = copy(revalidate = PathRef.Revalidate.Once) - override def toString: String = { + private def toStringPrefix: String = { val quick = if (this.quick) "qref:" else "ref:" + val valid = revalidate match { case PathRef.Revalidate.Never => "v0:" case PathRef.Revalidate.Once => "v1:" case PathRef.Revalidate.Always => "vn:" } val sig = String.format("%08x", this.sig: Integer) - quick + valid + sig + ":" + path.toString() + s"${quick}${valid}${sig}:" + } + + override def toString: String = { + toStringPrefix + path.toString() } + + // Instead of using `path` we need to use `mappedPath` to make the hashcode stable as cache key + override def hashCode(): Int = { + var h = MurmurHash3.productSeed + h = MurmurHash3.mix(h, "PathRef".hashCode) + h = MurmurHash3.mix(h, mappedPath.hashCode) + h = MurmurHash3.mix(h, quick.##) + h = MurmurHash3.mix(h, sig.##) + h = MurmurHash3.mix(h, revalidate.##) + MurmurHash3.finalizeHash(h, 4) + } + } object PathRef { @@ -195,12 +225,12 @@ object PathRef { implicit def jsonFormatter: RW[PathRef] = upickle.readwriter[String].bimap[PathRef]( p => { storeSerializedPaths(p) - p.toString() + p.toStringPrefix + MappedRoots.encodeKnownRootsInPath(p.path) }, { - case s"$prefix:$valid0:$hex:$pathString" if prefix == "ref" || prefix == "qref" => + case s"$prefix:$valid0:$hex:$pathVal" if prefix == "ref" || prefix == "qref" => - val path = os.Path(pathString) + val path = os.Path(MappedRoots.decodeKnownRootsInPath(pathVal)) val quick = prefix match { case "qref" => true case "ref" => false @@ -219,7 +249,7 @@ object PathRef { pr case s => mill.api.BuildCtx.withFilesystemCheckerDisabled( - PathRef(os.Path(s, currentOverrideModulePath.value)) + PathRef(os.Path(MappedRoots.decodeKnownRootsInPath(s), currentOverrideModulePath.value)) ) } ) diff --git a/core/api/test/src/mill/api/MappedRootsTests.scala b/core/api/test/src/mill/api/MappedRootsTests.scala new file mode 100644 index 000000000000..63ac9a391fff --- /dev/null +++ b/core/api/test/src/mill/api/MappedRootsTests.scala @@ -0,0 +1,48 @@ +package mill.api + +import utest.* + +import java.nio.file.Files +import mill.api.{MappedRoots => MR} + +object MappedRootsTests extends TestSuite { + val tests: Tests = Tests { + test("encode") { + withTmpDir { tmpDir => + val workspaceDir = tmpDir / "workspace" + val outDir = workspaceDir / "out" + MR.withMillDefaults(outPath = outDir, workspacePath = workspaceDir) { + + def check(path: os.Path, encContains: Seq[String], containsNot: Seq[String]) = { + val enc = MR.encodeKnownRootsInPath(path) + val dec = MR.decodeKnownRootsInPath(enc) + assert(path.toString == dec) + encContains.foreach(s => assert(enc.containsSlice(s))) + containsNot.foreach(s => assert(!enc.containsSlice(s))) + + path -> enc + } + + val file1 = tmpDir / "file1" + val file2 = workspaceDir / "file2" + val file3 = outDir / "file3" + + Seq( + "mapping" -> MR.get, + check(file1, Seq(file1.toString), Seq("$WORKSPACE", "$MILL_OUT")), + check(file2, Seq("$WORKSPACE/file2"), Seq("$MILL_OUT")), + check(file3, Seq("$MILL_OUT/file3"), Seq("$WORKSPACE")) + ) + } + } + } + } + + private def withTmpDir[T](body: os.Path => T): T = { + val tmpDir = os.Path(Files.createTempDirectory("")) + val res = body(tmpDir) + os.remove.all(tmpDir) + res + } + +} diff --git a/core/api/test/src/mill/api/PathRefTests.scala b/core/api/test/src/mill/api/PathRefTests.scala index dd8717a6949d..5c4f9790cd8c 100644 --- a/core/api/test/src/mill/api/PathRefTests.scala +++ b/core/api/test/src/mill/api/PathRefTests.scala @@ -19,6 +19,7 @@ object PathRefTests extends TestSuite { val sig2 = PathRef(file, quick).sig assert(sig1 != sig2) } + test("qref") - check(quick = true) test("ref") - check(quick = false) } @@ -43,12 +44,18 @@ object PathRefTests extends TestSuite { val file = tmpDir / "foo.txt" val content = "hello" os.write.over(file, content) - Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rw-rw----")) + Files.setPosixFilePermissions( + file.wrapped, + PosixFilePermissions.fromString("rw-rw----") + ) val rwSig = PathRef(file, quick).sig val rwSigb = PathRef(file, quick).sig assert(rwSig == rwSigb) - Files.setPosixFilePermissions(file.wrapped, PosixFilePermissions.fromString("rwxrw----")) + Files.setPosixFilePermissions( + file.wrapped, + PosixFilePermissions.fromString("rwxrw----") + ) val rwxSig = PathRef(file, quick).sig assert(rwSig != rwxSig) @@ -76,6 +83,7 @@ object PathRefTests extends TestSuite { val sig2 = PathRef(tmpDir, quick).sig assert(sig1 == sig2) } + test("qref") - check(quick = true) test("ref") - check(quick = false) } diff --git a/core/constants/src/mill/constants/PathVars.java b/core/constants/src/mill/constants/PathVars.java new file mode 100644 index 000000000000..574f5c61c5a1 --- /dev/null +++ b/core/constants/src/mill/constants/PathVars.java @@ -0,0 +1,22 @@ +package mill.constants; + +/** + * Central place containing all the path variables that Mill uses in PathRef or os.Path. + */ +public interface PathVars { + + /** + * Output directory where Mill workers' state and Mill tasks output should be + * written to + */ + String MILL_OUT = "MILL_OUT"; + + /** + * The Mill project workspace root directory. + */ + String WORKSPACE = "WORKSPACE"; + + String HOME = "HOME"; + + String ROOT = "ROOT"; +} diff --git a/core/exec/src/mill/exec/Execution.scala b/core/exec/src/mill/exec/Execution.scala index bdeae85d71b0..e4957024dc32 100644 --- a/core/exec/src/mill/exec/Execution.scala +++ b/core/exec/src/mill/exec/Execution.scala @@ -54,24 +54,24 @@ private[mill] case class Execution( offline: Boolean, enableTicker: Boolean ) = this( - baseLogger, - new JsonArrayLogger.Profile(os.Path(outPath) / millProfile), - os.Path(workspace), - os.Path(outPath), - os.Path(externalOutPath), - rootModule, - classLoaderSigHash, - classLoaderIdentityHash, - workerCache, - env, - failFast, - ec, - codeSignatures, - systemExit, - exclusiveSystemStreams, - getEvaluator, - offline, - enableTicker + baseLogger = baseLogger, + profileLogger = new JsonArrayLogger.Profile(os.Path(outPath) / millProfile), + workspace = os.Path(workspace), + outPath = os.Path(outPath), + externalOutPath = os.Path(externalOutPath), + rootModule = rootModule, + classLoaderSigHash = classLoaderSigHash, + classLoaderIdentityHash = classLoaderIdentityHash, + workerCache = workerCache, + env = env, + failFast = failFast, + ec = ec, + codeSignatures = codeSignatures, + systemExit = systemExit, + exclusiveSystemStreams = exclusiveSystemStreams, + getEvaluator = getEvaluator, + offline = offline, + enableTicker = enableTicker ) def withBaseLogger(newBaseLogger: Logger) = this.copy(baseLogger = newBaseLogger) diff --git a/core/exec/src/mill/exec/GroupExecution.scala b/core/exec/src/mill/exec/GroupExecution.scala index 021650710331..03644f734d22 100644 --- a/core/exec/src/mill/exec/GroupExecution.scala +++ b/core/exec/src/mill/exec/GroupExecution.scala @@ -83,7 +83,7 @@ trait GroupExecution { executionContext: mill.api.TaskCtx.Fork.Api, exclusive: Boolean, upstreamPathRefs: Seq[PathRef] - ): GroupExecution.Results = { + ): GroupExecution.Results = MappedRoots.withMillDefaults(outPath = outPath) { val inputsHash = { val externalInputsHash = MurmurHash3.orderedHash( @@ -242,8 +242,8 @@ trait GroupExecution { newEvaluated = newEvaluated.toSeq, cached = if (labelled.isInstanceOf[Task.Input[?]]) null else false, inputsHash = inputsHash, - previousInputsHash = cached.map(_._1).getOrElse(-1), - valueHashChanged = !cached.map(_._3).contains(valueHash), + previousInputsHash = cached.map(_.inputHash).getOrElse(-1), + valueHashChanged = !cached.map(_.valueHash).contains(valueHash), serializedPaths = serializedPaths ) } @@ -453,7 +453,7 @@ trait GroupExecution { inputsHash: Int, labelled: Task.Named[?], paths: ExecutionPaths - ): Option[(Int, Option[(Val, Seq[PathRef])], Int)] = { + ): Option[(inputHash: Int, valOpt: Option[(Val, Seq[PathRef])], valueHash: Int)] = { for { cached <- try Some(upickle.read[Cached](paths.meta.toIO, trace = false)) @@ -618,7 +618,7 @@ object GroupExecution { classLoader: ClassLoader )(t: => T): T = { // Tasks must be allowed to write to upstream worker's dest folders, because - // the point of workers is to manualy manage long-lived state which includes + // the point of workers is to manually manage long-lived state which includes // state on disk. val validWriteDests = deps.collect { case n: Task.Worker[?] => diff --git a/example/androidlib/java/1-hello-world/build.mill b/example/androidlib/java/1-hello-world/build.mill index ce26457d6b95..b4745fbad870 100644 --- a/example/androidlib/java/1-hello-world/build.mill +++ b/example/androidlib/java/1-hello-world/build.mill @@ -64,7 +64,7 @@ object app extends AndroidAppModule { /** Usage > ./mill show app.androidApk -".../out/app/androidApk.dest/app.apk" +"...$MILL_OUT/app/androidApk.dest/app.apk" */ diff --git a/example/androidlib/java/2-app-bundle/build.mill b/example/androidlib/java/2-app-bundle/build.mill index f05d58782d5d..007a41475265 100644 --- a/example/androidlib/java/2-app-bundle/build.mill +++ b/example/androidlib/java/2-app-bundle/build.mill @@ -37,7 +37,7 @@ object bundle extends AndroidAppBundle { /** Usage > ./mill show bundle.androidBundle -".../out/bundle/androidBundle.dest/signedBundle.aab" +"...$MILL_OUT/bundle/androidBundle.dest/signedBundle.aab" */ diff --git a/example/androidlib/java/4-sum-lib-java/build.mill b/example/androidlib/java/4-sum-lib-java/build.mill index 774ff1bdb271..1ab668d29318 100644 --- a/example/androidlib/java/4-sum-lib-java/build.mill +++ b/example/androidlib/java/4-sum-lib-java/build.mill @@ -129,7 +129,7 @@ Publish ... to /home/.../.m2/repository/... }, "payload": [ [ - ".../out/lib/androidAar.dest/library.aar", + "...$MILL_OUT/lib/androidAar.dest/library.aar", "lib-0.0.1.aar" ], ... diff --git a/example/androidlib/java/6-native-libs/build.mill b/example/androidlib/java/6-native-libs/build.mill index 669fee9f566a..1ddbfedf2271 100644 --- a/example/androidlib/java/6-native-libs/build.mill +++ b/example/androidlib/java/6-native-libs/build.mill @@ -66,7 +66,7 @@ object app extends AndroidNativeAppModule { // <1> /** Usage > ./mill show app.androidApk -".../out/app/androidApk.dest/app.apk" +"...$MILL_OUT/app/androidApk.dest/app.apk" > ./mill show app.createAndroidVirtualDevice ...Name: java-test, DeviceId: medium_phone... diff --git a/example/androidlib/kotlin/1-hello-kotlin/build.mill b/example/androidlib/kotlin/1-hello-kotlin/build.mill index b7b2c311a8a6..28a31b530ed8 100644 --- a/example/androidlib/kotlin/1-hello-kotlin/build.mill +++ b/example/androidlib/kotlin/1-hello-kotlin/build.mill @@ -72,7 +72,7 @@ object app extends AndroidAppKotlinModule { /** Usage > ./mill show app.androidApk -".../out/app/androidApk.dest/app.apk" +"...$MILL_OUT/app/androidApk.dest/app.apk" */ diff --git a/example/androidlib/kotlin/2-compose/build.mill b/example/androidlib/kotlin/2-compose/build.mill index 64fb8212e99e..4d3e9b9b8326 100644 --- a/example/androidlib/kotlin/2-compose/build.mill +++ b/example/androidlib/kotlin/2-compose/build.mill @@ -60,7 +60,7 @@ object app extends AndroidAppKotlinModule { /** Usage > ./mill show app.androidApk -".../out/app/androidApk.dest/app.apk" +"...$MILL_OUT/app/androidApk.dest/app.apk" > ./mill show app.createAndroidVirtualDevice diff --git a/example/androidlib/kotlin/4-sum-lib-kotlin/build.mill b/example/androidlib/kotlin/4-sum-lib-kotlin/build.mill index 013d5f7df22e..74e8f68f538e 100644 --- a/example/androidlib/kotlin/4-sum-lib-kotlin/build.mill +++ b/example/androidlib/kotlin/4-sum-lib-kotlin/build.mill @@ -131,7 +131,7 @@ Publish ... to /home/.../.m2/repository/... }, "payload": [ [ - ".../out/lib/androidAar.dest/library.aar", + "...$MILL_OUT/lib/androidAar.dest/library.aar", "lib-0.0.1.aar" ], ... diff --git a/example/cli/builtins/1-builtin-commands/build.mill b/example/cli/builtins/1-builtin-commands/build.mill index 4ac5ba3b98a4..e33ca895b722 100644 --- a/example/cli/builtins/1-builtin-commands/build.mill +++ b/example/cli/builtins/1-builtin-commands/build.mill @@ -146,8 +146,8 @@ Inputs: > ./mill show foo.compile { - "analysisFile": ".../out/foo/compile.dest/...", - "classes": ".../out/foo/compile.dest/classes" + "analysisFile": "...$MILL_OUT/foo/compile.dest/...", + "classes": "...$MILL_OUT/foo/compile.dest/classes" } */ diff --git a/example/depth/sandbox/2-test/build.mill b/example/depth/sandbox/2-test/build.mill index 70f409653e32..3288e086f056 100644 --- a/example/depth/sandbox/2-test/build.mill +++ b/example/depth/sandbox/2-test/build.mill @@ -47,8 +47,8 @@ object bar extends MyModule /** Usage > find . | grep generated.html -.../out/foo/test/testForked.dest/sandbox/generated.html -.../out/bar/test/testForked.dest/sandbox/generated.html +./out/foo/test/testForked.dest/sandbox/generated.html +./out/bar/test/testForked.dest/sandbox/generated.html > cat out/foo/test/testForked.dest/sandbox/generated.html

hello

@@ -81,7 +81,7 @@ object qux extends JavaModule { > find . | grep .html ... -.../out/qux/test/testForked.dest/sandbox/foo.html +./out/qux/test/testForked.dest/sandbox/foo.html > cat out/qux/test/testForked.dest/sandbox/foo.html

foo

diff --git a/example/extending/imports/1-mvn-deps/build.mill b/example/extending/imports/1-mvn-deps/build.mill index fda9104f287d..b47f2011ff0c 100644 --- a/example/extending/imports/1-mvn-deps/build.mill +++ b/example/extending/imports/1-mvn-deps/build.mill @@ -44,7 +44,7 @@ compiling 1 Java source... generated snippet.txt resource:

hello

world

> ./mill show foo.assembly -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" > ./out/foo/assembly.dest/out.jar # mac/linux generated snippet.txt resource:

hello

world

diff --git a/example/extending/imports/2-mvn-deps-scala/build.mill b/example/extending/imports/2-mvn-deps-scala/build.mill index da924f68e671..96352e65d657 100644 --- a/example/extending/imports/2-mvn-deps-scala/build.mill +++ b/example/extending/imports/2-mvn-deps-scala/build.mill @@ -34,7 +34,7 @@ compiling 1 Scala source... generated snippet.txt resource:

hello

world

> ./mill show bar.assembly -".../out/bar/assembly.dest/out.jar" +"...$MILL_OUT/bar/assembly.dest/out.jar" > ./out/bar/assembly.dest/out.jar # mac/linux generated snippet.txt resource:

hello

world

diff --git a/example/extending/metabuild/4-meta-build/build.mill b/example/extending/metabuild/4-meta-build/build.mill index b8b0a56cfd43..4beaa6e39c7e 100644 --- a/example/extending/metabuild/4-meta-build/build.mill +++ b/example/extending/metabuild/4-meta-build/build.mill @@ -48,7 +48,7 @@ Build-time HTML snippet:

hello

Run-time HTML snippet:

world

> ./mill show assembly -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" > ./out/assembly.dest/out.jar # mac/linux Build-time HTML snippet:

hello

diff --git a/example/extending/python/4-python-libs-bundle/build.mill b/example/extending/python/4-python-libs-bundle/build.mill index b1da6f0571a3..cfb86a1cf5b1 100644 --- a/example/extending/python/4-python-libs-bundle/build.mill +++ b/example/extending/python/4-python-libs-bundle/build.mill @@ -109,7 +109,7 @@ object qux extends PythonModule { Numpy : Sum: 150 | Pandas: Mean: 30.0, Max: 50 > ./mill show qux.bundle -".../out/qux/bundle.dest/bundle.pex" +"...$MILL_OUT/qux/bundle.dest/bundle.pex" > out/qux/bundle.dest/bundle.pex # running the PEX binary outside of Mill Numpy : Sum: 150 | Pandas: Mean: 30.0, Max: 50 diff --git a/example/extending/typescript/4-npm-deps-bundle/build.mill b/example/extending/typescript/4-npm-deps-bundle/build.mill index 006a4749c8a0..d44808e68ffb 100644 --- a/example/extending/typescript/4-npm-deps-bundle/build.mill +++ b/example/extending/typescript/4-npm-deps-bundle/build.mill @@ -117,7 +117,7 @@ object qux extends TypeScriptModule { Hello James Bond Professor > ./mill show qux.bundle -".../out/qux/bundle.dest/bundle.js" +"...$MILL_OUT/qux/bundle.dest/bundle.js" > node out/qux/bundle.dest/bundle.js James Bond prof Hello James Bond Professor diff --git a/example/fundamentals/cross/10-static-blog/build.mill b/example/fundamentals/cross/10-static-blog/build.mill index 0cb186843ed6..77d53d2886cd 100644 --- a/example/fundamentals/cross/10-static-blog/build.mill +++ b/example/fundamentals/cross/10-static-blog/build.mill @@ -113,7 +113,7 @@ def dist = Task { /** Usage > ./mill show "post[1-My-First-Post.md].render" -".../out/post/1-My-First-Post.md/render.dest/1-my-first-post.html" +"...$MILL_OUT/post/1-My-First-Post.md/render.dest/1-my-first-post.html" > cat out/post/1-My-First-Post.md/render.dest/1-my-first-post.html ... diff --git a/example/fundamentals/libraries/2-upickle/build.mill b/example/fundamentals/libraries/2-upickle/build.mill index fcac91f6de98..1a0dc0aa6a92 100644 --- a/example/fundamentals/libraries/2-upickle/build.mill +++ b/example/fundamentals/libraries/2-upickle/build.mill @@ -74,7 +74,7 @@ def taskPath = Task { /** Usage > ./mill show taskPath -".../out/taskPath.dest/file.txt" +"...$MILL_OUT/taskPath.dest/file.txt" */ @@ -92,7 +92,7 @@ def taskPathRef = Task { /** Usage > ./mill show taskPathRef -"ref.../out/taskPathRef.dest/file.txt" +"ref...$MILL_OUT/taskPathRef.dest/file.txt" */ diff --git a/example/fundamentals/modules/8-diy-java-modules/build.mill b/example/fundamentals/modules/8-diy-java-modules/build.mill index 4b6a508fd561..524bf846a5c4 100644 --- a/example/fundamentals/modules/8-diy-java-modules/build.mill +++ b/example/fundamentals/modules/8-diy-java-modules/build.mill @@ -150,7 +150,7 @@ object qux extends DiyJavaModule { } > ./mill show qux.assembly -".../out/qux/assembly.dest/assembly.jar" +"...$MILL_OUT/qux/assembly.dest/assembly.jar" > java -jar out/qux/assembly.dest/assembly.jar Foo.value: 31337 @@ -158,7 +158,7 @@ Bar.value: 271828 Qux.value: 9000 > ./mill show foo.assembly -".../out/foo/assembly.dest/assembly.jar" +"...$MILL_OUT/foo/assembly.dest/assembly.jar" > java -jar out/foo/assembly.dest/assembly.jar Foo.value: 31337 diff --git a/example/fundamentals/tasks/1-task-graph/build.mill b/example/fundamentals/tasks/1-task-graph/build.mill index a512437f7652..1c855464b708 100644 --- a/example/fundamentals/tasks/1-task-graph/build.mill +++ b/example/fundamentals/tasks/1-task-graph/build.mill @@ -55,7 +55,7 @@ def run(args: String*) = Task.Command { /** Usage > ./mill show assembly -".../out/assembly.dest/assembly.jar" +"...$MILL_OUT/assembly.dest/assembly.jar" > java -jar out/assembly.dest/assembly.jar i am cow Foo.value: 31337 diff --git a/example/fundamentals/tasks/2-primary-tasks/build.mill b/example/fundamentals/tasks/2-primary-tasks/build.mill index 570dd57b59ce..115a64d1657f 100644 --- a/example/fundamentals/tasks/2-primary-tasks/build.mill +++ b/example/fundamentals/tasks/2-primary-tasks/build.mill @@ -167,7 +167,7 @@ Generating classfiles Generating jar > ./mill show jar -".../out/jar.dest/foo.jar" +"...$MILL_OUT/jar.dest/foo.jar" */ diff --git a/example/javalib/basic/1-script/build.mill b/example/javalib/basic/1-script/build.mill index 9f7c5dadb44d..9ca3528a3096 100644 --- a/example/javalib/basic/1-script/build.mill +++ b/example/javalib/basic/1-script/build.mill @@ -18,7 +18,7 @@ compiling 1 Java source to... /** Usage > ./mill show Foo.java:assembly # show the output of the assembly task -".../out/Foo.java/assembly.dest/out.jar" +"...$MILL_OUT/Foo.java/assembly.dest/out.jar" > java -jar ./out/Foo.java/assembly.dest/out.jar --text hello

hello

diff --git a/example/javalib/basic/10-realistic/build.mill b/example/javalib/basic/10-realistic/build.mill index f23038a5b082..5a10ccda2820 100644 --- a/example/javalib/basic/10-realistic/build.mill +++ b/example/javalib/basic/10-realistic/build.mill @@ -101,7 +101,7 @@ Publishing Artifact(com.lihaoyi,qux,0.0.1) to ivy repo... ... > ./mill show foo.assembly # mac/linux -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" > ./out/foo/assembly.dest/out.jar # mac/linux foo version 0.0.1 diff --git a/example/javalib/publishing/5-jlink/build.mill b/example/javalib/publishing/5-jlink/build.mill index d9031d13d04f..f44598072418 100644 --- a/example/javalib/publishing/5-jlink/build.mill +++ b/example/javalib/publishing/5-jlink/build.mill @@ -47,7 +47,7 @@ object foo extends JavaModule, JlinkModule { > ./mill foo.jlinkAppImage > ./mill show foo.jlinkAppImage -".../out/foo/jlinkAppImage.dest/jlink-runtime" +"...$MILL_OUT/foo/jlinkAppImage.dest/jlink-runtime" > ./out/foo/jlinkAppImage.dest/jlink-runtime/bin/jlink ... foo.Bar main diff --git a/example/javalib/publishing/6-jpackage/build.mill b/example/javalib/publishing/6-jpackage/build.mill index 150c9f8be48a..db1d7535fb3a 100644 --- a/example/javalib/publishing/6-jpackage/build.mill +++ b/example/javalib/publishing/6-jpackage/build.mill @@ -50,7 +50,7 @@ object foo extends JavaModule, JpackageModule { > ./mill foo.jpackageAppImage > ./mill show foo.jpackageAppImage -".../out/foo/jpackageAppImage.dest/image" +"...$MILL_OUT/foo/jpackageAppImage.dest/image" */ // diff --git a/example/javalib/publishing/9-repackage-config/build.mill b/example/javalib/publishing/9-repackage-config/build.mill index 70d599571def..e234033c7ede 100644 --- a/example/javalib/publishing/9-repackage-config/build.mill +++ b/example/javalib/publishing/9-repackage-config/build.mill @@ -44,7 +44,7 @@ Bar.value:

world

Qux.value: 31337 > ./mill show foo.repackagedJar -".../out/foo/repackagedJar.dest/out.jar" +"...$MILL_OUT/foo/repackagedJar.dest/out.jar" > ./out/foo/repackagedJar.dest/out.jar Foo.value:

hello

diff --git a/example/kotlinlib/basic/1-script/build.mill b/example/kotlinlib/basic/1-script/build.mill index 679cd5d33ce1..b76ba7996a1b 100644 --- a/example/kotlinlib/basic/1-script/build.mill +++ b/example/kotlinlib/basic/1-script/build.mill @@ -18,7 +18,7 @@ Compiling 1 Kotlin sources to... /** Usage > ./mill show Foo.kt:assembly # show the output of the assembly task -".../out/Foo.kt/assembly.dest/out.jar" +"...$MILL_OUT/Foo.kt/assembly.dest/out.jar" > java -jar ./out/Foo.kt/assembly.dest/out.jar --text hello

hello

diff --git a/example/kotlinlib/basic/10-realistic/build.mill b/example/kotlinlib/basic/10-realistic/build.mill index 7cc8bd8d94a9..92110ad9afb5 100644 --- a/example/kotlinlib/basic/10-realistic/build.mill +++ b/example/kotlinlib/basic/10-realistic/build.mill @@ -109,7 +109,7 @@ Publishing Artifact(com.lihaoyi,qux,0.0.1) to ivy repo... ... > ./mill show foo.assembly # mac/linux -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" > ./out/foo/assembly.dest/out.jar # mac/linux foo version 0.0.1 diff --git a/example/kotlinlib/linting/4-kover/build.mill b/example/kotlinlib/linting/4-kover/build.mill index 37a393ff13cd..493e5786ded1 100644 --- a/example/kotlinlib/linting/4-kover/build.mill +++ b/example/kotlinlib/linting/4-kover/build.mill @@ -57,7 +57,7 @@ kover.xmlReport > ./mill show kover.htmlReport ... -...out/kover/htmlReport.dest/kover-report... +...$MILL_OUT/kover/htmlReport.dest/kover-report... > cat out/kover/htmlReport.dest/kover-report/index.html ... @@ -65,7 +65,7 @@ kover.xmlReport > ./mill show mill.kotlinlib.kover/htmlReportAll # collect reports from all modules ... -...out/mill.kotlinlib.kover.Kover/htmlReportAll.dest/kover-report... +...$MILL_OUT/mill.kotlinlib.kover.Kover/htmlReportAll.dest/kover-report... > cat out/mill.kotlinlib.kover.Kover/htmlReportAll.dest/kover-report/index.html ... diff --git a/example/kotlinlib/module/10-dependency-injection/build.mill b/example/kotlinlib/module/10-dependency-injection/build.mill index ed47d4eafa03..febeef53bc85 100644 --- a/example/kotlinlib/module/10-dependency-injection/build.mill +++ b/example/kotlinlib/module/10-dependency-injection/build.mill @@ -47,8 +47,8 @@ object dagger extends KspModule { > ./mill show dagger.generatedSources [ - "ref:v0:.../out/dagger/generatedSourcesWithKsp2.dest/generated/java", - "ref:v0:.../out/dagger/generatedSourcesWithKsp2.dest/generated/kotlin" + "ref:v0:...$MILL_OUT/dagger/generatedSourcesWithKsp2.dest/generated/java", + "ref:v0:...$MILL_OUT/dagger/generatedSourcesWithKsp2.dest/generated/kotlin" ] > ls out/dagger/generatedSourcesWithKsp2.dest/generated/java/com/example/dagger/ diff --git a/example/pythonlib/basic/1-simple/build.mill b/example/pythonlib/basic/1-simple/build.mill index 21ee94f61468..0982f38afd7a 100644 --- a/example/pythonlib/basic/1-simple/build.mill +++ b/example/pythonlib/basic/1-simple/build.mill @@ -87,7 +87,7 @@ OK ... > ./mill show foo.bundle # Creates Bundle for the python file -".../out/foo/bundle.dest/bundle.pex" +"...$MILL_OUT/foo/bundle.dest/bundle.pex" > out/foo/bundle.dest/bundle.pex --text "Hello Mill" # running the PEX binary outside of Mill

Hello Mill

diff --git a/example/pythonlib/module/1-common-config/build.mill b/example/pythonlib/module/1-common-config/build.mill index a399d898f067..95bf44f45294 100644 --- a/example/pythonlib/module/1-common-config/build.mill +++ b/example/pythonlib/module/1-common-config/build.mill @@ -75,7 +75,7 @@ MY_CUSTOM_ENV: my-env-value ... > ./mill show foo.bundle -".../out/foo/bundle.dest/bundle.pex" +"...$MILL_OUT/foo/bundle.dest/bundle.pex" > out/foo/bundle.dest/bundle.pex ... diff --git a/example/pythonlib/module/6-pex-config/build.mill b/example/pythonlib/module/6-pex-config/build.mill index ef6454532025..f27112c56e7a 100644 --- a/example/pythonlib/module/6-pex-config/build.mill +++ b/example/pythonlib/module/6-pex-config/build.mill @@ -44,7 +44,7 @@ object foo extends PythonModule { /** Usage > ./mill show foo.bundle -".../out/foo/bundle.dest/bundle.pex" +"...$MILL_OUT/foo/bundle.dest/bundle.pex" > out/foo/bundle.dest/bundle.pex ... diff --git a/example/pythonlib/publishing/1-publish-module/build.mill b/example/pythonlib/publishing/1-publish-module/build.mill index 75f91333158e..b6df9a67e92b 100644 --- a/example/pythonlib/publishing/1-publish-module/build.mill +++ b/example/pythonlib/publishing/1-publish-module/build.mill @@ -44,10 +44,10 @@ object `package` extends PythonModule, PublishModule { /** Usage > ./mill show sdist -".../out/sdist.dest/dist/testpkg_mill-0.0.2.tar.gz" +"...$MILL_OUT/sdist.dest/dist/testpkg_mill-0.0.2.tar.gz" > ./mill show wheel -".../out/wheel.dest/dist/testpkg_mill-0.0.2-py3-none-any.whl" +"...$MILL_OUT/wheel.dest/dist/testpkg_mill-0.0.2-py3-none-any.whl" */ // These files can then be `pip-installed` by other projects, or, if you're using Mill, you can diff --git a/example/scalalib/basic/1-script/build.mill b/example/scalalib/basic/1-script/build.mill index f582a2cf5af9..2bb637e95d35 100644 --- a/example/scalalib/basic/1-script/build.mill +++ b/example/scalalib/basic/1-script/build.mill @@ -66,7 +66,7 @@ Jvm Version: 11.0.28 /** Usage > ./mill show Foo.scala:assembly # show the output of the assembly task -".../out/Foo.scala/assembly.dest/out.jar" +"...$MILL_OUT/Foo.scala/assembly.dest/out.jar" > java -jar ./out/Foo.scala/assembly.dest/out.jar --text hello

hello

diff --git a/example/scalalib/basic/10-realistic/build.mill b/example/scalalib/basic/10-realistic/build.mill index b9ea519a6e6e..2ec7435d5e2f 100644 --- a/example/scalalib/basic/10-realistic/build.mill +++ b/example/scalalib/basic/10-realistic/build.mill @@ -126,7 +126,7 @@ Publishing Artifact(com.lihaoyi,bar_3,0.0.1) to ivy repo... Publishing Artifact(com.lihaoyi,qux,0.0.1) to ivy repo... > ./mill show foo[2.13.16].assembly # mac/linux -".../out/foo/2.13.16/assembly.dest/out.jar" +"...$MILL_OUT/foo/2.13.16/assembly.dest/out.jar" > ./out/foo/2.13.16/assembly.dest/out.jar # mac/linux foo version 0.0.1 diff --git a/example/scalalib/basic/3-simple/build.mill b/example/scalalib/basic/3-simple/build.mill index 1138995a9734..976dd2c6bab3 100644 --- a/example/scalalib/basic/3-simple/build.mill +++ b/example/scalalib/basic/3-simple/build.mill @@ -101,7 +101,7 @@ compiling 1 Scala source to... > ./mill assembly # bundle classfiles and libraries into a jar for deployment > ./mill show assembly # show the output of the assembly task -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" > java -jar ./out/assembly.dest/out.jar --text hello

hello

diff --git a/example/scalalib/basic/6-programmatic/build.mill b/example/scalalib/basic/6-programmatic/build.mill index 8a2bcb597a9c..85824ede4b1c 100644 --- a/example/scalalib/basic/6-programmatic/build.mill +++ b/example/scalalib/basic/6-programmatic/build.mill @@ -83,7 +83,7 @@ foo.run > ./mill foo.assembly # bundle classfiles and libraries into a jar for deployment > ./mill show foo.assembly # show the output of the assembly task -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" > java -jar ./out/foo/assembly.dest/out.jar --text hello

hello

diff --git a/example/scalalib/config/1-common-config/build.mill b/example/scalalib/config/1-common-config/build.mill index 0a8cee6fc8f7..40240b0f8319 100644 --- a/example/scalalib/config/1-common-config/build.mill +++ b/example/scalalib/config/1-common-config/build.mill @@ -14,7 +14,7 @@ my.custom.property: my-prop-value MY_CUSTOM_ENV: my-env-value > ./mill show assembly -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" > ./out/assembly.dest/out.jar # mac/linux Foo2.value:

hello2

diff --git a/example/scalalib/module/15-unidoc/build.mill b/example/scalalib/module/15-unidoc/build.mill index 94f1f27fb1d2..38d594591792 100644 --- a/example/scalalib/module/15-unidoc/build.mill +++ b/example/scalalib/module/15-unidoc/build.mill @@ -34,7 +34,7 @@ object foo extends ScalaModule, UnidocModule { /** Usage > ./mill show foo.unidocLocal -".../out/foo/unidocLocal.dest" +"...$MILL_OUT/foo/unidocLocal.dest" > cat out/foo/unidocLocal.dest/foo/Foo.html ... diff --git a/example/scalalib/module/2-common-config/build.mill b/example/scalalib/module/2-common-config/build.mill index 6103b6ff4cea..30d516f56333 100644 --- a/example/scalalib/module/2-common-config/build.mill +++ b/example/scalalib/module/2-common-config/build.mill @@ -105,7 +105,7 @@ my.custom.property: my-prop-value MY_CUSTOM_ENV: my-env-value > ./mill show assembly -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" > ./out/assembly.dest/out.jar # mac/linux Foo2.value:

hello2

diff --git a/example/scalalib/native/1-simple/build.mill b/example/scalalib/native/1-simple/build.mill index f6a43a8451e8..0566d3473c52 100644 --- a/example/scalalib/native/1-simple/build.mill +++ b/example/scalalib/native/1-simple/build.mill @@ -26,7 +26,7 @@ object `package` extends ScalaNativeModule {

hello

> ./mill show nativeLink # Build and link native binary -".../out/nativeLink.dest/out" +"...$MILL_OUT/nativeLink.dest/out" > ./out/nativeLink.dest/out --text hello # Run the executable

hello

diff --git a/example/scalalib/spark/3-semi-realistic/build.mill b/example/scalalib/spark/3-semi-realistic/build.mill index 73668b29a008..66d4c8678555 100644 --- a/example/scalalib/spark/3-semi-realistic/build.mill +++ b/example/scalalib/spark/3-semi-realistic/build.mill @@ -46,7 +46,7 @@ Summary Statistics by Category: > chmod +x spark-submit.sh > ./mill show assembly # prepare for spark-submit -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" > ./spark-submit.sh out/assembly.dest/out.jar foo.Foo resources/transactions.csv ... diff --git a/example/scalalib/web/4-scalajs-module/build.mill b/example/scalalib/web/4-scalajs-module/build.mill index b9eae71f61c6..6d56f90ae0d4 100644 --- a/example/scalalib/web/4-scalajs-module/build.mill +++ b/example/scalalib/web/4-scalajs-module/build.mill @@ -41,7 +41,7 @@ stringifiedJsObject: ["hello","world","!"] { ... ..."jsFileName": "main.js", - "dest": ".../out/foo/fullLinkJS.dest" + "dest": "...$MILL_OUT/foo/fullLinkJS.dest" } > node out/foo/fullLinkJS.dest/main.js # mac/linux diff --git a/example/scalalib/web/9-wasm/build.mill b/example/scalalib/web/9-wasm/build.mill index f85403e81e10..a8443d6e1643 100644 --- a/example/scalalib/web/9-wasm/build.mill +++ b/example/scalalib/web/9-wasm/build.mill @@ -27,7 +27,7 @@ object wasm extends ScalaJSModule { ... ..."jsFileName": "main.js", ... - "dest": ".../out/wasm/fastLinkJS.dest" + "dest": "...$MILL_OUT/wasm/fastLinkJS.dest" } > node --experimental-wasm-exnref out/wasm/fastLinkJS.dest/main.js # mac/linux diff --git a/example/thirdparty/android-endless-tunnel/build.mill b/example/thirdparty/android-endless-tunnel/build.mill index 7bbec86eb8fc..081b3aefb782 100644 --- a/example/thirdparty/android-endless-tunnel/build.mill +++ b/example/thirdparty/android-endless-tunnel/build.mill @@ -53,7 +53,7 @@ object `endless-tunnel` extends mill.api.Module { /** Usage > ./mill show endless-tunnel.app.androidApk -".../out/endless-tunnel/app/androidApk.dest/app.apk" +"...$MILL_OUT/endless-tunnel/app/androidApk.dest/app.apk" > ./mill show endless-tunnel.app.createAndroidVirtualDevice ...Name: cpp-test, DeviceId: medium_phone... diff --git a/integration/ide/build-classpath-contents/src/BuildClasspathContentsTests.scala b/integration/ide/build-classpath-contents/src/BuildClasspathContentsTests.scala index fad8eaa2ccec..14c7e1484118 100644 --- a/integration/ide/build-classpath-contents/src/BuildClasspathContentsTests.scala +++ b/integration/ide/build-classpath-contents/src/BuildClasspathContentsTests.scala @@ -1,4 +1,4 @@ -import mill.api.BuildCtx +import mill.api.{BuildCtx, MappedRoots, PathRef} import mill.testkit.UtestIntegrationTestSuite import utest.* @@ -6,56 +6,61 @@ object BuildClasspathContentsTests extends UtestIntegrationTestSuite { val tests: Tests = Tests { test("test") - integrationTest { tester => - val result1 = - tester.eval(("--meta-level", "1", "show", "compileClasspath"), stderr = os.Inherit) - val deserialized = upickle.read[Seq[mill.api.PathRef]](result1.out) - val millPublishedJars = deserialized - .map(_.path.last) - .filter(_.startsWith("mill-")) - .sorted - val millLocalClasspath = deserialized - .map(_.path) - .filter(_.startsWith(BuildCtx.workspaceRoot)) - .map(_.subRelativeTo(BuildCtx.workspaceRoot)) - .filter(!_.startsWith("out/integration")) - .filter(!_.startsWith("out/dist/localRepo.dest")) - .map(_.toString) - .sorted - if (sys.env("MILL_INTEGRATION_IS_PACKAGED_LAUNCHER") == "true") { - assertGoldenLiteral( - millPublishedJars, - List( - "mill-core-api-daemon_3-SNAPSHOT.jar", - "mill-core-api_3-SNAPSHOT.jar", - "mill-core-constants-SNAPSHOT.jar", - "mill-libs-androidlib-databinding_3-SNAPSHOT.jar", - "mill-libs-androidlib_3-SNAPSHOT.jar", - "mill-libs-daemon-client-SNAPSHOT.jar", - "mill-libs-daemon-server_3-SNAPSHOT.jar", - "mill-libs-javalib-api_3-SNAPSHOT.jar", - "mill-libs-javalib-testrunner-entrypoint-SNAPSHOT.jar", - "mill-libs-javalib-testrunner_3-SNAPSHOT.jar", - "mill-libs-javalib_3-SNAPSHOT.jar", - "mill-libs-javascriptlib_3-SNAPSHOT.jar", - "mill-libs-kotlinlib-api_3-SNAPSHOT.jar", - "mill-libs-kotlinlib-ksp2-api_3-SNAPSHOT.jar", - "mill-libs-kotlinlib_3-SNAPSHOT.jar", - "mill-libs-pythonlib_3-SNAPSHOT.jar", - "mill-libs-rpc_3-SNAPSHOT.jar", - "mill-libs-scalajslib-api_3-SNAPSHOT.jar", - "mill-libs-scalajslib_3-SNAPSHOT.jar", - "mill-libs-scalalib_3-SNAPSHOT.jar", - "mill-libs-scalanativelib-api_3-SNAPSHOT.jar", - "mill-libs-scalanativelib_3-SNAPSHOT.jar", - "mill-libs-script_3-SNAPSHOT.jar", - "mill-libs-util_3-SNAPSHOT.jar", - "mill-libs_3-SNAPSHOT.jar", - "mill-moduledefs_3-0.11.10.jar" + MappedRoots.withMapping(Seq( + "HOME" -> os.home, + "WORKSPACE" -> tester.workspacePath + )) { + val result1 = + tester.eval(("--meta-level", "1", "show", "compileClasspath"), stderr = os.Inherit) + val deserialized = upickle.read[Seq[mill.api.PathRef]](result1.out) + val millPublishedJars = deserialized + .map(_.path.last) + .filter(_.startsWith("mill-")) + .sorted + val millLocalClasspath = deserialized + .map(_.path) + .filter(_.startsWith(BuildCtx.workspaceRoot)) + .map(_.subRelativeTo(BuildCtx.workspaceRoot)) + .filter(!_.startsWith("out/integration")) + .filter(!_.startsWith("out/dist/localRepo.dest")) + .map(_.toString) + .sorted + if (sys.env("MILL_INTEGRATION_IS_PACKAGED_LAUNCHER") == "true") { + assertGoldenLiteral( + millPublishedJars, + List( + "mill-core-api-daemon_3-SNAPSHOT.jar", + "mill-core-api_3-SNAPSHOT.jar", + "mill-core-constants-SNAPSHOT.jar", + "mill-libs-androidlib-databinding_3-SNAPSHOT.jar", + "mill-libs-androidlib_3-SNAPSHOT.jar", + "mill-libs-daemon-client-SNAPSHOT.jar", + "mill-libs-daemon-server_3-SNAPSHOT.jar", + "mill-libs-javalib-api_3-SNAPSHOT.jar", + "mill-libs-javalib-testrunner-entrypoint-SNAPSHOT.jar", + "mill-libs-javalib-testrunner_3-SNAPSHOT.jar", + "mill-libs-javalib_3-SNAPSHOT.jar", + "mill-libs-javascriptlib_3-SNAPSHOT.jar", + "mill-libs-kotlinlib-api_3-SNAPSHOT.jar", + "mill-libs-kotlinlib-ksp2-api_3-SNAPSHOT.jar", + "mill-libs-kotlinlib_3-SNAPSHOT.jar", + "mill-libs-pythonlib_3-SNAPSHOT.jar", + "mill-libs-rpc_3-SNAPSHOT.jar", + "mill-libs-scalajslib-api_3-SNAPSHOT.jar", + "mill-libs-scalajslib_3-SNAPSHOT.jar", + "mill-libs-scalalib_3-SNAPSHOT.jar", + "mill-libs-scalanativelib-api_3-SNAPSHOT.jar", + "mill-libs-scalanativelib_3-SNAPSHOT.jar", + "mill-libs-script_3-SNAPSHOT.jar", + "mill-libs-util_3-SNAPSHOT.jar", + "mill-libs_3-SNAPSHOT.jar", + "mill-moduledefs_3-0.11.10.jar" + ) ) - ) - assert(millLocalClasspath == Nil) - } else { - sys.error("This test must be run in `packaged` mode, not `local`") + assert(millLocalClasspath == Nil) + } else { + sys.error("This test must be run in `packaged` mode, not `local`") + } } } } diff --git a/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala b/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala index 8fabf73fab97..4625dba1a362 100644 --- a/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala +++ b/integration/invalidation/multi-level-editing/src/MultiLevelBuildTests.scala @@ -1,5 +1,6 @@ package mill.integration +import mill.api.MappedRoots import mill.testkit.{IntegrationTester, UtestIntegrationTestSuite} import mill.constants.OutFiles.* import mill.daemon.RunnerState @@ -48,7 +49,13 @@ trait MultiLevelBuildTests extends UtestIntegrationTestSuite { yield { val path = tester.workspacePath / "out" / Seq.fill(depth)(millBuild) / millRunnerState - if (os.exists(path)) upickle.read[RunnerState.Frame.Logged](os.read(path)) -> path + if (os.exists(path)) + MappedRoots.withMillDefaults( + outPath = tester.workspacePath / "out", + workspacePath = tester.workspacePath + ) { + upickle.read[RunnerState.Frame.Logged](os.read(path)) -> path + } else RunnerState.Frame.Logged(Map(), Seq(), Seq(), None, Seq(), 0) -> path } } diff --git a/libs/javalib/src/mill/javalib/TestModule.scala b/libs/javalib/src/mill/javalib/TestModule.scala index eebdd5e4c6ab..7c03f4a132dd 100644 --- a/libs/javalib/src/mill/javalib/TestModule.scala +++ b/libs/javalib/src/mill/javalib/TestModule.scala @@ -1,14 +1,10 @@ package mill.javalib import mill.T -import mill.api.Result +import mill.api.{DefaultTaskModule, MappedRoots, PathRef, Result, Task, TaskCtx} import mill.api.daemon.internal.TestModuleApi import mill.api.daemon.internal.TestReporter import mill.api.daemon.internal.bsp.{BspBuildTarget, BspModuleApi} -import mill.api.PathRef -import mill.api.Task -import mill.api.TaskCtx -import mill.api.DefaultTaskModule import mill.javalib.bsp.BspModule import mill.util.Jvm import mill.api.JsonFormatters.given @@ -212,7 +208,10 @@ trait TestModule ) val argsFile = Task.dest / "testargs" - os.write(argsFile, upickle.write(testArgs)) + MappedRoots.withMapping(Seq()) { + // Don't use placeholders, so we only have local absolute paths + os.write(argsFile, upickle.write(testArgs)) + } val testRunnerClasspathArg = jvmWorker().scalalibClasspath() diff --git a/libs/javalib/src/mill/javalib/TestModuleUtil.scala b/libs/javalib/src/mill/javalib/TestModuleUtil.scala index 553312dbd871..39f22f4aee9a 100644 --- a/libs/javalib/src/mill/javalib/TestModuleUtil.scala +++ b/libs/javalib/src/mill/javalib/TestModuleUtil.scala @@ -1,7 +1,6 @@ package mill.javalib -import mill.api.{PathRef, TaskCtx} -import mill.api.Result +import mill.api.{BuildCtx, Logger, MappedRoots, PathRef, Result, TaskCtx} import mill.api.daemon.internal.TestReporter import mill.util.Jvm import mill.api.internal.Util @@ -13,10 +12,7 @@ import java.time.temporal.ChronoUnit import java.time.{Instant, LocalDateTime, ZoneId} import scala.xml.Elem import scala.collection.mutable -import mill.api.Logger - import java.util.concurrent.ConcurrentHashMap -import mill.api.BuildCtx import mill.javalib.testrunner.{GetTestTasksMain, TestArgs, TestResult, TestRunnerUtils} import os.Path @@ -152,7 +148,10 @@ final class TestModuleUtil( val argsFile = baseFolder / "testargs" val sandbox = baseFolder / "sandbox" - os.write(argsFile, upickle.write(testArgs), createFolders = true) + MappedRoots.withMapping(Seq()) { + // Don't use placeholders, so we only have local absolute paths + os.write(argsFile, upickle.write(testArgs), createFolders = true) + } os.makeDir.all(sandbox) diff --git a/libs/javalib/test/src/mill/javalib/HelloJavaTests.scala b/libs/javalib/test/src/mill/javalib/HelloJavaTests.scala index 58224f0532dc..80d8070ad779 100644 --- a/libs/javalib/test/src/mill/javalib/HelloJavaTests.scala +++ b/libs/javalib/test/src/mill/javalib/HelloJavaTests.scala @@ -40,9 +40,9 @@ object HelloJavaTests extends TestSuite { assert( result1.value == result2.value, + result1.evalCount != 0, result2.evalCount == 0, result3.evalCount != 0, - result3.evalCount != 0, os.walk(result1.value.classes.path).exists(_.last == "Core.class"), !os.walk(result1.value.classes.path).exists(_.last == "Main.class"), os.walk(result3.value.classes.path).exists(_.last == "Main.class"), diff --git a/libs/javalib/testrunner/entrypoint/src/mill/javalib/testrunner/entrypoint/TestRunnerMain.java b/libs/javalib/testrunner/entrypoint/src/mill/javalib/testrunner/entrypoint/TestRunnerMain.java index 5930e98e6198..18e8d21283ec 100644 --- a/libs/javalib/testrunner/entrypoint/src/mill/javalib/testrunner/entrypoint/TestRunnerMain.java +++ b/libs/javalib/testrunner/entrypoint/src/mill/javalib/testrunner/entrypoint/TestRunnerMain.java @@ -14,6 +14,10 @@ * nested classloaders. */ public class TestRunnerMain { + /** + * + * @param args arg1: classpath, arg2 testArgs-file + */ public static void main(String[] args) throws Exception { URL[] testRunnerClasspath = Stream.of(args[0].split(",")) .map(s -> { diff --git a/libs/javalib/worker/src/mill/javalib/worker/JvmWorkerImpl.scala b/libs/javalib/worker/src/mill/javalib/worker/JvmWorkerImpl.scala index 2cd6e126d0b9..4c81a9f98819 100644 --- a/libs/javalib/worker/src/mill/javalib/worker/JvmWorkerImpl.scala +++ b/libs/javalib/worker/src/mill/javalib/worker/JvmWorkerImpl.scala @@ -346,13 +346,25 @@ class JvmWorkerImpl(args: JvmWorkerArgs) extends JvmWorkerApi with AutoCloseable (in, out) => { val serverToClient = use(BufferedReader(InputStreamReader(in))) val clientToServer = use(PrintStream(out)) - val wireTransport = - MillRpcWireTransport.ViaStreams( - debugName, - serverToClient, - clientToServer, - writeSynchronizer = clientToServer - ) + + class Transport() + extends MillRpcWireTransport.ViaStreams( + debugName, + serverToClient, + clientToServer, + writeSynchronizer = clientToServer + ) { + override def writeSerialized[A: upickle.Writer]( + message: A, + log: String => Unit + ): Unit = { + // RPC communication is local and uncached, so we don't want to use any root mapping + MappedRoots.withMapping(Seq()) { + super.writeSerialized(message, log) + } + } + } + val wireTransport = Transport() val init = ZincWorkerRpcServer.Initialize(compilerBridgeWorkspace = compilerBridge.workspace) diff --git a/libs/scalalib/test/src/mill/scalalib/HelloWorldTests.scala b/libs/scalalib/test/src/mill/scalalib/HelloWorldTests.scala index edfa402c6639..eb2a7e5f2d4a 100644 --- a/libs/scalalib/test/src/mill/scalalib/HelloWorldTests.scala +++ b/libs/scalalib/test/src/mill/scalalib/HelloWorldTests.scala @@ -159,101 +159,121 @@ object HelloWorldTests extends TestSuite { test("compile") { test("fromScratch") - UnitTester(HelloWorld, sourceRoot = resourcePath).scoped { eval => - val Right(result) = eval.apply(HelloWorld.core.compile): @unchecked - - val classesPath = eval.outPath / "core/compile.dest/classes" - val analysisFile = result.value.analysisFile - val outputFiles = os.walk(result.value.classes.path) - val expectedClassfiles = compileClassfiles.map(classesPath / _) - assert( - result.value.classes.path == classesPath, - os.exists(analysisFile), - outputFiles.nonEmpty, - outputFiles.forall(expectedClassfiles.contains), - result.evalCount > 0 - ) + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + val Right(result) = eval.apply(HelloWorld.core.compile): @unchecked + + val classesPath = eval.outPath / "core/compile.dest/classes" + val analysisFile = result.value.analysisFile + val outputFiles = os.walk(result.value.classes.path) + val expectedClassfiles = compileClassfiles.map(classesPath / _) + assert( + result.value.classes.path == classesPath, + os.exists(analysisFile), + outputFiles.nonEmpty, + outputFiles.forall(expectedClassfiles.contains), + result.evalCount > 0 + ) - // don't recompile if nothing changed - val Right(result2) = eval.apply(HelloWorld.core.compile): @unchecked + // don't recompile if nothing changed + val Right(result2) = eval.apply(HelloWorld.core.compile): @unchecked - assert(result2.evalCount == 0) + assert(result2.evalCount == 0) - // Make sure we *do not* end up compiling the compiler bridge, since - // it's using a pre-compiled bridge value - assert(!os.exists( - eval.outPath / "mill/scalalib/JvmWorkerModule/internalWorker.dest" / s"zinc-${zincVersion}" - )) + // Make sure we *do not* end up compiling the compiler bridge, since + // it's using a pre-compiled bridge value + assert(!os.exists( + eval.outPath / "mill/scalalib/JvmWorkerModule/internalWorker.dest" / s"zinc-${zincVersion}" + )) + } } test("nonPreCompiledBridge") - UnitTester( HelloWorldNonPrecompiledBridge, sourceRoot = resourcePath ).scoped { eval => - val Right(result) = eval.apply(HelloWorldNonPrecompiledBridge.core.compile): @unchecked + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + val Right(result) = eval.apply(HelloWorldNonPrecompiledBridge.core.compile): @unchecked - val classesPath = eval.outPath / "core/compile.dest/classes" + val classesPath = eval.outPath / "core/compile.dest/classes" - val analysisFile = result.value.analysisFile - val outputFiles = os.walk(result.value.classes.path) - val expectedClassfiles = compileClassfiles.map(classesPath / _) - assert( - result.value.classes.path == classesPath, - os.exists(analysisFile), - outputFiles.nonEmpty, - outputFiles.forall(expectedClassfiles.contains), - result.evalCount > 0 - ) + val analysisFile = result.value.analysisFile + val outputFiles = os.walk(result.value.classes.path) + val expectedClassfiles = compileClassfiles.map(classesPath / _) + assert( + result.value.classes.path == classesPath, + os.exists(analysisFile), + outputFiles.nonEmpty, + outputFiles.forall(expectedClassfiles.contains), + result.evalCount > 0 + ) - // don't recompile if nothing changed - val Right(result2) = eval.apply(HelloWorldNonPrecompiledBridge.core.compile): @unchecked + // don't recompile if nothing changed + val Right(result2) = eval.apply(HelloWorldNonPrecompiledBridge.core.compile): @unchecked - assert(result2.evalCount == 0) + assert(result2.evalCount == 0) - // Make sure we *do* end up compiling the compiler bridge, since it's - // *not* using a pre-compiled bridge value - assert(os.exists( - eval.outPath / "mill.javalib.JvmWorkerModule/internalWorker.dest" / s"zinc-${zincVersion}" - )) + // Make sure we *do* end up compiling the compiler bridge, since it's + // *not* using a pre-compiled bridge value + assert(os.exists( + eval.outPath / "mill.javalib.JvmWorkerModule/internalWorker.dest" / s"zinc-${zincVersion}" + )) + } } test("recompileOnChange") - UnitTester(HelloWorld, sourceRoot = resourcePath).scoped { eval => - val Right(result) = eval.apply(HelloWorld.core.compile): @unchecked - assert(result.evalCount > 0) + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + val Right(result) = eval.apply(HelloWorld.core.compile): @unchecked + assert(result.evalCount > 0) - os.write.append(HelloWorld.moduleDir / "core/src/Main.scala", "\n") + os.write.append(HelloWorld.moduleDir / "core/src/Main.scala", "\n") - val Right(result2) = eval.apply(HelloWorld.core.compile): @unchecked - assert(result2.evalCount > 0, result2.evalCount < result.evalCount) + val Right(result2) = eval.apply(HelloWorld.core.compile): @unchecked + assert(result2.evalCount > 0, result2.evalCount < result.evalCount) + } } test("failOnError") - UnitTester(HelloWorld, sourceRoot = resourcePath).scoped { eval => - os.write.append(HelloWorld.moduleDir / "core/src/Main.scala", "val x: ") + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + os.write.append(HelloWorld.moduleDir / "core/src/Main.scala", "val x: ") - val Left(ExecResult.Failure("Compilation failed")) = - eval.apply(HelloWorld.core.compile): @unchecked + val Left(ExecResult.Failure("Compilation failed")) = + eval.apply(HelloWorld.core.compile): @unchecked - val paths = ExecutionPaths.resolve(eval.outPath, HelloWorld.core.compile) + val paths = ExecutionPaths.resolve(eval.outPath, HelloWorld.core.compile) - assert( - os.walk(paths.dest / "classes").isEmpty, - !os.exists(paths.meta) - ) - // Works when fixed - os.write.over( - HelloWorld.moduleDir / "core/src/Main.scala", - os.read(HelloWorld.moduleDir / "core/src/Main.scala").dropRight( - "val x: ".length + assert( + os.walk(paths.dest / "classes").isEmpty, + !os.exists(paths.meta) + ) + // Works when fixed + os.write.over( + HelloWorld.moduleDir / "core/src/Main.scala", + os.read(HelloWorld.moduleDir / "core/src/Main.scala").dropRight( + "val x: ".length + ) ) - ) - val Right(_) = eval.apply(HelloWorld.core.compile): @unchecked + val Right(_) = eval.apply(HelloWorld.core.compile): @unchecked + } } test("passScalacOptions") - UnitTester( HelloWorldFatalWarnings, sourceRoot = resourcePath ).scoped { eval => - // compilation fails because of "-Xfatal-warnings" flag - val Left(ExecResult.Failure("Compilation failed")) = - eval.apply(HelloWorldFatalWarnings.core.compile): @unchecked + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + // compilation fails because of "-Xfatal-warnings" flag + val Left(ExecResult.Failure("Compilation failed")) = + eval.apply(HelloWorldFatalWarnings.core.compile): @unchecked + } } } @@ -266,39 +286,47 @@ object HelloWorldTests extends TestSuite { test("jar") { test("nonEmpty") - UnitTester(HelloWorldWithMain, resourcePath).scoped { eval => - val Right(result) = eval.apply(HelloWorldWithMain.core.jar): @unchecked + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + val Right(result) = eval.apply(HelloWorldWithMain.core.jar): @unchecked - assert( - os.exists(result.value.path), - result.evalCount > 0 - ) + assert( + os.exists(result.value.path), + result.evalCount > 0 + ) - Using.resource(new JarFile(result.value.path.toIO)) { jarFile => - val entries = jarFile.entries().asScala.map(_.getName).toSeq.sorted + Using.resource(new JarFile(result.value.path.toIO)) { jarFile => + val entries = jarFile.entries().asScala.map(_.getName).toSeq.sorted - val otherFiles = Seq( - "META-INF/", - "META-INF/MANIFEST.MF", - "reference.conf" - ) - val expectedFiles = (compileClassfiles.map(_.toString()) ++ otherFiles).sorted + val otherFiles = Seq( + "META-INF/", + "META-INF/MANIFEST.MF", + "reference.conf" + ) + val expectedFiles = (compileClassfiles.map(_.toString()) ++ otherFiles).sorted - assert( - entries.nonEmpty, - entries == expectedFiles - ) + assert( + entries.nonEmpty, + entries == expectedFiles + ) - val mainClass = jarMainClass(jarFile) - assert(mainClass.contains("Main")) + val mainClass = jarMainClass(jarFile) + assert(mainClass.contains("Main")) + } } } test("logOutputToFile") - UnitTester(HelloWorld, resourcePath).scoped { eval => - val outPath = eval.outPath - eval.apply(HelloWorld.core.compile) - - val logFile = outPath / "core/compile.log" - assert(os.exists(logFile)) + if (scala.util.Properties.isJavaAtLeast(21)) + "Skipping on Java 21+ due to too old Scala version" + else { + val outPath = eval.outPath + eval.apply(HelloWorld.core.compile) + + val logFile = outPath / "core/compile.log" + assert(os.exists(logFile)) + } } } } diff --git a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala index 3a19d2ce9721..be8186daeab9 100644 --- a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala +++ b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala @@ -7,12 +7,11 @@ import mill.api.daemon.internal.{ PathRefApi, RootModuleApi } -import mill.api.{Logger, Result, SystemStreams, Val} +import mill.api.{BuildCtx, Logger, MappedRoots, PathRef, Result, SelectMode, SystemStreams, Val} import mill.constants.CodeGenConstants.* import mill.constants.OutFiles.{millBuild, millRunnerState} import mill.api.daemon.Watchable import mill.api.internal.RootModule -import mill.api.{BuildCtx, PathRef, SelectMode} import mill.internal.PrefixLogger import mill.meta.MillBuildRootModule import mill.meta.CliImports @@ -70,11 +69,13 @@ class MillBuildBootstrap( val runnerState = evaluateRec(0) for ((frame, depth) <- runnerState.frames.zipWithIndex) { - os.write.over( - recOut(output, depth) / millRunnerState, - upickle.write(frame.loggedData, indent = 4), - createFolders = true - ) + MappedRoots.withMillDefaults(outPath = output) { + os.write.over( + recOut(output, depth) / millRunnerState, + upickle.write(frame.loggedData, indent = 4), + createFolders = true + ) + } } Watching.Result( @@ -85,6 +86,14 @@ class MillBuildBootstrap( } def evaluateRec(depth: Int): RunnerState = { + + // We need relocatable PathRef for meta-builds for a stable classpathSig + MappedRoots.requireMappedPaths( + mill.constants.PathVars.WORKSPACE, + mill.constants.PathVars.HOME, + mill.constants.PathVars.MILL_OUT + ) + logger.withChromeProfile(s"meta-level $depth") { // println(s"+evaluateRec($depth) " + recRoot(projectRoot, depth)) val currentRoot = recRoot(projectRoot, depth) @@ -253,24 +262,25 @@ class MillBuildBootstrap( case Result.Success((buildFileApi)) => Using.resource(makeEvaluator( - projectRoot, - output, - keepGoing, - env, - logger, - ec, - allowPositionalCommandArgs, - systemExit, - streams0, - selectiveExecution, - offline, - newWorkerCache, - nestedState.frames.headOption.map(_.codeSignatures).getOrElse(Map.empty), - buildFileApi.rootModule, + projectRoot = projectRoot, + output = output, + keepGoing = keepGoing, + env = env, + logger = logger, + ec = ec, + allowPositionalCommandArgs = allowPositionalCommandArgs, + systemExit = systemExit, + streams0 = streams0, + selectiveExecution = selectiveExecution, + offline = offline, + workerCache = newWorkerCache, + codeSignatures = + nestedState.frames.headOption.map(_.codeSignatures).getOrElse(Map.empty), + rootModule = buildFileApi.rootModule, // We want to use the grandparent buildHash, rather than the parent // buildHash, because the parent build changes are instead detected // by analyzing the scriptImportGraph in a more fine-grained manner. - nestedState + millClassloaderSigHash = nestedState .frames .dropRight(1) .headOption @@ -278,13 +288,13 @@ class MillBuildBootstrap( .getOrElse(millBootClasspathPathRefs) .map(p => (os.Path(p.javaPath), p.sig)) .hashCode(), - nestedState + millClassloaderIdentityHash = nestedState .frames .headOption .flatMap(_.classLoaderOpt) .map(_.hashCode()) .getOrElse(0), - depth, + depth = depth, actualBuildFileName = nestedState.buildFile, enableTicker = enableTicker )) { evaluator => diff --git a/runner/daemon/src/mill/daemon/MillDaemonMain.scala b/runner/daemon/src/mill/daemon/MillDaemonMain.scala index 0fe2bf0eaac0..30389f594d3e 100644 --- a/runner/daemon/src/mill/daemon/MillDaemonMain.scala +++ b/runner/daemon/src/mill/daemon/MillDaemonMain.scala @@ -1,28 +1,29 @@ package mill.daemon -import mill.api.{BuildCtx, SystemStreams} +import mill.api.{MappedRoots, SystemStreams} import mill.client.ClientUtil import mill.client.lock.{Lock, Locks} -import mill.constants.{OutFiles, OutFolderMode} +import mill.constants.OutFolderMode import mill.server.Server import scala.concurrent.duration.* import scala.util.{Failure, Properties, Success, Try} object MillDaemonMain { - case class Args(daemonDir: os.Path, outMode: OutFolderMode, rest: Seq[String]) + case class Args(daemonDir: os.Path, outMode: OutFolderMode, outDir: os.Path, rest: Seq[String]) object Args { def apply(appName: String, args: Array[String]): Either[String, Args] = { def usage(extra: String = "") = - s"usage: $appName $extra" + s"usage: $appName $extra" args match { - case Array(daemonDir, outModeStr, rest*) => + case Array(daemonDir, outModeStr, outDir, rest*) => Try(OutFolderMode.fromString(outModeStr)) match { case Failure(_) => val possibleValues = OutFolderMode.values.map(_.asString).mkString(", ") Left(usage(s"\n\n must be one of $possibleValues but was '$outModeStr'")) - case Success(outMode) => Right(apply(os.Path(daemonDir), outMode, rest)) + case Success(outMode) => + Right(apply(os.Path(daemonDir), outMode, os.Path(outDir), rest)) } case _ => Left(usage()) } @@ -38,29 +39,32 @@ object MillDaemonMain { val args = Args(getClass.getName, args0).fold(err => throw IllegalArgumentException(err), identity) - if (Properties.isWin) - // temporarily disabling FFM use by coursier, which has issues with the way - // Mill manages class loaders, throwing things like - // UnsatisfiedLinkError: Native Library C:\Windows\System32\ole32.dll already loaded in another classloader - sys.props("coursier.windows.disable-ffm") = "true" + MappedRoots.withMillDefaults(outPath = args.outDir) { + if (Properties.isWin) + // temporarily disabling FFM use by coursier, which has issues with the way + // Mill manages class loaders, throwing things like + // UnsatisfiedLinkError: Native Library C:\Windows\System32\ole32.dll already loaded in another classloader + sys.props("coursier.windows.disable-ffm") = "true" - // Take into account proxy-related Java properties - coursier.Resolve.proxySetup() + // Take into account proxy-related Java properties + coursier.Resolve.proxySetup() - mill.api.SystemStreamsUtils.withTopLevelSystemStreamProxy { - Server.overrideSigIntHandling() + mill.api.SystemStreamsUtils.withTopLevelSystemStreamProxy { + Server.overrideSigIntHandling() - val acceptTimeout = - Try(System.getProperty("mill.server_timeout").toInt.millis).getOrElse(30.minutes) + val acceptTimeout = + Try(System.getProperty("mill.server_timeout").toInt.millis).getOrElse(30.minutes) - new MillDaemonMain( - daemonDir = args.daemonDir, - acceptTimeout = acceptTimeout, - Locks.files(args.daemonDir.toString), - outMode = args.outMode - ).run() + new MillDaemonMain( + daemonDir = args.daemonDir, + acceptTimeout = acceptTimeout, + Locks.files(args.daemonDir.toString), + outMode = args.outMode, + outDir = args.outDir + ).run() - System.exit(ClientUtil.ExitServerCodeWhenIdle()) + System.exit(ClientUtil.ExitServerCodeWhenIdle()) + } } } } @@ -68,18 +72,18 @@ class MillDaemonMain( daemonDir: os.Path, acceptTimeout: FiniteDuration, locks: Locks, - outMode: OutFolderMode + outMode: OutFolderMode, + outDir: os.Path ) extends mill.server.MillDaemonServer[RunnerState]( - daemonDir, - acceptTimeout, - locks + daemonDir = daemonDir, + acceptTimeout = acceptTimeout, + locks = locks, + outDir = outDir ) { def stateCache0 = RunnerState.empty - val out: os.Path = os.Path(OutFiles.outFor(outMode), BuildCtx.workspaceRoot) - - val outLock = MillMain0.doubleLock(out) + val outLock = MillMain0.doubleLock(outDir) def main0( args: Array[String], @@ -103,7 +107,8 @@ class MillDaemonMain( initialSystemProperties = initialSystemProperties, systemExit = systemExit, daemonDir = daemonDir, - outLock = outLock + outLock = outLock, + outDir = outDir ) catch MillMain0.handleMillException(streams.err, stateCache) } diff --git a/runner/daemon/src/mill/daemon/MillMain0.scala b/runner/daemon/src/mill/daemon/MillMain0.scala index 47d480858aea..a8c8e5e667a4 100644 --- a/runner/daemon/src/mill/daemon/MillMain0.scala +++ b/runner/daemon/src/mill/daemon/MillMain0.scala @@ -3,11 +3,10 @@ package mill.daemon import ch.epfl.scala.bsp4j.BuildClient import mill.api.daemon.internal.bsp.BspServerHandle import mill.api.daemon.internal.{CompileProblemReporter, EvaluatorApi} -import mill.api.{Logger, MillException, Result, SystemStreams} +import mill.api.{BuildCtx, Logger, MappedRoots, MillException, Result, SystemStreams} import mill.bsp.BSP import mill.client.lock.{DoubleLock, Lock} -import mill.constants.{DaemonFiles, OutFiles, OutFolderMode} -import mill.api.BuildCtx +import mill.constants.{DaemonFiles, OutFiles} import mill.internal.{ Colors, JsonArrayLogger, @@ -65,12 +64,12 @@ object MillMain0 { private def withStreams[T]( bspMode: Boolean, - streams: SystemStreams - )(thunk: SystemStreams => T): T = + streams: SystemStreams, + outDir: os.Path + )(thunk: SystemStreams => T): T = { if (bspMode) { // In BSP mode, don't let anything other than the BSP server write to stdout and read from stdin - val outDir = BuildCtx.workspaceRoot / os.RelPath(OutFiles.outFor(OutFolderMode.BSP)) val outFileStream = os.write.outputStream( outDir / "mill-bsp/out.log", createFolders = true @@ -97,6 +96,7 @@ object MillMain0 { mill.api.SystemStreamsUtils.withStreams(streams) { thunk(streams) } + } def main0( args: Array[String], @@ -109,416 +109,438 @@ object MillMain0 { initialSystemProperties: Map[String, String], systemExit: Server.StopServer, daemonDir: os.Path, - outLock: Lock + outLock: Lock, + outDir: os.Path ): (Boolean, RunnerState) = - mill.api.daemon.internal.MillScalaParser.current.withValue(MillScalaParserImpl) { - os.SubProcess.env.withValue(env) { - val parserResult = MillCliConfig.parse(args) - // Detect when we're running in BSP mode as early as possible, - // and ensure we don't log to the default stdout or use the default - // stdin, meant to be used for BSP JSONRPC communication, where those - // logs would be lost. - // This is especially helpful if anything unexpectedly goes wrong - // early on, when developing on Mill or debugging things for example. - val bspMode = parserResult.toOption.exists(_.bsp.value) - withStreams(bspMode, streams0) { streams => - parserResult match { - // Cannot parse args - case Result.Failure(msg) => - streams.err.println(msg) - (false, RunnerState.empty) - - case Result.Success(config) if config.help.value => - streams.out.println(MillCliConfig.longUsageText) - (true, RunnerState.empty) - - case Result.Success(config) if config.helpAdvanced.value => - streams.out.println(MillCliConfig.helpAdvancedUsageText) - (true, RunnerState.empty) - - case Result.Success(config) if config.showVersion.value => - def prop(k: String) = System.getProperty(k, s"") - - val javaVersion = prop("java.version") - val javaVendor = prop("java.vendor") - val javaHome = prop("java.home") - val fileEncoding = prop("file.encoding") - val osName = prop("os.name") - val osVersion = prop("os.version") - val osArch = prop("os.arch") - streams.out.println( - s"""Mill Build Tool version ${BuildInfo.millVersion} - |Java version: $javaVersion, vendor: $javaVendor, runtime: $javaHome - |Default locale: ${Locale.getDefault()}, platform encoding: $fileEncoding - |OS name: "$osName", version: $osVersion, arch: $osArch""".stripMargin - ) - (true, RunnerState.empty) - - case Result.Success(config) if config.noDaemonEnabled > 1 => - streams.err.println( - "Only one of -i/--interactive, --no-daemon or --bsp may be given" - ) - (false, RunnerState.empty) - - // Check non-negative --meta-level option - case Result.Success(config) if config.metaLevel.exists(_ < 0) => - streams.err.println("--meta-level cannot be negative") - (false, RunnerState.empty) - - case Result.Success(config) => - val noColorViaEnv = env.get("NO_COLOR").exists(_.nonEmpty) - val colored = config.color.getOrElse(mainInteractive && !noColorViaEnv) - val colors = - if (colored) mill.internal.Colors.Default else mill.internal.Colors.BlackWhite - - checkMillVersionFromFile(BuildCtx.workspaceRoot, streams.err) - - val maybeThreadCount = - parseThreadCount(config.threadCountRaw, Runtime.getRuntime.availableProcessors()) - - // special BSP mode, in which we spawn a server and register the current evaluator when-ever we start to eval a dedicated command - val bspMode = config.bsp.value && config.leftoverArgs.value.isEmpty - val outMode = if (bspMode) OutFolderMode.BSP else OutFolderMode.REGULAR - val bspInstallModeJobCountOpt = { - def defaultJobCount = - maybeThreadCount.toOption.getOrElse(BSP.defaultJobCount) - - val viaEmulatedExternalCommand = Option.when( - !config.bsp.value && - (config.leftoverArgs.value.headOption.contains("mill.bsp.BSP/install") || - config.leftoverArgs.value.headOption.contains("mill.bsp/install")) - ) { - config.leftoverArgs.value.tail match { - case Seq() => defaultJobCount - case Seq("--jobs", value) => - val asIntOpt = value.toIntOption - asIntOpt.getOrElse { + MappedRoots.withMillDefaults(outPath = outDir) { + mill.api.daemon.internal.MillScalaParser.current.withValue(MillScalaParserImpl) { + os.SubProcess.env.withValue(env) { + val parserResult = MillCliConfig.parse(args) + // Detect when we're running in BSP mode as early as possible, + // and ensure we don't log to the default stdout or use the default + // stdin, meant to be used for BSP JSONRPC communication, where those + // logs would be lost. + // This is especially helpful if anything unexpectedly goes wrong + // early on, when developing on Mill or debugging things for example. + val bspMode = parserResult.toOption.exists(_.bsp.value) + withStreams(bspMode, streams0, outDir) { streams => + parserResult match { + // Cannot parse args + case Result.Failure(msg) => + streams.err.println(msg) + (false, RunnerState.empty) + + case Result.Success(config) if config.help.value => + streams.out.println(MillCliConfig.longUsageText) + (true, RunnerState.empty) + + case Result.Success(config) if config.helpAdvanced.value => + streams.out.println(MillCliConfig.helpAdvancedUsageText) + (true, RunnerState.empty) + + case Result.Success(config) if config.showVersion.value => + def prop(k: String) = System.getProperty(k, s"") + + val javaVersion = prop("java.version") + val javaVendor = prop("java.vendor") + val javaHome = prop("java.home") + val fileEncoding = prop("file.encoding") + val osName = prop("os.name") + val osVersion = prop("os.version") + val osArch = prop("os.arch") + streams.out.println( + s"""Mill Build Tool version ${BuildInfo.millVersion} + |Java version: $javaVersion, vendor: $javaVendor, runtime: $javaHome + |Default locale: ${Locale.getDefault()}, platform encoding: $fileEncoding + |OS name: "$osName", version: $osVersion, arch: $osArch""".stripMargin + ) + (true, RunnerState.empty) + + case Result.Success(config) if config.noDaemonEnabled > 1 => + streams.err.println( + "Only one of -i/--interactive, --no-daemon or --bsp may be given" + ) + (false, RunnerState.empty) + + // Check non-negative --meta-level option + case Result.Success(config) if config.metaLevel.exists(_ < 0) => + streams.err.println("--meta-level cannot be negative") + (false, RunnerState.empty) + + case Result.Success(config) => + val noColorViaEnv = env.get("NO_COLOR").exists(_.nonEmpty) + val colored = config.color.getOrElse(mainInteractive && !noColorViaEnv) + val colors = + if (colored) mill.internal.Colors.Default else mill.internal.Colors.BlackWhite + + checkMillVersionFromFile(BuildCtx.workspaceRoot, streams.err) + + val maybeThreadCount = + parseThreadCount(config.threadCountRaw, Runtime.getRuntime.availableProcessors()) + + // special BSP mode, in which we spawn a server and register the current evaluator when-ever we start to eval a dedicated command + val bspMode = config.bsp.value && config.leftoverArgs.value.isEmpty + + val bspInstallModeJobCountOpt = { + def defaultJobCount = + maybeThreadCount.toOption.getOrElse(BSP.defaultJobCount) + + val viaEmulatedExternalCommand = Option.when( + !config.bsp.value && + (config.leftoverArgs.value.headOption.contains("mill.bsp.BSP/install") || + config.leftoverArgs.value.headOption.contains("mill.bsp/install")) + ) { + config.leftoverArgs.value.tail match { + case Seq() => defaultJobCount + case Seq("--jobs", value) => + val asIntOpt = value.toIntOption + asIntOpt.getOrElse { + streams.err.println( + s"Warning: ignoring --jobs value passed to ${config.leftoverArgs.value.head}" + ) + defaultJobCount + } + case _ => streams.err.println( - s"Warning: ignoring --jobs value passed to ${config.leftoverArgs.value.head}" + s"Warning: ignoring leftover arguments passed to ${config.leftoverArgs.value.head}" ) defaultJobCount - } - case _ => - streams.err.println( - s"Warning: ignoring leftover arguments passed to ${config.leftoverArgs.value.head}" - ) - defaultJobCount + } } - } - viaEmulatedExternalCommand.orElse { - Option.when(config.bspInstall.value)(defaultJobCount) + viaEmulatedExternalCommand.orElse { + Option.when(config.bspInstall.value)(defaultJobCount) + } } - } - val enableTicker = config.ticker - .orElse(config.enableTicker) - .orElse(Option.when(config.tabComplete.value)(false)) - .orElse(Option.when(config.disableTicker.value)(false)) - .getOrElse(true) - - val (success, nextStateCache) = { - if (bspInstallModeJobCountOpt.isDefined) { - BSP.install(bspInstallModeJobCountOpt.get, config.debugLog.value, streams.err) - (true, stateCache) - } else if ( - !bspMode && !config.jshell.value && !config.repl.value && config.leftoverArgs.value.isEmpty - ) { - println(MillCliConfig.shortUsageText) - - (true, stateCache) - - } else if (maybeThreadCount.errorOpt.isDefined) { - streams.err.println(maybeThreadCount.errorOpt.get) - (false, stateCache) - - } else { - val userSpecifiedProperties = - userSpecifiedProperties0 ++ config.extraSystemProperties - - val threadCount = maybeThreadCount.toOption.get - - def createEc(): Option[ThreadPoolExecutor] = - if (threadCount == 1) None - else Some(mill.exec.ExecutionContexts.createExecutor(threadCount)) - - val out = os.Path(OutFiles.outFor(outMode), BuildCtx.workspaceRoot) - Using.resources(new TailManager(daemonDir), createEc()) { (tailManager, ec) => - def runMillBootstrap( - skipSelectiveExecution: Boolean, - prevState: Option[RunnerState], - tasksAndParams: Seq[String], - streams: SystemStreams, - millActiveCommandMessage: String, - loggerOpt: Option[Logger] = None, - reporter: EvaluatorApi => Int => Option[CompileProblemReporter] = - _ => _ => None, - extraEnv: Seq[(String, String)] = Nil, - metaLevelOverride: Option[Int] = None - ) = MillDaemonServer.withOutLock( - config.noBuildLock.value, - config.noWaitForBuildLock.value, - out, - millActiveCommandMessage, - streams, - outLock - ) { - def proceed(logger: Logger): Watching.Result[RunnerState] = { - // Enter key pressed, removing mill-selective-execution.json to - // ensure all tasks re-run even though no inputs may have changed - // - // Do this by removing the file rather than disabling selective execution, - // because we still want to generate the selective execution metadata json - // for subsequent runs that may use it - if (skipSelectiveExecution) os.remove(out / OutFiles.millSelectiveExecution) - mill.api.SystemStreamsUtils.withStreams(logger.streams) { - mill.api.FilesystemCheckerEnabled.withValue( - !config.noFilesystemChecker.value - ) { - tailManager.withOutErr(logger.streams.out, logger.streams.err) { - new MillBuildBootstrap( - projectRoot = BuildCtx.workspaceRoot, - output = out, - // In BSP server, we want to evaluate as many tasks as possible, - // in order to give as many results as available in BSP responses - keepGoing = bspMode || config.keepGoing.value, - imports = config.imports, - env = env ++ extraEnv, - ec = ec, - tasksAndParams = tasksAndParams, - prevRunnerState = prevState.getOrElse(stateCache), - logger = logger, - requestedMetaLevel = config.metaLevel.orElse(metaLevelOverride), - allowPositionalCommandArgs = config.allowPositional.value, - systemExit = systemExit, - streams0 = streams, - selectiveExecution = config.watch.value, - offline = config.offline.value, - reporter = reporter, - enableTicker = enableTicker - ).evaluate() + val enableTicker = config.ticker + .orElse(config.enableTicker) + .orElse(Option.when(config.tabComplete.value)(false)) + .orElse(Option.when(config.disableTicker.value)(false)) + .getOrElse(true) + + val (success, nextStateCache) = { + if (bspInstallModeJobCountOpt.isDefined) { + BSP.install(bspInstallModeJobCountOpt.get, config.debugLog.value, streams.err) + (true, stateCache) + } else if ( + !bspMode && !config.jshell.value && !config.repl.value && config.leftoverArgs.value.isEmpty + ) { + println(MillCliConfig.shortUsageText) + + (true, stateCache) + + } else if (maybeThreadCount.errorOpt.isDefined) { + streams.err.println(maybeThreadCount.errorOpt.get) + (false, stateCache) + + } else { + val userSpecifiedProperties = + userSpecifiedProperties0 ++ config.extraSystemProperties + + val threadCount = maybeThreadCount.toOption.get + + def createEc(): Option[ThreadPoolExecutor] = + if (threadCount == 1) None + else Some(mill.exec.ExecutionContexts.createExecutor(threadCount)) + + Using.resources(new TailManager(daemonDir), createEc()) { (tailManager, ec) => + def runMillBootstrap( + skipSelectiveExecution: Boolean, + prevState: Option[RunnerState], + tasksAndParams: Seq[String], + streams: SystemStreams, + millActiveCommandMessage: String, + loggerOpt: Option[Logger] = None, + reporter: EvaluatorApi => Int => Option[CompileProblemReporter] = + _ => _ => None, + extraEnv: Seq[(String, String)] = Nil, + metaLevelOverride: Option[Int] = None + ): Watching.Result[RunnerState] = MillDaemonServer.withOutLock( + config.noBuildLock.value, + config.noWaitForBuildLock.value, + outDir, + millActiveCommandMessage, + streams, + outLock + ) { + def proceed(logger: Logger): Watching.Result[RunnerState] = { + // Enter key pressed, removing mill-selective-execution.json to + // ensure all tasks re-run even though no inputs may have changed + // + // Do this by removing the file rather than disabling selective execution, + // because we still want to generate the selective execution metadata json + // for subsequent runs that may use it + if (skipSelectiveExecution) + os.remove(outDir / OutFiles.millSelectiveExecution) + mill.api.SystemStreamsUtils.withStreams(logger.streams) { + mill.api.FilesystemCheckerEnabled.withValue( + !config.noFilesystemChecker.value + ) { + tailManager.withOutErr(logger.streams.out, logger.streams.err) { + new MillBuildBootstrap( + projectRoot = BuildCtx.workspaceRoot, + output = outDir, + // In BSP server, we want to evaluate as many tasks as possible, + // in order to give as many results as available in BSP responses + keepGoing = bspMode || config.keepGoing.value, + imports = config.imports, + env = env ++ extraEnv, + ec = ec, + tasksAndParams = tasksAndParams, + prevRunnerState = prevState.getOrElse(stateCache), + logger = logger, + requestedMetaLevel = config.metaLevel.orElse(metaLevelOverride), + allowPositionalCommandArgs = config.allowPositional.value, + systemExit = systemExit, + streams0 = streams, + selectiveExecution = config.watch.value, + offline = config.offline.value, + reporter = reporter, + enableTicker = enableTicker + ).evaluate() + } } } } - } - loggerOpt match { - case Some(logger) => - proceed(logger) - case None => - Using.resource(getLogger( - streams, - config, - enableTicker = enableTicker, - daemonDir, - colored = colored, - colors = colors, - out = out - )) { logger => + loggerOpt match { + case Some(logger) => proceed(logger) - } + case None => + Using.resource(getLogger( + streams, + config, + enableTicker = enableTicker, + daemonDir, + colored = colored, + colors = colors, + out = outDir + )) { logger => + proceed(logger) + } + } } - } - if (config.jshell.value) { - val bootstrapped = runMillBootstrap( - skipSelectiveExecution = false, - Some(stateCache), - Seq("jshell") ++ config.leftoverArgs.value, - streams, - "jshell", - metaLevelOverride = Some(1) - ) - - (true, bootstrapped.result) - } else if (config.repl.value) { - val bootstrapped = runMillBootstrap( - skipSelectiveExecution = false, - Some(stateCache), - Seq("console") ++ config.leftoverArgs.value, - streams, - "repl", - metaLevelOverride = Some(1) - ) - - (true, bootstrapped.result) - } else if (config.tabComplete.value) { - val bootstrapped = runMillBootstrap( - skipSelectiveExecution = false, - Some(stateCache), - Seq( - "mill.tabcomplete.TabCompleteModule/complete" - ) ++ config.leftoverArgs.value, - streams, - "tab-completion" - ) - - (true, bootstrapped.result) - } else if (bspMode) { - val bspLogger = getBspLogger(streams, config) - var prevRunnerStateOpt = Option.empty[RunnerState] - val (bspServerHandle, buildClient) = - startBspServer(streams0, outLock, bspLogger) - var keepGoing = true - var errored = false - val initCommandLogger = new PrefixLogger(bspLogger, Seq("init")) - val watchLogger = new PrefixLogger(bspLogger, Seq("watch")) - while (keepGoing) { - val watchRes = runMillBootstrap( - false, - prevRunnerStateOpt, - Seq("version"), - initCommandLogger.streams, - "BSP:initialize", - loggerOpt = Some(initCommandLogger), - reporter = ev => { - val bspIdByModule = mill.bsp.worker.BspEvaluators( - BuildCtx.workspaceRoot, - Seq(ev), - _ => (), - Nil - ).bspIdByModule - mill.bsp.worker.Utils.getBspLoggedReporterPool( - "", - bspIdByModule, - buildClient - ) - } + if (config.jshell.value) { + val bootstrapped = runMillBootstrap( + skipSelectiveExecution = false, + Some(stateCache), + Seq("jshell") ++ config.leftoverArgs.value, + streams, + "jshell", + metaLevelOverride = Some(1) ) - for (err <- watchRes.error) - bspLogger.streams.err.println(err) - - prevRunnerStateOpt = Some(watchRes.result) - - val sessionResultFuture = bspServerHandle.startSession( - watchRes.result.frames.flatMap(_.evaluator), - errored = watchRes.error.nonEmpty, - watched = watchRes.watched + (true, bootstrapped.result) + } else if (config.repl.value) { + val bootstrapped = runMillBootstrap( + skipSelectiveExecution = false, + Some(stateCache), + Seq("console") ++ config.leftoverArgs.value, + streams, + "repl", + metaLevelOverride = Some(1) ) - def waitWithoutWatching() = { - Some { - try Success(Await.result(sessionResultFuture, Duration.Inf)) - catch { - case NonFatal(ex) => - Failure(ex) - } - } - } + (true, bootstrapped.result) + } else if (config.tabComplete.value) { + val bootstrapped = runMillBootstrap( + skipSelectiveExecution = false, + Some(stateCache), + Seq( + "mill.tabcomplete.TabCompleteModule/complete" + ) ++ config.leftoverArgs.value, + streams, + "tab-completion" + ) - val res = - if (config.bspWatch) { - try { - Watching.watchAndWait( - watchRes.watched, - Watching.WatchArgs( - setIdle = setIdle, - colors = mill.internal.Colors.BlackWhite, - useNotify = config.watchViaFsNotify, - daemonDir = daemonDir - ), - () => sessionResultFuture.value, + (true, bootstrapped.result) + } else if (bspMode) { + val bspLogger = getBspLogger(streams, config) + var prevRunnerStateOpt = Option.empty[RunnerState] + val (bspServerHandle, buildClient) = + startBspServer(streams0, outLock, bspLogger, outDir) + var keepGoing = true + var errored = false + val initCommandLogger = new PrefixLogger(bspLogger, Seq("init")) + val watchLogger = new PrefixLogger(bspLogger, Seq("watch")) + while (keepGoing) { + val watchRes = runMillBootstrap( + false, + prevRunnerStateOpt, + Seq("version"), + initCommandLogger.streams, + "BSP:initialize", + loggerOpt = Some(initCommandLogger), + reporter = ev => { + val bspIdByModule = mill.bsp.worker.BspEvaluators( + BuildCtx.workspaceRoot, + Seq(ev), + _ => (), + Nil + ).bspIdByModule + mill.bsp.worker.Utils.getBspLoggedReporterPool( "", - watchLogger.info(_) + bspIdByModule, + buildClient ) - } catch { - case e: Exception => - val sw = new java.io.StringWriter - e.printStackTrace(new java.io.PrintWriter(sw)) - watchLogger.info( - "Watching of build sources failed:" + e + "\n" + sw - ) - waitWithoutWatching() } - } else { - watchLogger.info("Watching of build sources disabled") - waitWithoutWatching() + ) + + for (err <- watchRes.error) + bspLogger.streams.err.println(err) + + prevRunnerStateOpt = Some(watchRes.result) + + val sessionResultFuture = bspServerHandle.startSession( + watchRes.result.frames.flatMap(_.evaluator), + errored = watchRes.error.nonEmpty, + watched = watchRes.watched + ) + + def waitWithoutWatching() = { + Some { + try Success(Await.result(sessionResultFuture, Duration.Inf)) + catch { + case NonFatal(ex) => + Failure(ex) + } + } } - // Suspend any BSP request until the next call to startSession - // (that is, until we've attempted to re-compile the build) - bspServerHandle.resetSession() + val res = + if (config.bspWatch) { + try { + Watching.watchAndWait( + watchRes.watched, + Watching.WatchArgs( + setIdle = setIdle, + colors = mill.internal.Colors.BlackWhite, + useNotify = config.watchViaFsNotify, + daemonDir = daemonDir + ), + () => sessionResultFuture.value, + "", + watchLogger.info(_) + ) + } catch { + case e: Exception => + val sw = new java.io.StringWriter + e.printStackTrace(new java.io.PrintWriter(sw)) + watchLogger.info( + "Watching of build sources failed:" + e + "\n" + sw + ) + waitWithoutWatching() + } + } else { + watchLogger.info("Watching of build sources disabled") + waitWithoutWatching() + } - res match { - case None => - // Some watched meta-build files changed - case Some(Failure(ex)) => - streams.err.println("BSP server threw an exception, exiting") - ex.printStackTrace(streams.err) - errored = true - keepGoing = false - case Some(Success(BspServerResult.ReloadWorkspace)) => - // reload asked by client - case Some(Success(BspServerResult.Shutdown)) => - streams.err.println("BSP shutdown asked by client, exiting") - // shutdown asked by client - keepGoing = false - // should make the lsp4j-managed BSP server exit - streams.in.close() + // Suspend any BSP request until the next call to startSession + // (that is, until we've attempted to re-compile the build) + bspServerHandle.resetSession() + + res match { + case None => + // Some watched meta-build files changed + case Some(Failure(ex)) => + streams.err.println("BSP server threw an exception, exiting") + ex.printStackTrace(streams.err) + errored = true + keepGoing = false + case Some(Success(BspServerResult.ReloadWorkspace)) => + // reload asked by client + case Some(Success(BspServerResult.Shutdown)) => + streams.err.println("BSP shutdown asked by client, exiting") + // shutdown asked by client + keepGoing = false + // should make the lsp4j-managed BSP server exit + streams.in.close() + } } - } - streams.err.println("Exiting BSP runner loop") - - (!errored, RunnerState(None, Nil, None)) - } else if ( - config.leftoverArgs.value == Seq("mill.idea.GenIdea/idea") || - config.leftoverArgs.value == Seq("mill.idea.GenIdea/") || - config.leftoverArgs.value == Seq("mill.idea/") - ) { - val runnerState = - runMillBootstrap(false, None, Seq("version"), streams, "BSP:initialize") - new mill.idea.GenIdeaImpl( - runnerState.result.frames.flatMap(_.evaluator) - ).run() - (true, RunnerState(None, Nil, None)) - } else if ( - config.leftoverArgs.value == Seq("mill.eclipse.GenEclipse/eclipse") || - config.leftoverArgs.value == Seq("mill.eclipse.GenEclipse/") || - config.leftoverArgs.value == Seq("mill.eclipse/") - ) { - val runnerState = - runMillBootstrap(false, None, Seq("version"), streams, "BSP:initialize") - new mill.eclipse.GenEclipseImpl( - runnerState.result.frames.flatMap(_.evaluator) - ).run() - (true, RunnerState(None, Nil, None)) - } else { - Watching.watchLoop( - ringBell = config.ringBell.value, - watch = Option.when(config.watch.value)(Watching.WatchArgs( - setIdle = setIdle, - colors, - useNotify = config.watchViaFsNotify, - daemonDir = daemonDir - )), - streams = streams, - evaluate = - (skipSelectiveExecution: Boolean, prevState: Option[RunnerState]) => { - adjustJvmProperties(userSpecifiedProperties, initialSystemProperties) - runMillBootstrap( - skipSelectiveExecution, - prevState, - config.leftoverArgs.value, - streams, - config.leftoverArgs.value.mkString(" ") - ) - } - ) + streams.err.println("Exiting BSP runner loop") + + (!errored, RunnerState(None, Nil, None)) + } else if ( + config.leftoverArgs.value == Seq("mill.idea.GenIdea/idea") || + config.leftoverArgs.value == Seq("mill.idea.GenIdea/") || + config.leftoverArgs.value == Seq("mill.idea/") + ) { + val runnerState = + runMillBootstrap( + false, + None, + Seq("version"), + streams, + "BSP:initialize" + ) + new mill.idea.GenIdeaImpl( + runnerState.result.frames.flatMap(_.evaluator) + ).run() + (true, RunnerState(None, Nil, None)) + } else if ( + config.leftoverArgs.value == Seq("mill.eclipse.GenEclipse/eclipse") || + config.leftoverArgs.value == Seq("mill.eclipse.GenEclipse/") || + config.leftoverArgs.value == Seq("mill.eclipse/") + ) { + val runnerState = + runMillBootstrap( + false, + None, + Seq("version"), + streams, + "BSP:initialize" + ) + new mill.eclipse.GenEclipseImpl( + runnerState.result.frames.flatMap(_.evaluator) + ).run() + (true, RunnerState(None, Nil, None)) + } else { + Watching.watchLoop( + ringBell = config.ringBell.value, + watch = Option.when(config.watch.value)(Watching.WatchArgs( + setIdle = setIdle, + colors, + useNotify = config.watchViaFsNotify, + daemonDir = daemonDir + )), + streams = streams, + evaluate = + ( + skipSelectiveExecution: Boolean, + prevState: Option[RunnerState] + ) => { + adjustJvmProperties( + userSpecifiedProperties, + initialSystemProperties + ) + runMillBootstrap( + skipSelectiveExecution, + prevState, + config.leftoverArgs.value, + streams, + config.leftoverArgs.value.mkString(" ") + ) + } + ) + } } } } - } - if (config.ringBell.value) { - if (success) println("\u0007") - else { - println("\u0007") - Thread.sleep(250) - println("\u0007") + + if (config.ringBell.value) { + if (success) println("\u0007") + else { + println("\u0007") + Thread.sleep(250) + println("\u0007") + } } - } - (success, nextStateCache) + (success, nextStateCache) + } } } } @@ -532,13 +554,13 @@ object MillMain0 { def startBspServer( bspStreams: SystemStreams, outLock: Lock, - bspLogger: Logger + bspLogger: Logger, + outDir: os.Path ): (BspServerHandle, BuildClient) = { bspLogger.info("Trying to load BSP server...") - val wsRoot = BuildCtx.workspaceRoot - val outFolder = wsRoot / os.RelPath(OutFiles.outFor(OutFolderMode.BSP)) - val logDir = outFolder / "mill-bsp" + BuildCtx.workspaceRoot + val logDir = outDir / "mill-bsp" os.makeDir.all(logDir) val bspServerHandleRes = @@ -549,7 +571,7 @@ object MillMain0 { true, outLock, bspLogger, - outFolder + outDir ).get bspLogger.info("BSP server started") diff --git a/runner/daemon/src/mill/daemon/MillNoDaemonMain.scala b/runner/daemon/src/mill/daemon/MillNoDaemonMain.scala index f70dbd2282a7..bcbb5754876a 100644 --- a/runner/daemon/src/mill/daemon/MillNoDaemonMain.scala +++ b/runner/daemon/src/mill/daemon/MillNoDaemonMain.scala @@ -2,13 +2,13 @@ package mill.daemon import mill.constants.{DaemonFiles, OutFiles, Util} import mill.daemon.MillMain0.{handleMillException, main0} -import mill.api.BuildCtx import mill.server.Server import scala.jdk.CollectionConverters.* import scala.util.Properties object MillNoDaemonMain { + def main(args0: Array[String]): Unit = mill.api.SystemStreamsUtils.withTopLevelSystemStreamProxy { val initialSystemStreams = mill.api.SystemStreams.original @@ -28,9 +28,8 @@ object MillNoDaemonMain { .fold(err => throw IllegalArgumentException(err), identity) val processId = Server.computeProcessId() - val out = os.Path(OutFiles.outFor(args.outMode), BuildCtx.workspaceRoot) Server.watchProcessIdFile( - out / OutFiles.millNoDaemon / s"pid-$processId" / DaemonFiles.processId, + args.outDir / OutFiles.millNoDaemon / s"pid-$processId" / DaemonFiles.processId, processId, running = () => true, exit = msg => { @@ -39,7 +38,7 @@ object MillNoDaemonMain { } ) - val outLock = MillMain0.doubleLock(out) + val outLock = MillMain0.doubleLock(args.outDir) val (result, _) = try main0( @@ -53,7 +52,8 @@ object MillNoDaemonMain { initialSystemProperties = sys.props.toMap, systemExit = ( /*reason*/ _, exitCode) => sys.exit(exitCode), daemonDir = args.daemonDir, - outLock = outLock + outLock = outLock, + outDir = args.outDir ) catch handleMillException(initialSystemStreams.err, ()) diff --git a/runner/launcher/src/mill/launcher/MillLauncherMain.java b/runner/launcher/src/mill/launcher/MillLauncherMain.java index 65b0fa749f16..e9cfc3deefd2 100644 --- a/runner/launcher/src/mill/launcher/MillLauncherMain.java +++ b/runner/launcher/src/mill/launcher/MillLauncherMain.java @@ -1,5 +1,6 @@ package mill.launcher; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -54,6 +55,7 @@ public static void main(String[] args) throws Exception { var outMode = bspMode ? OutFolderMode.BSP : OutFolderMode.REGULAR; exitInTestsAfterBspCheck(); var outDir = OutFiles.outFor(outMode); + var outPath = new File(outDir).getAbsoluteFile(); if (outMode == OutFolderMode.BSP) { System.err.println( @@ -87,8 +89,8 @@ public static void main(String[] args) throws Exception { if (runNoDaemon) { String mainClass = bspMode ? "mill.daemon.MillBspMain" : "mill.daemon.MillNoDaemonMain"; // start in no-server mode - int exitCode = - MillProcessLauncher.launchMillNoDaemon(args, outMode, runnerClasspath, mainClass); + int exitCode = MillProcessLauncher.launchMillNoDaemon( + args, outMode, outPath, runnerClasspath, mainClass); System.exit(exitCode); } else { var logs = new java.util.ArrayList(); @@ -107,9 +109,9 @@ public static void main(String[] args) throws Exception { Optional.empty(), -1) { public LaunchedServer initServer(Path daemonDir, Locks locks) throws Exception { - return new LaunchedServer.OsProcess( - MillProcessLauncher.launchMillDaemon(daemonDir, outMode, runnerClasspath) - .toHandle()); + return new LaunchedServer.OsProcess(MillProcessLauncher.launchMillDaemon( + daemonDir, outMode, outPath, runnerClasspath) + .toHandle()); } }; diff --git a/runner/launcher/src/mill/launcher/MillProcessLauncher.java b/runner/launcher/src/mill/launcher/MillProcessLauncher.java index 97a4c194cd37..885723df8299 100644 --- a/runner/launcher/src/mill/launcher/MillProcessLauncher.java +++ b/runner/launcher/src/mill/launcher/MillProcessLauncher.java @@ -20,7 +20,7 @@ public class MillProcessLauncher { static int launchMillNoDaemon( - String[] args, OutFolderMode outMode, String[] runnerClasspath, String mainClass) + String[] args, OutFolderMode outMode, File outDir, String[] runnerClasspath, String mainClass) throws Exception { final String sig = String.format("%08x", UUID.randomUUID().hashCode()); final Path processDir = @@ -35,6 +35,7 @@ static int launchMillNoDaemon( l.add(mainClass); l.add(processDir.toAbsolutePath().toString()); l.add(outMode.asString()); + l.add(outDir.toString()); l.addAll(millOpts(outMode)); l.addAll(Arrays.asList(args)); @@ -60,12 +61,14 @@ static int launchMillNoDaemon( } } - static Process launchMillDaemon(Path daemonDir, OutFolderMode outMode, String[] runnerClasspath) + static Process launchMillDaemon( + Path daemonDir, OutFolderMode outMode, File outDir, String[] runnerClasspath) throws Exception { List l = new ArrayList<>(millLaunchJvmCommand(outMode, runnerClasspath)); l.add("mill.daemon.MillDaemonMain"); l.add(daemonDir.toFile().getCanonicalPath()); l.add(outMode.asString()); + l.add(outDir.toString()); ProcessBuilder builder = new ProcessBuilder() .command(l) @@ -261,11 +264,6 @@ static List millLaunchJvmCommand(OutFolderMode outMode, String[] runnerC return vmOptions; } - static String[] cachedComputedValue( - OutFolderMode outMode, String name, String key, Supplier block) { - return cachedComputedValue0(outMode, name, key, block, arr -> true); - } - static String[] cachedComputedValue0( OutFolderMode outMode, String name, @@ -303,6 +301,11 @@ static String[] cachedComputedValue0( } } + static String[] cachedComputedValue( + OutFolderMode outMode, String name, String key, Supplier block) { + return cachedComputedValue0(outMode, name, key, block, arr -> true); + } + static int getTerminalDim(String s, boolean inheritError) throws Exception { Process proc = new ProcessBuilder() .command("tput", s) diff --git a/runner/server/src/mill/server/MillDaemonServer.scala b/runner/server/src/mill/server/MillDaemonServer.scala index f0354fe67bec..71838fd96495 100644 --- a/runner/server/src/mill/server/MillDaemonServer.scala +++ b/runner/server/src/mill/server/MillDaemonServer.scala @@ -22,6 +22,7 @@ abstract class MillDaemonServer[State]( daemonDir: os.Path, acceptTimeout: FiniteDuration, locks: Locks, + outDir: os.Path, testLogEvenWhenServerIdWrong: Boolean = false ) extends ProxyStreamServer(Server.Args( daemonDir = daemonDir, @@ -31,7 +32,6 @@ abstract class MillDaemonServer[State]( bufferSize = 4 * 1024 )) { def outLock: mill.client.lock.Lock - def out: os.Path private var stateCache: State = stateCache0 @@ -71,7 +71,7 @@ abstract class MillDaemonServer[State]( MillDaemonServer.withOutLock( noBuildLock = false, noWaitForBuildLock = false, - out = out, + out = outDir, millActiveCommandMessage = "checking server mill version and java version", streams = new mill.api.daemon.SystemStreams( new PrintStream(mill.api.daemon.DummyOutputStream), diff --git a/runner/server/test/src/mill/server/ClientServerTests.scala b/runner/server/test/src/mill/server/ClientServerTests.scala index 265472b582f8..6a5acfd20e60 100644 --- a/runner/server/test/src/mill/server/ClientServerTests.scala +++ b/runner/server/test/src/mill/server/ClientServerTests.scala @@ -26,17 +26,16 @@ object ClientServerTests extends TestSuite { testLogEvenWhenServerIdWrong: Boolean, commandSleepMillis: Int = 0 ) extends MillDaemonServer[Option[Int]]( - daemonDir, - 1000.millis, - locks, - testLogEvenWhenServerIdWrong + daemonDir = daemonDir, + acceptTimeout = 1000.millis, + locks = locks, + outDir = os.temp.dir(), + testLogEvenWhenServerIdWrong = testLogEvenWhenServerIdWrong ) with Runnable { override def outLock = mill.client.lock.Lock.memory() - override def out = os.temp.dir() - def stateCache0 = None override def serverLog0(s: String) = { diff --git a/testkit/src/mill/testkit/UnitTester.scala b/testkit/src/mill/testkit/UnitTester.scala index 3b90d4771683..8b9e8b3d6dc9 100644 --- a/testkit/src/mill/testkit/UnitTester.scala +++ b/testkit/src/mill/testkit/UnitTester.scala @@ -1,12 +1,20 @@ package mill.testkit import mill.Task -import mill.api.{BuildCtx, DummyInputStream, ExecResult, Result, SystemStreams, Val} +import mill.api.{ + BuildCtx, + DummyInputStream, + Evaluator, + ExecResult, + MappedRoots, + Result, + SelectMode, + SystemStreams, + Val +} import mill.api.ExecResult.OuterStack import mill.constants.OutFiles.millChromeProfile import mill.constants.OutFiles.millProfile -import mill.api.Evaluator -import mill.api.SelectMode import mill.internal.JsonArrayLogger import mill.resolve.Resolve @@ -229,7 +237,9 @@ class UnitTester( def scoped[T](tester: UnitTester => T): T = { try { BuildCtx.workspaceRoot0.withValue(module.moduleDir) { - tester(this) + MappedRoots.withMillDefaults(outPath = outPath) { + tester(this) + } } } finally close() } diff --git a/website/blog/modules/ROOT/pages/12-direct-style-build-tool.adoc b/website/blog/modules/ROOT/pages/12-direct-style-build-tool.adoc index a58837e17ede..d16806d8ee01 100644 --- a/website/blog/modules/ROOT/pages/12-direct-style-build-tool.adoc +++ b/website/blog/modules/ROOT/pages/12-direct-style-build-tool.adoc @@ -74,7 +74,7 @@ Test foo.FooTest.testSimple finished, ... 0 failed, 0 ignored, 2 total, ... > ./mill show foo.assembly -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" > ./out/foo/assembly.dest/out.jar --text hello

hello

diff --git a/website/blog/modules/ROOT/pages/7-graal-native-executables.adoc b/website/blog/modules/ROOT/pages/7-graal-native-executables.adoc index 2eeb3eff3cf9..1e952c1aae48 100644 --- a/website/blog/modules/ROOT/pages/7-graal-native-executables.adoc +++ b/website/blog/modules/ROOT/pages/7-graal-native-executables.adoc @@ -107,7 +107,7 @@ outside of the build tool: [source,console] ---- $ ./mill show foo.assembly -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" $ out/foo/assembly.dest/out.jar --text "hello world"

hello world

@@ -159,7 +159,7 @@ Now, we can use build a native image using `foo.nativeImage`: [source,console] ---- $ ./mill show foo.nativeImage -".../out/foo/nativeImage.dest/native-executable" +"...$MILL_OUT/foo/nativeImage.dest/native-executable" $ out/foo/nativeImage.dest/native-executable --text "hello world"

hello world

@@ -229,7 +229,7 @@ _Executable Assembly_ ---- $ time ./mill show foo.assembly [1-41] [info] compiling 1 Java source... -".../out/foo/assembly.dest/out.jar" +"...$MILL_OUT/foo/assembly.dest/out.jar" ./mill show foo.assembly 0.12s user 0.06s system 21% cpu 0.818 total ---- @@ -243,7 +243,7 @@ $ time ./mill show foo.nativeImage [1-50] [2/8] Performing analysis... [****] (7.9s @ 0.77GB) ... [1-50] Finished generating 'native-executable' in 26.0s. -".../out/foo/nativeImage.dest/native-executable" +"...$MILL_OUT/foo/nativeImage.dest/native-executable" ./mill show foo.nativeImage 0.70s user 1.11s system 7% cpu 24.762 total ---- diff --git a/website/blog/modules/ROOT/pages/9-mill-faster-assembly-jars.adoc b/website/blog/modules/ROOT/pages/9-mill-faster-assembly-jars.adoc index 2818d48fb406..69f00125d926 100644 --- a/website/blog/modules/ROOT/pages/9-mill-faster-assembly-jars.adoc +++ b/website/blog/modules/ROOT/pages/9-mill-faster-assembly-jars.adoc @@ -124,7 +124,7 @@ to build an assembly that we can run using `java -jar`: [source,console] ---- > ./mill show assembly -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" Total time: 27s $ ls -lh out/assembly.dest/out.jar @@ -325,7 +325,7 @@ the code and re-build the assembly: > echo "class dummy" >> src/main/scala/foo/Foo.scala > ./mill show assembly -".../out/assembly.dest/out.jar" +"...$MILL_OUT/assembly.dest/out.jar" Total time: 1s > sbt assembly