From 8ec73a8d313ab281d554e3c17a660840d84e04f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Mon, 5 May 2025 20:50:27 +0300 Subject: [PATCH 01/18] WIP: fs-watching --- build.mill | 4 +- core/api/src/mill/api/Watchable.scala | 21 ++- libs/main/src/mill/main/MainModule.scala | 4 +- runner/package.mill | 1 + .../src/mill/runner/MillBuildBootstrap.scala | 3 +- runner/src/mill/runner/MillMain.scala | 6 +- runner/src/mill/runner/Watching.scala | 149 +++++++++++++----- 7 files changed, 135 insertions(+), 53 deletions(-) diff --git a/build.mill b/build.mill index d02b609f7b4a..eb9d291c3956 100644 --- a/build.mill +++ b/build.mill @@ -159,7 +159,9 @@ object Deps { val junitInterface = mvn"com.github.sbt:junit-interface:0.13.3" val commonsIo = mvn"commons-io:commons-io:2.18.0" val log4j2Core = mvn"org.apache.logging.log4j:log4j-core:2.24.3" - val osLib = mvn"com.lihaoyi::os-lib:0.11.5-M2" + val osLibVersion = "0.11.5-M2" + val osLib = mvn"com.lihaoyi::os-lib:${osLibVersion}" + val osLibWatch = mvn"com.lihaoyi::os-lib-watch:${osLibVersion}" val pprint = mvn"com.lihaoyi::pprint:0.9.0" val mainargs = mvn"com.lihaoyi::mainargs:0.7.6" val millModuledefsVersion = "0.11.3-M5" diff --git a/core/api/src/mill/api/Watchable.scala b/core/api/src/mill/api/Watchable.scala index c137c36eba94..14257ef5fbc0 100644 --- a/core/api/src/mill/api/Watchable.scala +++ b/core/api/src/mill/api/Watchable.scala @@ -8,6 +8,23 @@ package mill.api */ private[mill] sealed trait Watchable private[mill] object Watchable { - case class Path(p: java.nio.file.Path, quick: Boolean, signature: Int) extends Watchable - case class Value(f: () => Long, signature: Long, pretty: String) extends Watchable + /** A [[Watchable]] that is being watched via polling. */ + private[mill] sealed trait Pollable extends Watchable + + /** A [[Watchable]] that is being watched via a notification system (like inotify). */ + private[mill] sealed trait Notifiable extends Watchable + + /** + * @param p the path to watch + * @param quick if true, only watch file attributes + * @param signature the initial hash of the path contents + */ + case class Path(p: java.nio.file.Path, quick: Boolean, signature: Int) extends Notifiable + + /** + * @param f the expression to watch, returns some sort of hash + * @param signature the initial hash from the first invocation of the expression + * @param pretty human-readable name + */ + case class Value(f: () => Long, signature: Long, pretty: String) extends Pollable } diff --git a/libs/main/src/mill/main/MainModule.scala b/libs/main/src/mill/main/MainModule.scala index b9db102b23c9..a0f1427e111e 100644 --- a/libs/main/src/mill/main/MainModule.scala +++ b/libs/main/src/mill/main/MainModule.scala @@ -22,8 +22,8 @@ abstract class MainRootModule()(implicit * [[show]], [[inspect]], [[plan]], etc. */ trait MainModule extends BaseModule with MainModuleApi { - protected[mill] val watchedValues: mutable.Buffer[Watchable] = mutable.Buffer.empty[Watchable] - protected[mill] val evalWatchedValues: mutable.Buffer[Watchable] = mutable.Buffer.empty[Watchable] + protected[mill] val watchedValues: mutable.Buffer[Watchable] = mutable.Buffer.empty + protected[mill] val evalWatchedValues: mutable.Buffer[Watchable] = mutable.Buffer.empty object interp { def watchValue[T](v0: => T)(implicit fn: sourcecode.FileName, ln: sourcecode.Line): T = { val v = v0 diff --git a/runner/package.mill b/runner/package.mill index 3304e13ecaf2..d7f416bb05c9 100644 --- a/runner/package.mill +++ b/runner/package.mill @@ -18,6 +18,7 @@ object `package` extends build.MillPublishScalaModule { def mvnDeps = Seq( build.Deps.sourcecode, build.Deps.osLib, + build.Deps.osLibWatch, build.Deps.mainargs, build.Deps.upickle, build.Deps.pprint, diff --git a/runner/src/mill/runner/MillBuildBootstrap.scala b/runner/src/mill/runner/MillBuildBootstrap.scala index 1aa1f27b72c3..29c7fa7c1a0c 100644 --- a/runner/src/mill/runner/MillBuildBootstrap.scala +++ b/runner/src/mill/runner/MillBuildBootstrap.scala @@ -277,7 +277,8 @@ class MillBuildBootstrap( // look at the `moduleWatched` of one frame up (`prevOuterFrameOpt`), // and not the `moduleWatched` from the current frame (`prevFrameOpt`) val moduleWatchChanged = - prevOuterFrameOpt.exists(_.moduleWatched.exists(w => !Watching.validate(w))) + // QUESTION: is polling appropriate here? + prevOuterFrameOpt.exists(_.moduleWatched.exists(w => !Watching.validateAnyWatchable(w))) val classLoader = if (runClasspathChanged || moduleWatchChanged) { // Make sure we close the old classloader every time we create a new diff --git a/runner/src/mill/runner/MillMain.scala b/runner/src/mill/runner/MillMain.scala index bb756fc95564..9186195aecfa 100644 --- a/runner/src/mill/runner/MillMain.scala +++ b/runner/src/mill/runner/MillMain.scala @@ -352,9 +352,8 @@ object MillMain { if (config.watch.value) os.remove(out / OutFiles.millSelectiveExecution) Watching.watchLoop( ringBell = config.ringBell.value, - watch = config.watch.value, + watch = Option.when(config.watch.value)(Watching.WatchArgs(setIdle = setIdle, colors)), streams = streams, - setIdle = setIdle, evaluate = (enterKeyPressed: Boolean, prevState: Option[RunnerState]) => { adjustJvmProperties(userSpecifiedProperties, initialSystemProperties) runMillBootstrap( @@ -363,8 +362,7 @@ object MillMain { config.leftoverArgs.value, streams ) - }, - colors = colors + } ) } } diff --git a/runner/src/mill/runner/Watching.scala b/runner/src/mill/runner/Watching.scala index 48fdba512be5..0eb9392cd244 100644 --- a/runner/src/mill/runner/Watching.scala +++ b/runner/src/mill/runner/Watching.scala @@ -1,13 +1,14 @@ package mill.runner -import mill.internal.Colors -import mill.define.internal.Watchable -import mill.define.PathRef import mill.api.SystemStreams import mill.api.internal.internal +import mill.define.PathRef +import mill.define.internal.Watchable +import mill.internal.Colors import java.io.InputStream import scala.annotation.tailrec +import scala.util.Using /** * Logic around the "watch and wait" functionality in Mill: re-run on change, @@ -17,40 +18,68 @@ import scala.annotation.tailrec object Watching { case class Result[T](watched: Seq[Watchable], error: Option[String], result: T) + trait Evaluate[T] { + def apply(enterKeyPressed: Boolean, previousState: Option[T]): Result[T] + } + + case class WatchArgs( + setIdle: Boolean => Unit, + colors: Colors + ) + + /** + * @param ringBell whether to emit bells + * @param watch if false just runs once and returns + */ def watchLoop[T]( ringBell: Boolean, - watch: Boolean, + watch: Option[WatchArgs], streams: SystemStreams, - setIdle: Boolean => Unit, - evaluate: (Boolean, Option[T]) => Result[T], - colors: Colors + evaluate: Evaluate[T] ): (Boolean, T) = { - var prevState: Option[T] = None - var enterKeyPressed = false - while (true) { - val Result(watchables, errorOpt, result) = evaluate(enterKeyPressed, prevState) - prevState = Some(result) + def handleError(errorOpt: Option[String]): Unit = { errorOpt.foreach(streams.err.println) - if (ringBell) { - if (errorOpt.isEmpty) println("\u0007") - else { - println("\u0007") - Thread.sleep(250) - println("\u0007") - } - } + doRingBell(hasError = errorOpt.isDefined) + } - if (!watch) { - return (errorOpt.isEmpty, result) - } + def doRingBell(hasError: Boolean): Unit = { + if (!ringBell) return - val alreadyStale = watchables.exists(w => !validate(w)) - enterKeyPressed = false - if (!alreadyStale) { - enterKeyPressed = Watching.watchAndWait(streams, setIdle, streams.in, watchables, colors) + println("\u0007") + if (hasError) { + // If we have an error ring the bell again + Thread.sleep(250) + println("\u0007") } } - ??? + + watch match { + case None => + val Result(watchables, errorOpt, result) = + evaluate(enterKeyPressed = false, previousState = None) + handleError(errorOpt) + (errorOpt.isEmpty, result) + + case Some(watchArgs) => + var prevState: Option[T] = None + var enterKeyPressed = false + + while (true) { + val Result(watchables, errorOpt, result) = evaluate(enterKeyPressed, prevState) + prevState = Some(result) + handleError(errorOpt) + + // Do not enter watch if already stale, re-evaluate instantly. + val alreadyStale = watchables.exists(w => !validateAnyWatchable(w)) + if (alreadyStale) { + enterKeyPressed = false + } else { + enterKeyPressed = watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) + } + } + // QUESTION: this never exits? + throw new IllegalStateException("unreachable") + } } def watchAndWait( @@ -61,10 +90,14 @@ object Watching { colors: Colors ): Boolean = { setIdle(true) - val watchedPaths = watched.collect { case p: Watchable.Path => p.p } - val watchedValues = watched.size - watchedPaths.size + val (watchedPollables, watchedPaths) = watched.partitionMap { + case w: Watchable.Pollable => Left(w) + case p: Watchable.Path => Right(p) + } + val watchedValueCount = watched.size - watchedPaths.size - val watchedValueStr = if (watchedValues == 0) "" else s" and $watchedValues other values" + val watchedValueStr = + if (watchedValueCount == 0) "" else s" and $watchedValueCount other values" streams.err.println( colors.info( @@ -72,17 +105,35 @@ object Watching { ).toString ) - val enterKeyPressed = statWatchWait(watched, stdin) - setIdle(false) - enterKeyPressed + @volatile var pathChangesDetected = false + Using.resource(os.watch.watch( + watchedPaths.map(path => os.Path(path.p)), + onEvent = _ => pathChangesDetected = true, + logger = (eventType, data) => { + streams.out.println(colors.info(s"[watch] $eventType: ${pprint.apply(data)}")) + } + )) { _ => + val enterKeyPressed = + statWatchWait(watchedPollables, stdin, notifiablesChanged = () => pathChangesDetected) + setIdle(false) + enterKeyPressed + } } - // Returns `true` if enter key is pressed to re-run tasks explicitly - def statWatchWait(watched: Seq[Watchable], stdin: InputStream): Boolean = { + /** + * @param notifiablesChanged returns true if any of the notifiables have changed + * + * @return `true` if enter key is pressed to re-run tasks explicitly, false if changes in watched files occured. + */ + def statWatchWait( + watched: Seq[Watchable.Pollable], + stdin: InputStream, + notifiablesChanged: () => Boolean + ): Boolean = { val buffer = new Array[Byte](4 * 1024) @tailrec def statWatchWait0(): Boolean = { - if (watched.forall(w => validate(w))) { + if (!notifiablesChanged() && watched.forall(w => validate(w))) { if (lookForEnterKey()) { true } else { @@ -96,11 +147,13 @@ object Watching { if (stdin.available() == 0) false else stdin.read(buffer) match { case 0 | -1 => false - case n => + case bytesRead => buffer.indexOf('\n') match { case -1 => lookForEnterKey() - case i => - if (i >= n) lookForEnterKey() + case index => + // If we found the newline further than the bytes read, that means it's not from this read and thus we + // should try reading again. + if (index >= bytesRead) lookForEnterKey() else true } } @@ -109,13 +162,23 @@ object Watching { statWatchWait0() } - def validate(w: Watchable) = poll(w) == signature(w) - def poll(w: Watchable) = w match { + /** @return true if the watchable did not change. */ + inline def validate(w: Watchable.Pollable): Boolean = validateAnyWatchable(w) + + /** + * As [[validate]] but accepts any [[Watchable]] for the cases when we do not want to use a notification system. + * + * Normally you should use [[validate]] so that types would guide your implementation. + */ + def validateAnyWatchable(w: Watchable): Boolean = poll(w) == signature(w) + + def poll(w: Watchable): Long = w match { case Watchable.Path(p, quick, sig) => new PathRef(os.Path(p), quick, sig, PathRef.Revalidate.Once).recomputeSig() case Watchable.Value(f, sig, pretty) => f() } - def signature(w: Watchable) = w match { + + def signature(w: Watchable): Long = w match { case Watchable.Path(p, quick, sig) => new PathRef(os.Path(p), quick, sig, PathRef.Revalidate.Once).sig case Watchable.Value(f, sig, pretty) => sig From b4af95fdcdcc1f1325c45fee3e6f64f6731d21af Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 18:01:12 +0000 Subject: [PATCH 02/18] [autofix.ci] apply automated fixes --- core/api/src/mill/api/Watchable.scala | 1 + runner/daemon/src/mill/daemon/MillMain.scala | 5 ++++- runner/daemon/src/mill/daemon/Watching.scala | 3 ++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/api/src/mill/api/Watchable.scala b/core/api/src/mill/api/Watchable.scala index 14257ef5fbc0..b47d99bef5a3 100644 --- a/core/api/src/mill/api/Watchable.scala +++ b/core/api/src/mill/api/Watchable.scala @@ -8,6 +8,7 @@ package mill.api */ private[mill] sealed trait Watchable private[mill] object Watchable { + /** A [[Watchable]] that is being watched via polling. */ private[mill] sealed trait Pollable extends Watchable diff --git a/runner/daemon/src/mill/daemon/MillMain.scala b/runner/daemon/src/mill/daemon/MillMain.scala index 38a56c2169dd..f87ef181ef28 100644 --- a/runner/daemon/src/mill/daemon/MillMain.scala +++ b/runner/daemon/src/mill/daemon/MillMain.scala @@ -350,7 +350,10 @@ object MillMain { if (config.watch.value) os.remove(out / OutFiles.millSelectiveExecution) Watching.watchLoop( ringBell = config.ringBell.value, - watch = Option.when(config.watch.value)(Watching.WatchArgs(setIdle = setIdle, colors)), + watch = Option.when(config.watch.value)(Watching.WatchArgs( + setIdle = setIdle, + colors + )), streams = streams, evaluate = (enterKeyPressed: Boolean, prevState: Option[RunnerState]) => { adjustJvmProperties(userSpecifiedProperties, initialSystemProperties) diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index ff3345f47ff7..76145b0d7bbf 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -74,7 +74,8 @@ object Watching { if (alreadyStale) { enterKeyPressed = false } else { - enterKeyPressed = watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) + enterKeyPressed = + watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) } } // QUESTION: this never exits? From e7faaa8c462128297a3470d616b4d0203ef8b3dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 6 May 2025 10:02:47 +0300 Subject: [PATCH 03/18] fixes --- build.mill | 2 +- .../src/mill/daemon/MillBuildBootstrap.scala | 1 - runner/daemon/src/mill/daemon/Watching.scala | 39 ++++++++++++++----- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/build.mill b/build.mill index ff30a9543fdd..0b619c1ff607 100644 --- a/build.mill +++ b/build.mill @@ -159,7 +159,7 @@ object Deps { val junitInterface = mvn"com.github.sbt:junit-interface:0.13.3" val commonsIo = mvn"commons-io:commons-io:2.18.0" val log4j2Core = mvn"org.apache.logging.log4j:log4j-core:2.24.3" - val osLibVersion = "0.11.5-M2" + val osLibVersion = "0.11.5-M2-DIRTY1df4c00c" val osLib = mvn"com.lihaoyi::os-lib:${osLibVersion}" val osLibWatch = mvn"com.lihaoyi::os-lib-watch:${osLibVersion}" val pprint = mvn"com.lihaoyi::pprint:0.9.0" diff --git a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala index fd0f30204a88..5c569befc3a2 100644 --- a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala +++ b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala @@ -277,7 +277,6 @@ class MillBuildBootstrap( // look at the `moduleWatched` of one frame up (`prevOuterFrameOpt`), // and not the `moduleWatched` from the current frame (`prevFrameOpt`) val moduleWatchChanged = - // QUESTION: is polling appropriate here? prevOuterFrameOpt.exists(_.moduleWatched.exists(w => !Watching.validateAnyWatchable(w))) val classLoader = if (runClasspathChanged || moduleWatchChanged) { diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index ff3345f47ff7..5cdbac0546eb 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -64,6 +64,7 @@ object Watching { var prevState: Option[T] = None var enterKeyPressed = false + // Exits when the thread gets interruped. while (true) { val Result(watchables, errorOpt, result) = evaluate(enterKeyPressed, prevState) prevState = Some(result) @@ -74,10 +75,10 @@ object Watching { if (alreadyStale) { enterKeyPressed = false } else { - enterKeyPressed = watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) + enterKeyPressed = + watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) } } - // QUESTION: this never exits? throw new IllegalStateException("unreachable") } } @@ -90,28 +91,46 @@ object Watching { colors: Colors ): Boolean = { setIdle(true) - val (watchedPollables, watchedPaths) = watched.partitionMap { + val (watchedPollables, watchedPathsSeq) = watched.partitionMap { case w: Watchable.Pollable => Left(w) case p: Watchable.Path => Right(p) } - val watchedValueCount = watched.size - watchedPaths.size + val watchedPathsSet = watchedPathsSeq.iterator.map(p => os.Path(p.p)).toSet + val watchedValueCount = watched.size - watchedPathsSeq.size val watchedValueStr = if (watchedValueCount == 0) "" else s" and $watchedValueCount other values" streams.err.println( colors.info( - s"Watching for changes to ${watchedPaths.size} paths$watchedValueStr... (Enter to re-run, Ctrl-C to exit)" + s"Watching for changes to ${watchedPathsSeq.size} paths$watchedValueStr... (Enter to re-run, Ctrl-C to exit)" ).toString ) @volatile var pathChangesDetected = false + + // oslib watch only works with folders, so we have to watch the parent folders instead + val osLibWatchPaths = watchedPathsSet.iterator.map(p => p / "..").toSet +// mill.constants.DebugLog( +// colors.info(s"[watch:watched-paths] ${osLibWatchPaths.mkString("\n")}").toString +// ) + Using.resource(os.watch.watch( - watchedPaths.map(path => os.Path(path.p)), - onEvent = _ => pathChangesDetected = true, - logger = (eventType, data) => { - streams.out.println(colors.info(s"[watch] $eventType: ${pprint.apply(data)}")) - } + osLibWatchPaths.toSeq, + onEvent = changedPaths => { + // Make sure that the changed paths are actually the ones in our watch list and not some adjacent files in the + // same folder + val hasWatchedPath = changedPaths.exists(p => watchedPathsSet.contains(p)) +// mill.constants.DebugLog(colors.info( +// s"[watch:changed-paths] (hasWatchedPath=$hasWatchedPath) ${changedPaths.mkString("\n")}" +// ).toString) + if (hasWatchedPath) { + pathChangesDetected = true + } + }, +// logger = (eventType, data) => { +// mill.constants.DebugLog(colors.info(s"[watch] $eventType: ${pprint.apply(data)}").toString) +// } )) { _ => val enterKeyPressed = statWatchWait(watchedPollables, stdin, notifiablesChanged = () => pathChangesDetected) From 3e7c22cb22ca02ca5a7b8fb965f9359da7fa861f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Thu, 8 May 2025 12:56:34 +0300 Subject: [PATCH 04/18] Forward porting from 0.12.x branch. --- build.mill | 2 +- .../src/mill/daemon/MillCliConfig.scala | 5 + runner/daemon/src/mill/daemon/MillMain.scala | 4 +- runner/daemon/src/mill/daemon/Watching.scala | 140 +++++++++++++----- 4 files changed, 109 insertions(+), 42 deletions(-) diff --git a/build.mill b/build.mill index 0b619c1ff607..1ac11075adf8 100644 --- a/build.mill +++ b/build.mill @@ -159,7 +159,7 @@ object Deps { val junitInterface = mvn"com.github.sbt:junit-interface:0.13.3" val commonsIo = mvn"commons-io:commons-io:2.18.0" val log4j2Core = mvn"org.apache.logging.log4j:log4j-core:2.24.3" - val osLibVersion = "0.11.5-M2-DIRTY1df4c00c" + val osLibVersion = "0.11.5-M7" val osLib = mvn"com.lihaoyi::os-lib:${osLibVersion}" val osLibWatch = mvn"com.lihaoyi::os-lib-watch:${osLibVersion}" val pprint = mvn"com.lihaoyi::pprint:0.9.0" diff --git a/runner/daemon/src/mill/daemon/MillCliConfig.scala b/runner/daemon/src/mill/daemon/MillCliConfig.scala index e72d0b2d6061..01101fcdec38 100644 --- a/runner/daemon/src/mill/daemon/MillCliConfig.scala +++ b/runner/daemon/src/mill/daemon/MillCliConfig.scala @@ -97,6 +97,11 @@ case class MillCliConfig( doc = """Watch and re-run the given tasks when when their inputs change.""" ) watch: Flag = Flag(), + @arg( + name = "watch-via-fs-notify", + doc = "Use filesystem based file watching instead of polling based one (defaults to true).", + ) + watchViaFsNotify: Boolean = true, @arg( short = 's', doc = diff --git a/runner/daemon/src/mill/daemon/MillMain.scala b/runner/daemon/src/mill/daemon/MillMain.scala index 7b42bd59438e..7b8368aa86e9 100644 --- a/runner/daemon/src/mill/daemon/MillMain.scala +++ b/runner/daemon/src/mill/daemon/MillMain.scala @@ -359,7 +359,9 @@ object MillMain { ringBell = config.ringBell.value, watch = Option.when(config.watch.value)(Watching.WatchArgs( setIdle = setIdle, - colors + colors, + useNotify = config.watchViaFsNotify, + serverDir = serverDir )), streams = streams, evaluate = (enterKeyPressed: Boolean, prevState: Option[RunnerState]) => { diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index 5cdbac0546eb..201fcb48d67a 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -2,8 +2,8 @@ package mill.daemon import mill.api.SystemStreams import mill.api.internal.internal -import mill.define.PathRef import mill.define.internal.Watchable +import mill.define.{PathRef, WorkspaceRoot} import mill.internal.Colors import java.io.InputStream @@ -22,9 +22,15 @@ object Watching { def apply(enterKeyPressed: Boolean, previousState: Option[T]): Result[T] } + /** + * @param useNotify whether to use filesystem based watcher. If it is false uses polling. + * @param serverDir the directory for storing logs of the mill server + */ case class WatchArgs( setIdle: Boolean => Unit, - colors: Colors + colors: Colors, + useNotify: Boolean, + serverDir: os.Path ) /** @@ -75,8 +81,7 @@ object Watching { if (alreadyStale) { enterKeyPressed = false } else { - enterKeyPressed = - watchAndWait(streams, watchArgs.setIdle, streams.in, watchables, watchArgs.colors) + enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) } } throw new IllegalStateException("unreachable") @@ -85,12 +90,11 @@ object Watching { def watchAndWait( streams: SystemStreams, - setIdle: Boolean => Unit, stdin: InputStream, watched: Seq[Watchable], - colors: Colors + watchArgs: WatchArgs ): Boolean = { - setIdle(true) + watchArgs.setIdle(true) val (watchedPollables, watchedPathsSeq) = watched.partitionMap { case w: Watchable.Pollable => Left(w) case p: Watchable.Path => Right(p) @@ -101,42 +105,98 @@ object Watching { val watchedValueStr = if (watchedValueCount == 0) "" else s" and $watchedValueCount other values" - streams.err.println( - colors.info( - s"Watching for changes to ${watchedPathsSeq.size} paths$watchedValueStr... (Enter to re-run, Ctrl-C to exit)" + streams.err.println { + val viaFsNotify = if (watchArgs.useNotify) " (via fsnotify)" else "" + watchArgs.colors.info( + s"Watching for changes to ${watchedPathsSeq.size} paths$viaFsNotify$watchedValueStr... (Enter to re-run, Ctrl-C to exit)" ).toString - ) - - @volatile var pathChangesDetected = false - - // oslib watch only works with folders, so we have to watch the parent folders instead - val osLibWatchPaths = watchedPathsSet.iterator.map(p => p / "..").toSet -// mill.constants.DebugLog( -// colors.info(s"[watch:watched-paths] ${osLibWatchPaths.mkString("\n")}").toString -// ) - - Using.resource(os.watch.watch( - osLibWatchPaths.toSeq, - onEvent = changedPaths => { - // Make sure that the changed paths are actually the ones in our watch list and not some adjacent files in the - // same folder - val hasWatchedPath = changedPaths.exists(p => watchedPathsSet.contains(p)) -// mill.constants.DebugLog(colors.info( -// s"[watch:changed-paths] (hasWatchedPath=$hasWatchedPath) ${changedPaths.mkString("\n")}" -// ).toString) - if (hasWatchedPath) { - pathChangesDetected = true - } - }, -// logger = (eventType, data) => { -// mill.constants.DebugLog(colors.info(s"[watch] $eventType: ${pprint.apply(data)}").toString) -// } - )) { _ => - val enterKeyPressed = - statWatchWait(watchedPollables, stdin, notifiablesChanged = () => pathChangesDetected) - setIdle(false) + } + + def doWatch(notifiablesChanged: () => Boolean) = { + val enterKeyPressed = statWatchWait(watchedPollables, stdin, notifiablesChanged) + watchArgs.setIdle(false) enterKeyPressed } + + def doWatchPolling() = + doWatch(notifiablesChanged = () => watchedPathsSeq.exists(p => !validateAnyWatchable(p))) + + def doWatchFsNotify() = { + Using.resource(os.write.outputStream(watchArgs.serverDir / "fsNotifyWatchLog")) { watchLog => + def writeToWatchLog(s: String): Unit = { + watchLog.write(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + watchLog.write('\n') + } + + @volatile var pathChangesDetected = false + + // oslib watch only works with folders, so we have to watch the parent folders instead + + writeToWatchLog( + s"[watched-paths:unfiltered] ${watchedPathsSet.toSeq.sorted.mkString("\n")}" + ) + + val workspaceRoot = WorkspaceRoot.workspaceRoot + + /** Paths that are descendants of [[workspaceRoot]]. */ + val pathsUnderWorkspaceRoot = watchedPathsSet.filter { path => + val isUnderWorkspaceRoot = path.startsWith(workspaceRoot) + if (!isUnderWorkspaceRoot) { + streams.err.println(watchArgs.colors.error( + s"Watched path $path is outside workspace root $workspaceRoot, this is unsupported." + ).toString()) + } + + isUnderWorkspaceRoot + } + + // If I have 'root/a/b/c' + // + // Then I want to watch: + // root/a/b/c + // root/a/b + // root/a + // root + val filterPaths = pathsUnderWorkspaceRoot.flatMap { path => + path.relativeTo(workspaceRoot).segments.inits.map(segments => workspaceRoot / segments) + } + writeToWatchLog(s"[watched-paths:filtered] ${filterPaths.toSeq.sorted.mkString("\n")}") + + Using.resource(os.watch.watch( + // Just watch the root folder + Seq(workspaceRoot), + filter = path => { + val shouldBeWatched = + filterPaths.contains(path) || watchedPathsSet.exists(watchedPath => + path.startsWith(watchedPath) + ) + writeToWatchLog(s"[filter] (shouldBeWatched=$shouldBeWatched) $path") + shouldBeWatched + }, + onEvent = changedPaths => { + // Make sure that the changed paths are actually the ones in our watch list and not some adjacent files in the + // same folder + val hasWatchedPath = + changedPaths.exists(p => + watchedPathsSet.exists(watchedPath => p.startsWith(watchedPath)) + ) + writeToWatchLog( + s"[changed-paths] (hasWatchedPath=$hasWatchedPath) ${changedPaths.mkString("\n")}" + ) + if (hasWatchedPath) { + pathChangesDetected = true + } + }, + logger = (eventType, data) => + writeToWatchLog(s"[watch:event] $eventType: ${pprint.apply(data).plainText}") + )) { _ => + doWatch(notifiablesChanged = () => pathChangesDetected) + } + } + } + + if (watchArgs.useNotify) doWatchFsNotify() + else doWatchPolling() } /** From 51cbab535be36ab4ad3ecc996ba23ed0b3d37671 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Thu, 8 May 2025 09:58:36 +0000 Subject: [PATCH 05/18] [autofix.ci] apply automated fixes --- runner/daemon/src/mill/daemon/MillCliConfig.scala | 2 +- runner/daemon/src/mill/daemon/MillMain.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/runner/daemon/src/mill/daemon/MillCliConfig.scala b/runner/daemon/src/mill/daemon/MillCliConfig.scala index 01101fcdec38..38ee92167e66 100644 --- a/runner/daemon/src/mill/daemon/MillCliConfig.scala +++ b/runner/daemon/src/mill/daemon/MillCliConfig.scala @@ -99,7 +99,7 @@ case class MillCliConfig( watch: Flag = Flag(), @arg( name = "watch-via-fs-notify", - doc = "Use filesystem based file watching instead of polling based one (defaults to true).", + doc = "Use filesystem based file watching instead of polling based one (defaults to true)." ) watchViaFsNotify: Boolean = true, @arg( diff --git a/runner/daemon/src/mill/daemon/MillMain.scala b/runner/daemon/src/mill/daemon/MillMain.scala index 7b8368aa86e9..27451f188066 100644 --- a/runner/daemon/src/mill/daemon/MillMain.scala +++ b/runner/daemon/src/mill/daemon/MillMain.scala @@ -359,8 +359,8 @@ object MillMain { ringBell = config.ringBell.value, watch = Option.when(config.watch.value)(Watching.WatchArgs( setIdle = setIdle, - colors, - useNotify = config.watchViaFsNotify, + colors, + useNotify = config.watchViaFsNotify, serverDir = serverDir )), streams = streams, From c331c4605714c427056fd2a54981ce6e4ee6a5a0 Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Fri, 9 May 2025 08:29:39 +0000 Subject: [PATCH 06/18] [autofix.ci] apply automated fixes --- runner/daemon/src/mill/daemon/MillMain.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/runner/daemon/src/mill/daemon/MillMain.scala b/runner/daemon/src/mill/daemon/MillMain.scala index e8f702f7b21a..c7a0c8347703 100644 --- a/runner/daemon/src/mill/daemon/MillMain.scala +++ b/runner/daemon/src/mill/daemon/MillMain.scala @@ -350,13 +350,13 @@ object MillMain { if (config.watch.value) os.remove(out / OutFiles.millSelectiveExecution) Watching.watchLoop( ringBell = config.ringBell.value, - watch = Option.when(config.watch.value)(Watching.WatchArgs( - setIdle = setIdle, - colors, - useNotify = config.watchViaFsNotify, - serverDir = serverDir - )), - streams = streams, + watch = Option.when(config.watch.value)(Watching.WatchArgs( + setIdle = setIdle, + colors, + useNotify = config.watchViaFsNotify, + serverDir = serverDir + )), + streams = streams, evaluate = (enterKeyPressed: Boolean, prevState: Option[RunnerState]) => { adjustJvmProperties(userSpecifiedProperties, initialSystemProperties) runMillBootstrap( From 27260726eebc788e636380ce627f89db603a8ab7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Thu, 8 May 2025 17:00:23 +0300 Subject: [PATCH 07/18] Catch if `writeToWatchLog` tries to write to a closed file. --- runner/daemon/src/mill/daemon/Watching.scala | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index 201fcb48d67a..01b3312add3d 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -7,6 +7,7 @@ import mill.define.{PathRef, WorkspaceRoot} import mill.internal.Colors import java.io.InputStream +import java.nio.channels.ClosedChannelException import scala.annotation.tailrec import scala.util.Using @@ -124,8 +125,12 @@ object Watching { def doWatchFsNotify() = { Using.resource(os.write.outputStream(watchArgs.serverDir / "fsNotifyWatchLog")) { watchLog => def writeToWatchLog(s: String): Unit = { - watchLog.write(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)) - watchLog.write('\n') + try { + watchLog.write(s.getBytes(java.nio.charset.StandardCharsets.UTF_8)) + watchLog.write('\n') + } catch { + case _: ClosedChannelException => /* do nothing, the file is already closed */ + } } @volatile var pathChangesDetected = false From 9021c1902c9692d1141a7d70a0853e6a29a2fa9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 08:11:41 +0300 Subject: [PATCH 08/18] CI test debug --- .github/workflows/run-tests.yml | 674 ++++++++++++++++++-------------- 1 file changed, 372 insertions(+), 302 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 1d24ed632777..92186485c45b 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,322 +1,392 @@ # Uncommment this to replace the rest of the file when you want to debug stuff in CI -#name: Run Debug -# -#on: -# push: -# pull_request: -# workflow_dispatch: -# -#jobs: -# debug: -# runs-on: ubuntu-latest -## runs-on: windows-latest -# steps: -# - uses: actions/checkout@v4 -# with: { fetch-depth: 1 } -# -# - run: "echo temurin:11 > .mill-jvm-version" -# -# - uses: sbt/setup-sbt@v1 -# -# -# - run: ./mill 'integration.migrating[init].local.server.testOnly' mill.integration.MillInitSbtGatlingTests -# +name: Run Debug +on: + push: + pull_request: + workflow_dispatch: + +jobs: + debug: + runs-on: ubuntu-latest +# runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 1 } + - run: "echo temurin:11 > .mill-jvm-version" -# We run full CI on push builds to main and on all pull requests -# -# To maximize bug-catching changes while keeping CI times reasonable, we run -# all tests on Linux, scattered between Java 11/17, except for one job run -# on MacOS instead and a subset of jobs also run on windows + - uses: sbt/setup-sbt@v1 -name: Run Tests + - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" mill.integration.WatchSourceTests.sources.show -on: - push: - branches-ignore: - - '**-patch-**' - pull_request: - types: - - opened - - reopened - - synchronize - - ready_for_review - workflow_dispatch: + debug-concurrent: + runs-on: ubuntu-latest +# runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 1 } -# cancel older runs of a pull request; -# this will not cancel anything for normal git pushes -concurrency: - # * For runs on other repos, always use the `ref_name` so each branch only can have one concurrent run - # * For runs on `com-lihaoyi/mill`, use `head_ref` to allow one concurrent run per PR, but `run_id` to - # allow multiple concurrent runs in master - group: cancel-old-pr-runs-${{ github.workflow }}-${{ (github.repository != 'com-lihaoyi/mill' && github.ref_name) || (github.head_ref || github.run_id) }} - cancel-in-progress: true + - run: "echo temurin:11 > .mill-jvm-version" -jobs: - # Jobs are listed in rough order of priority: if multiple jobs fail, the first job - # in the list should be the one that's most worth looking into - build-linux: - if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) - uses: ./.github/workflows/pre-build.yml - with: - os: ubuntu-latest - shell: bash - - build-windows: - if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) - uses: ./.github/workflows/pre-build.yml - with: - os: windows-latest - shell: powershell - - test-docs: - if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) + - uses: sbt/setup-sbt@v1 + + + - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" + + debug-very-concurrent: + runs-on: ubuntu-latest +# runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 1 } + + - run: "echo temurin:11 > .mill-jvm-version" + + - uses: sbt/setup-sbt@v1 + + + - run: ./mill "integration.invalidation[__].packaged.server.testForked" + + debug2: runs-on: ubuntu-latest +# runs-on: windows-latest steps: - uses: actions/checkout@v4 with: { fetch-depth: 1 } - - run: ./mill -i website.fastPages + website.checkBrokenLinks - - cross-plat: - if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - # Run these with Mill native launcher as a smoketest - include: - - os: ubuntu-24.04-arm - millargs: "'example.thirdparty[fansi].native.server'" - java-version: 17 - - - os: macos-latest - millargs: "'example.thirdparty[acyclic].native.server'" - java-version: 11 - - - os: macos-13 - millargs: "'example.thirdparty[jimfs].native.server'" - java-version: 11 + - run: "echo temurin:11 > .mill-jvm-version" + + - uses: sbt/setup-sbt@v1 + + + - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" mill.integration.WatchSourceTests.sources.noshow + + debug2-concurrent: + runs-on: ubuntu-latest +# runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + with: { fetch-depth: 1 } + + - run: "echo temurin:11 > .mill-jvm-version" + + - uses: sbt/setup-sbt@v1 + + + - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" + + debug2-very-concurrent: + runs-on: ubuntu-latest +# runs-on: windows-latest steps: - uses: actions/checkout@v4 with: { fetch-depth: 1 } - - uses: ./.github/actions/pre-build-setup - with: - os: ${{ matrix.os }} - java-version: ${{ matrix.java-version }} - shell: bash - - - uses: ./.github/actions/post-build-setup - with: - java-version: ${{ matrix.java-version }} - os: ${{ matrix.os }} - - - uses: ./.github/actions/post-build-selective - with: - millargs: ${{ matrix.millargs }} - coursierarchive: "/tmp" - shell: bash - - linux: - needs: build-linux - strategy: - fail-fast: false - matrix: - - include: - # For most tests, run them arbitrarily on Java 11 or Java 17 on Linux, and - # on the opposite version on Windows below, so we get decent coverage of - # each test on each Java version and each operating system - # We also try to group tests together to manually balance out the runtimes of each jobs - - java-version: 17 - millargs: "'{core,testkit,runner}.__.test'" - install-android-sdk: false - install-sbt: true - - - java-version: 17 - millargs: "'libs.{main,scalalib,testrunner}.__.test'" - install-android-sdk: false - install-sbt: true - - - java-version: 11 - millargs: "'libs.{scalajslib,scalanativelib,kotlinlib,pythonlib,javascriptlib}.__.test'" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "contrib.__.test" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "example.javalib.__.local.server" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "example.kotlinlib.__.local.server" - install-android-sdk: false - install-sbt: false - - # Run this one using `.native` as a smoketest. Also make sure the java-version - # is the same as that used in the `build-linux` job to avoid diverging code - # hashes (https://github.com/com-lihaoyi/mill/pull/4410) - - java-version: 11 - millargs: "example.scalalib.__.native.server" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "'example.androidlib.__.local.server'" - install-android-sdk: true - install-sbt: false - - - java-version: 17 - millargs: "'example.thirdparty[androidtodo].local.server'" - install-android-sdk: true - install-sbt: false - - - java-version: 17 - millargs: "'example.thirdparty[android-endless-tunnel].local.server'" - install-android-sdk: true - install-sbt: false - - - java-version: 17 - millargs: "'{example,integration}.migrating.__.local.server'" - install-android-sdk: false - install-sbt: true - - - java-version: 17 - millargs: "'example.{pythonlib,javascriptlib}.__.local.server'" - install-android-sdk: false - install-sbt: false - - - java-version: 11 - millargs: "'example.thirdparty[{mockito,commons-io}].local.server'" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "'example.thirdparty[{netty,gatling}].local.server'" - install-android-sdk: false - install-sbt: false - - - java-version: '17' - millargs: "'example.thirdparty[arrow].local.server'" - install-android-sdk: false - install-sbt: false - - - java-version: 11 - millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server'" - install-android-sdk: false - install-sbt: false - - - java-version: 11 - millargs: "'integration.{failure,feature,ide}.__.packaged.server'" - install-android-sdk: false - install-sbt: false - - # run this specifically in `native` mode to make sure our non-JVM native image - # launcher is able to bootstrap everything necessary without a JVM installed - - java-version: 17 - millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" - install-android-sdk: false - - # These invalidation tests need to be exercised in both execution modes - # to make sure they work with and without -i/--no-server being passed - - java-version: 17 - millargs: "'integration.invalidation.__.packaged.fork'" - install-android-sdk: false - install-sbt: false - - - java-version: 17 - millargs: "'integration.invalidation.__.packaged.server'" - install-android-sdk: false - install-sbt: false - - uses: ./.github/workflows/post-build-selective.yml - with: - install-android-sdk: ${{ matrix.install-android-sdk }} - install-sbt: ${{ matrix.install-sbt || false }} - java-version: ${{ matrix.java-version }} - millargs: ${{ matrix.millargs }} - shell: bash - - windows: - needs: build-windows - strategy: - fail-fast: false - matrix: - include: - # just run a subset of examples/ on Windows, because for some reason running - # the whole suite can take hours on windows v.s. half an hour on linux - # - # * One job unit tests, - # * One job each for local/packaged/native tests - # * At least one job for each of fork/server tests, and example/integration tests - - java-version: 11 - millargs: '"libs.{main,scalalib}.__.test"' - install-sbt: false - - - java-version: 11 - millargs: '"example.scalalib.{basic,publishing}.__.local.fork"' - install-sbt: false - - - java-version: 11 - millargs: '"example.migrating.{scalalib,javalib}.__.local.fork"' - install-sbt: true - - - java-version: 17 - millargs: "'integration.{feature,failure}.__.packaged.fork'" - install-sbt: false - - - java-version: 11 # Run this with Mill native launcher as a smoketest - millargs: "'integration.invalidation.__.native.server'" - install-sbt: false - - - java-version: 17 - millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" - - uses: ./.github/workflows/post-build-selective.yml - with: - os: windows-latest - java-version: ${{ matrix.java-version }} - millargs: ${{ matrix.millargs }} - # Provide a shorter coursier archive folder to avoid hitting path-length bugs when - # running the graal native image binary on windows - coursierarchive: "C:/coursier-arc" - shell: powershell - install-sbt: ${{ matrix.install-sbt || false }} - - itest: - needs: build-linux - strategy: - fail-fast: false - matrix: - include: - # bootstrap tests - - java-version: 11 # Have one job on oldest JVM - buildcmd: ci/test-dist-run.sh && ci/test-install-local.sh - - java-version: 17 # Have one job on default JVM - buildcmd: ci/test-mill-bootstrap.sh - - uses: ./.github/workflows/post-build-raw.yml - with: - java-version: ${{ matrix.java-version }} - buildcmd: ${{ matrix.buildcmd }} - - # Scalafmt, Mima, and Scalafix job runs last because it's the least important: - # usually just an automated or mechanical manual fix to do before merging - lint-autofix: - needs: build-linux - uses: ./.github/workflows/post-build-raw.yml - with: - java-version: '17' - buildcmd: | - set -eux - ./mill -i mill.scalalib.scalafmt.ScalafmtModule/scalafmt --check + __.fix --check + mill.javalib.palantirformat.PalantirFormatModule/ --check + mill.kotlinlib.ktlint.KtlintModule/checkFormatAll + - run: "echo temurin:11 > .mill-jvm-version" + + - uses: sbt/setup-sbt@v1 + + + - run: ./mill -i "integration.invalidation[__].packaged.fork.testForked" + + +# +# +## We run full CI on push builds to main and on all pull requests +## +## To maximize bug-catching changes while keeping CI times reasonable, we run +## all tests on Linux, scattered between Java 11/17, except for one job run +## on MacOS instead and a subset of jobs also run on windows +# +# +#name: Run Tests +# +#on: +# push: +# branches-ignore: +# - '**-patch-**' +# pull_request: +# types: +# - opened +# - reopened +# - synchronize +# - ready_for_review +# workflow_dispatch: +# +## cancel older runs of a pull request; +## this will not cancel anything for normal git pushes +#concurrency: +# # * For runs on other repos, always use the `ref_name` so each branch only can have one concurrent run +# # * For runs on `com-lihaoyi/mill`, use `head_ref` to allow one concurrent run per PR, but `run_id` to +# # allow multiple concurrent runs in master +# group: cancel-old-pr-runs-${{ github.workflow }}-${{ (github.repository != 'com-lihaoyi/mill' && github.ref_name) || (github.head_ref || github.run_id) }} +# cancel-in-progress: true +# +#jobs: +# # Jobs are listed in rough order of priority: if multiple jobs fail, the first job +# # in the list should be the one that's most worth looking into +# build-linux: +# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) +# uses: ./.github/workflows/pre-build.yml +# with: +# os: ubuntu-latest +# shell: bash +# +# build-windows: +# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) +# uses: ./.github/workflows/pre-build.yml +# with: +# os: windows-latest +# shell: powershell +# +# test-docs: +# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: { fetch-depth: 1 } +# +# - run: ./mill -i website.fastPages + website.checkBrokenLinks +# +# cross-plat: +# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) +# runs-on: ${{ matrix.os }} +# strategy: +# fail-fast: false +# matrix: +# # Run these with Mill native launcher as a smoketest +# include: +# - os: ubuntu-24.04-arm +# millargs: "'example.thirdparty[fansi].native.server'" +# java-version: 17 +# +# - os: macos-latest +# millargs: "'example.thirdparty[acyclic].native.server'" +# java-version: 11 +# +# - os: macos-13 +# millargs: "'example.thirdparty[jimfs].native.server'" +# java-version: 11 +# steps: +# - uses: actions/checkout@v4 +# with: { fetch-depth: 1 } +# +# - uses: ./.github/actions/pre-build-setup +# with: +# os: ${{ matrix.os }} +# java-version: ${{ matrix.java-version }} +# shell: bash +# +# - uses: ./.github/actions/post-build-setup +# with: +# java-version: ${{ matrix.java-version }} +# os: ${{ matrix.os }} +# +# - uses: ./.github/actions/post-build-selective +# with: +# millargs: ${{ matrix.millargs }} +# coursierarchive: "/tmp" +# shell: bash +# +# linux: +# needs: build-linux +# strategy: +# fail-fast: false +# matrix: +# +# include: +# # For most tests, run them arbitrarily on Java 11 or Java 17 on Linux, and +# # on the opposite version on Windows below, so we get decent coverage of +# # each test on each Java version and each operating system +# # We also try to group tests together to manually balance out the runtimes of each jobs +# - java-version: 17 +# millargs: "'{core,testkit,runner}.__.test'" +# install-android-sdk: false +# install-sbt: true +# +# - java-version: 17 +# millargs: "'libs.{main,scalalib,testrunner}.__.test'" +# install-android-sdk: false +# install-sbt: true +# +# - java-version: 11 +# millargs: "'libs.{scalajslib,scalanativelib,kotlinlib,pythonlib,javascriptlib}.__.test'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "contrib.__.test" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "example.javalib.__.local.server" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "example.kotlinlib.__.local.server" +# install-android-sdk: false +# install-sbt: false +# +# # Run this one using `.native` as a smoketest. Also make sure the java-version +# # is the same as that used in the `build-linux` job to avoid diverging code +# # hashes (https://github.com/com-lihaoyi/mill/pull/4410) +# - java-version: 11 +# millargs: "example.scalalib.__.native.server" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "'example.androidlib.__.local.server'" +# install-android-sdk: true +# install-sbt: false +# +# - java-version: 17 +# millargs: "'example.thirdparty[androidtodo].local.server'" +# install-android-sdk: true +# install-sbt: false +# +# - java-version: 17 +# millargs: "'example.thirdparty[android-endless-tunnel].local.server'" +# install-android-sdk: true +# install-sbt: false +# +# - java-version: 17 +# millargs: "'{example,integration}.migrating.__.local.server'" +# install-android-sdk: false +# install-sbt: true +# +# - java-version: 17 +# millargs: "'example.{pythonlib,javascriptlib}.__.local.server'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 11 +# millargs: "'example.thirdparty[{mockito,commons-io}].local.server'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "'example.thirdparty[{netty,gatling}].local.server'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: '17' +# millargs: "'example.thirdparty[arrow].local.server'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 11 +# millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 11 +# millargs: "'integration.{failure,feature,ide}.__.packaged.server'" +# install-android-sdk: false +# install-sbt: false +# +# # run this specifically in `native` mode to make sure our non-JVM native image +# # launcher is able to bootstrap everything necessary without a JVM installed +# - java-version: 17 +# millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" +# install-android-sdk: false +# +# # These invalidation tests need to be exercised in both execution modes +# # to make sure they work with and without -i/--no-server being passed +# - java-version: 17 +# millargs: "'integration.invalidation.__.packaged.fork'" +# install-android-sdk: false +# install-sbt: false +# +# - java-version: 17 +# millargs: "'integration.invalidation.__.packaged.server'" +# install-android-sdk: false +# install-sbt: false +# +# uses: ./.github/workflows/post-build-selective.yml +# with: +# install-android-sdk: ${{ matrix.install-android-sdk }} +# install-sbt: ${{ matrix.install-sbt || false }} +# java-version: ${{ matrix.java-version }} +# millargs: ${{ matrix.millargs }} +# shell: bash +# +# windows: +# needs: build-windows +# strategy: +# fail-fast: false +# matrix: +# include: +# # just run a subset of examples/ on Windows, because for some reason running +# # the whole suite can take hours on windows v.s. half an hour on linux +# # +# # * One job unit tests, +# # * One job each for local/packaged/native tests +# # * At least one job for each of fork/server tests, and example/integration tests +# - java-version: 11 +# millargs: '"libs.{main,scalalib}.__.test"' +# install-sbt: false +# +# - java-version: 11 +# millargs: '"example.scalalib.{basic,publishing}.__.local.fork"' +# install-sbt: false +# +# - java-version: 11 +# millargs: '"example.migrating.{scalalib,javalib}.__.local.fork"' +# install-sbt: true +# +# - java-version: 17 +# millargs: "'integration.{feature,failure}.__.packaged.fork'" +# install-sbt: false +# +# - java-version: 11 # Run this with Mill native launcher as a smoketest +# millargs: "'integration.invalidation.__.native.server'" +# install-sbt: false +# +# - java-version: 17 +# millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" +# +# uses: ./.github/workflows/post-build-selective.yml +# with: +# os: windows-latest +# java-version: ${{ matrix.java-version }} +# millargs: ${{ matrix.millargs }} +# # Provide a shorter coursier archive folder to avoid hitting path-length bugs when +# # running the graal native image binary on windows +# coursierarchive: "C:/coursier-arc" +# shell: powershell +# install-sbt: ${{ matrix.install-sbt || false }} +# +# itest: +# needs: build-linux +# strategy: +# fail-fast: false +# matrix: +# include: +# # bootstrap tests +# - java-version: 11 # Have one job on oldest JVM +# buildcmd: ci/test-dist-run.sh && ci/test-install-local.sh +# - java-version: 17 # Have one job on default JVM +# buildcmd: ci/test-mill-bootstrap.sh +# +# uses: ./.github/workflows/post-build-raw.yml +# with: +# java-version: ${{ matrix.java-version }} +# buildcmd: ${{ matrix.buildcmd }} +# +# # Scalafmt, Mima, and Scalafix job runs last because it's the least important: +# # usually just an automated or mechanical manual fix to do before merging +# lint-autofix: +# needs: build-linux +# uses: ./.github/workflows/post-build-raw.yml +# with: +# java-version: '17' +# buildcmd: | +# set -eux +# ./mill -i mill.scalalib.scalafmt.ScalafmtModule/scalafmt --check + __.fix --check + mill.javalib.palantirformat.PalantirFormatModule/ --check + mill.kotlinlib.ktlint.KtlintModule/checkFormatAll From 2dcf557cc5044f42a0909298b4356d1f8f92089c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 08:25:54 +0300 Subject: [PATCH 09/18] CI test debug --- .github/workflows/run-tests.yml | 43 ++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 92186485c45b..b5e434a84c4f 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -23,6 +23,13 @@ jobs: - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" mill.integration.WatchSourceTests.sources.show + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug-watch-log-artifact + include-hidden-files: true + debug-concurrent: runs-on: ubuntu-latest # runs-on: windows-latest @@ -37,6 +44,13 @@ jobs: - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug-concurrent-watch-log-artifact + include-hidden-files: true + debug-very-concurrent: runs-on: ubuntu-latest # runs-on: windows-latest @@ -51,6 +65,13 @@ jobs: - run: ./mill "integration.invalidation[__].packaged.server.testForked" + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug-very-concurrent-watch-log-artifact + include-hidden-files: true + debug2: runs-on: ubuntu-latest # runs-on: windows-latest @@ -62,9 +83,16 @@ jobs: - uses: sbt/setup-sbt@v1 - - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" mill.integration.WatchSourceTests.sources.noshow + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug2-watch-log-artifact + include-hidden-files: true + + debug2-concurrent: runs-on: ubuntu-latest # runs-on: windows-latest @@ -79,6 +107,13 @@ jobs: - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug2-concurrent-watch-log-artifact + include-hidden-files: true + debug2-very-concurrent: runs-on: ubuntu-latest # runs-on: windows-latest @@ -93,6 +128,12 @@ jobs: - run: ./mill -i "integration.invalidation[__].packaged.fork.testForked" + - uses: actions/upload-artifact@v4 + if: always() # always run even if the previous step fails + with: + path: out/**/fsNotifyWatchLog + name: debug2-very-concurrent-watch-log-artifact + include-hidden-files: true # # From dfbba1f5b76a07df3500e293c752b6171df90cb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 08:29:37 +0300 Subject: [PATCH 10/18] debug watch tests with sleeps --- .../watch-source-input/src/WatchSourceInputTests.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala index 15d6efb974dd..8b035736318f 100644 --- a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala +++ b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala @@ -83,6 +83,7 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents initial-foo1 initial-foo2 Running qux bar contents initial-bar" ) + Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "foo1.txt", "edited-foo1") awaitCompletionMarker(tester, "quxRan1") expectedErr.append( @@ -92,7 +93,8 @@ object WatchSourceTests extends WatchTests { expectedShows.append( "Running qux foo contents edited-foo1 initial-foo2 Running qux bar contents initial-bar" ) - + + Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "foo2.txt", "edited-foo2") awaitCompletionMarker(tester, "quxRan2") expectedErr.append( @@ -103,6 +105,7 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents initial-bar" ) + Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "bar.txt", "edited-bar") awaitCompletionMarker(tester, "quxRan3") expectedErr.append( @@ -113,6 +116,7 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents edited-bar" ) + Thread.sleep(1000) // Wait for the watching to begin os.write.append(workspacePath / "build.mill", "\ndef unrelated = true") awaitCompletionMarker(tester, "initialized1") expectedOut.append( @@ -124,6 +128,7 @@ object WatchSourceTests extends WatchTests { ) if (show) expectedOut.append("{}") + Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "watchValue.txt", "exit") awaitCompletionMarker(tester, "initialized2") expectedOut.append("Setting up build.mill") From 1b797e97c2b25482677bf1bf0d00324938b73e87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 09:45:04 +0300 Subject: [PATCH 11/18] Move stale checking after we start the watch --- .../src/WatchSourceInputTests.scala | 7 +----- runner/daemon/src/mill/daemon/Watching.scala | 22 +++++++++---------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala index 8b035736318f..15d6efb974dd 100644 --- a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala +++ b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala @@ -83,7 +83,6 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents initial-foo1 initial-foo2 Running qux bar contents initial-bar" ) - Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "foo1.txt", "edited-foo1") awaitCompletionMarker(tester, "quxRan1") expectedErr.append( @@ -93,8 +92,7 @@ object WatchSourceTests extends WatchTests { expectedShows.append( "Running qux foo contents edited-foo1 initial-foo2 Running qux bar contents initial-bar" ) - - Thread.sleep(1000) // Wait for the watching to begin + os.write.over(workspacePath / "foo2.txt", "edited-foo2") awaitCompletionMarker(tester, "quxRan2") expectedErr.append( @@ -105,7 +103,6 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents initial-bar" ) - Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "bar.txt", "edited-bar") awaitCompletionMarker(tester, "quxRan3") expectedErr.append( @@ -116,7 +113,6 @@ object WatchSourceTests extends WatchTests { "Running qux foo contents edited-foo1 edited-foo2 Running qux bar contents edited-bar" ) - Thread.sleep(1000) // Wait for the watching to begin os.write.append(workspacePath / "build.mill", "\ndef unrelated = true") awaitCompletionMarker(tester, "initialized1") expectedOut.append( @@ -128,7 +124,6 @@ object WatchSourceTests extends WatchTests { ) if (show) expectedOut.append("{}") - Thread.sleep(1000) // Wait for the watching to begin os.write.over(workspacePath / "watchValue.txt", "exit") awaitCompletionMarker(tester, "initialized2") expectedOut.append("Setting up build.mill") diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index 01b3312add3d..b9f2bf5e4ac9 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -62,8 +62,7 @@ object Watching { watch match { case None => - val Result(watchables, errorOpt, result) = - evaluate(enterKeyPressed = false, previousState = None) + val Result(watchables, errorOpt, result) = evaluate(enterKeyPressed = false, previousState = None) handleError(errorOpt) (errorOpt.isEmpty, result) @@ -77,19 +76,13 @@ object Watching { prevState = Some(result) handleError(errorOpt) - // Do not enter watch if already stale, re-evaluate instantly. - val alreadyStale = watchables.exists(w => !validateAnyWatchable(w)) - if (alreadyStale) { - enterKeyPressed = false - } else { - enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) - } + enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) } throw new IllegalStateException("unreachable") } } - def watchAndWait( + private def watchAndWait( streams: SystemStreams, stdin: InputStream, watched: Seq[Watchable], @@ -195,7 +188,14 @@ object Watching { logger = (eventType, data) => writeToWatchLog(s"[watch:event] $eventType: ${pprint.apply(data).plainText}") )) { _ => - doWatch(notifiablesChanged = () => pathChangesDetected) + // If already stale, re-evaluate instantly. + // + // We need to do this to prevent any changes from slipping through the gap between the last evaluation and + // starting the watch. + val alreadyStale = watched.exists(w => !validateAnyWatchable(w)) + + if (alreadyStale) false + else doWatch(notifiablesChanged = () => pathChangesDetected) } } } From ad139ab7a218eb37b0f4732a58056eed5729264d Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 06:47:10 +0000 Subject: [PATCH 12/18] [autofix.ci] apply automated fixes --- runner/daemon/src/mill/daemon/Watching.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index b9f2bf5e4ac9..eab4ef5403d2 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -62,7 +62,8 @@ object Watching { watch match { case None => - val Result(watchables, errorOpt, result) = evaluate(enterKeyPressed = false, previousState = None) + val Result(watchables, errorOpt, result) = + evaluate(enterKeyPressed = false, previousState = None) handleError(errorOpt) (errorOpt.isEmpty, result) From 69390dd23ba850c8b5e27d3898d598199af451d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 09:50:34 +0300 Subject: [PATCH 13/18] Refactor setIdle --- runner/daemon/src/mill/daemon/Watching.scala | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index b9f2bf5e4ac9..fde2872ac78d 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -76,7 +76,13 @@ object Watching { prevState = Some(result) handleError(errorOpt) - enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) + try { + watchArgs.setIdle(true) + enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) + } + finally { + watchArgs.setIdle(false) + } } throw new IllegalStateException("unreachable") } @@ -88,7 +94,6 @@ object Watching { watched: Seq[Watchable], watchArgs: WatchArgs ): Boolean = { - watchArgs.setIdle(true) val (watchedPollables, watchedPathsSeq) = watched.partitionMap { case w: Watchable.Pollable => Left(w) case p: Watchable.Path => Right(p) @@ -108,7 +113,6 @@ object Watching { def doWatch(notifiablesChanged: () => Boolean) = { val enterKeyPressed = statWatchWait(watchedPollables, stdin, notifiablesChanged) - watchArgs.setIdle(false) enterKeyPressed } From 2a84a72a20d864f3ae20ac02e95accd0591742c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 09:58:18 +0300 Subject: [PATCH 14/18] Make tests more robust. --- .../watch-source-input/src/WatchSourceInputTests.scala | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala index 15d6efb974dd..345e7c63e548 100644 --- a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala +++ b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala @@ -44,12 +44,7 @@ trait WatchTests extends UtestIntegrationTestSuite { val expectedShows0 = mutable.Buffer.empty[String] val res = f(expectedOut, expectedErr, expectedShows0) val (shows, out) = res.out.linesIterator.toVector.partition(_.startsWith("\"")) - val err = res.err.linesIterator.toVector - .filter(!_.contains("Compiling compiler interface...")) - .filter(!_.contains("Watching for changes")) - .filter(!_.contains("[info] compiling")) - .filter(!_.contains("[info] done compiling")) - .filter(!_.contains("mill-server/ exitCode file not found")) + val err = res.err.linesIterator.toVector.filter(s => s.startsWith("Setting up ") || s.startsWith("Running ")) assert(out == expectedOut) From a4de28112dc3ac6e588964b7934e755a4003740b Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Tue, 13 May 2025 07:00:42 +0000 Subject: [PATCH 15/18] [autofix.ci] apply automated fixes --- .../watch-source-input/src/WatchSourceInputTests.scala | 4 +++- runner/daemon/src/mill/daemon/Watching.scala | 3 +-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala index 345e7c63e548..610d41c354a4 100644 --- a/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala +++ b/integration/invalidation/watch-source-input/src/WatchSourceInputTests.scala @@ -44,7 +44,9 @@ trait WatchTests extends UtestIntegrationTestSuite { val expectedShows0 = mutable.Buffer.empty[String] val res = f(expectedOut, expectedErr, expectedShows0) val (shows, out) = res.out.linesIterator.toVector.partition(_.startsWith("\"")) - val err = res.err.linesIterator.toVector.filter(s => s.startsWith("Setting up ") || s.startsWith("Running ")) + val err = res.err.linesIterator.toVector.filter(s => + s.startsWith("Setting up ") || s.startsWith("Running ") + ) assert(out == expectedOut) diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index ffc3440df39f..18125c5431f5 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -80,8 +80,7 @@ object Watching { try { watchArgs.setIdle(true) enterKeyPressed = watchAndWait(streams, streams.in, watchables, watchArgs) - } - finally { + } finally { watchArgs.setIdle(false) } } From 35deeed2592456ca6cb12c8df9d4bb780bb55c35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Tue, 13 May 2025 12:00:14 +0300 Subject: [PATCH 16/18] Revert `run-tests.yml` --- .github/workflows/run-tests.yml | 705 ++++++++++++++------------------ 1 file changed, 297 insertions(+), 408 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b5e434a84c4f..1d24ed632777 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -1,433 +1,322 @@ # Uncommment this to replace the rest of the file when you want to debug stuff in CI -name: Run Debug +#name: Run Debug +# +#on: +# push: +# pull_request: +# workflow_dispatch: +# +#jobs: +# debug: +# runs-on: ubuntu-latest +## runs-on: windows-latest +# steps: +# - uses: actions/checkout@v4 +# with: { fetch-depth: 1 } +# +# - run: "echo temurin:11 > .mill-jvm-version" +# +# - uses: sbt/setup-sbt@v1 +# +# +# - run: ./mill 'integration.migrating[init].local.server.testOnly' mill.integration.MillInitSbtGatlingTests +# -on: - push: - pull_request: - workflow_dispatch: -jobs: - debug: - runs-on: ubuntu-latest -# runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 1 } - - run: "echo temurin:11 > .mill-jvm-version" +# We run full CI on push builds to main and on all pull requests +# +# To maximize bug-catching changes while keeping CI times reasonable, we run +# all tests on Linux, scattered between Java 11/17, except for one job run +# on MacOS instead and a subset of jobs also run on windows - - uses: sbt/setup-sbt@v1 +name: Run Tests - - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" mill.integration.WatchSourceTests.sources.show +on: + push: + branches-ignore: + - '**-patch-**' + pull_request: + types: + - opened + - reopened + - synchronize + - ready_for_review + workflow_dispatch: - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails - with: - path: out/**/fsNotifyWatchLog - name: debug-watch-log-artifact - include-hidden-files: true +# cancel older runs of a pull request; +# this will not cancel anything for normal git pushes +concurrency: + # * For runs on other repos, always use the `ref_name` so each branch only can have one concurrent run + # * For runs on `com-lihaoyi/mill`, use `head_ref` to allow one concurrent run per PR, but `run_id` to + # allow multiple concurrent runs in master + group: cancel-old-pr-runs-${{ github.workflow }}-${{ (github.repository != 'com-lihaoyi/mill' && github.ref_name) || (github.head_ref || github.run_id) }} + cancel-in-progress: true - debug-concurrent: +jobs: + # Jobs are listed in rough order of priority: if multiple jobs fail, the first job + # in the list should be the one that's most worth looking into + build-linux: + if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) + uses: ./.github/workflows/pre-build.yml + with: + os: ubuntu-latest + shell: bash + + build-windows: + if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) + uses: ./.github/workflows/pre-build.yml + with: + os: windows-latest + shell: powershell + + test-docs: + if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) runs-on: ubuntu-latest -# runs-on: windows-latest steps: - uses: actions/checkout@v4 with: { fetch-depth: 1 } - - run: "echo temurin:11 > .mill-jvm-version" - - - uses: sbt/setup-sbt@v1 - - - - run: ./mill "integration.invalidation[watch-source-input].packaged.server.testForked" - - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails - with: - path: out/**/fsNotifyWatchLog - name: debug-concurrent-watch-log-artifact - include-hidden-files: true - - debug-very-concurrent: - runs-on: ubuntu-latest -# runs-on: windows-latest + - run: ./mill -i website.fastPages + website.checkBrokenLinks + + cross-plat: + if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + # Run these with Mill native launcher as a smoketest + include: + - os: ubuntu-24.04-arm + millargs: "'example.thirdparty[fansi].native.server'" + java-version: 17 + + - os: macos-latest + millargs: "'example.thirdparty[acyclic].native.server'" + java-version: 11 + + - os: macos-13 + millargs: "'example.thirdparty[jimfs].native.server'" + java-version: 11 steps: - uses: actions/checkout@v4 with: { fetch-depth: 1 } - - run: "echo temurin:11 > .mill-jvm-version" - - - uses: sbt/setup-sbt@v1 - - - - run: ./mill "integration.invalidation[__].packaged.server.testForked" - - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails + - uses: ./.github/actions/pre-build-setup with: - path: out/**/fsNotifyWatchLog - name: debug-very-concurrent-watch-log-artifact - include-hidden-files: true - - debug2: - runs-on: ubuntu-latest -# runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 1 } - - - run: "echo temurin:11 > .mill-jvm-version" - - - uses: sbt/setup-sbt@v1 - - - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" mill.integration.WatchSourceTests.sources.noshow + os: ${{ matrix.os }} + java-version: ${{ matrix.java-version }} + shell: bash - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails + - uses: ./.github/actions/post-build-setup with: - path: out/**/fsNotifyWatchLog - name: debug2-watch-log-artifact - include-hidden-files: true + java-version: ${{ matrix.java-version }} + os: ${{ matrix.os }} - - debug2-concurrent: - runs-on: ubuntu-latest -# runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 1 } - - - run: "echo temurin:11 > .mill-jvm-version" - - - uses: sbt/setup-sbt@v1 - - - - run: ./mill -i "integration.invalidation[watch-source-input].packaged.fork.testForked" - - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails + - uses: ./.github/actions/post-build-selective with: - path: out/**/fsNotifyWatchLog - name: debug2-concurrent-watch-log-artifact - include-hidden-files: true - - debug2-very-concurrent: - runs-on: ubuntu-latest -# runs-on: windows-latest - steps: - - uses: actions/checkout@v4 - with: { fetch-depth: 1 } - - - run: "echo temurin:11 > .mill-jvm-version" - - - uses: sbt/setup-sbt@v1 - - - - run: ./mill -i "integration.invalidation[__].packaged.fork.testForked" - - - uses: actions/upload-artifact@v4 - if: always() # always run even if the previous step fails - with: - path: out/**/fsNotifyWatchLog - name: debug2-very-concurrent-watch-log-artifact - include-hidden-files: true - -# -# -## We run full CI on push builds to main and on all pull requests -## -## To maximize bug-catching changes while keeping CI times reasonable, we run -## all tests on Linux, scattered between Java 11/17, except for one job run -## on MacOS instead and a subset of jobs also run on windows -# -# -#name: Run Tests -# -#on: -# push: -# branches-ignore: -# - '**-patch-**' -# pull_request: -# types: -# - opened -# - reopened -# - synchronize -# - ready_for_review -# workflow_dispatch: -# -## cancel older runs of a pull request; -## this will not cancel anything for normal git pushes -#concurrency: -# # * For runs on other repos, always use the `ref_name` so each branch only can have one concurrent run -# # * For runs on `com-lihaoyi/mill`, use `head_ref` to allow one concurrent run per PR, but `run_id` to -# # allow multiple concurrent runs in master -# group: cancel-old-pr-runs-${{ github.workflow }}-${{ (github.repository != 'com-lihaoyi/mill' && github.ref_name) || (github.head_ref || github.run_id) }} -# cancel-in-progress: true -# -#jobs: -# # Jobs are listed in rough order of priority: if multiple jobs fail, the first job -# # in the list should be the one that's most worth looking into -# build-linux: -# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) -# uses: ./.github/workflows/pre-build.yml -# with: -# os: ubuntu-latest -# shell: bash -# -# build-windows: -# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) -# uses: ./.github/workflows/pre-build.yml -# with: -# os: windows-latest -# shell: powershell -# -# test-docs: -# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v4 -# with: { fetch-depth: 1 } -# -# - run: ./mill -i website.fastPages + website.checkBrokenLinks -# -# cross-plat: -# if: (github.event.action == 'ready_for_review') || (github.event.pull_request.draft == false) -# runs-on: ${{ matrix.os }} -# strategy: -# fail-fast: false -# matrix: -# # Run these with Mill native launcher as a smoketest -# include: -# - os: ubuntu-24.04-arm -# millargs: "'example.thirdparty[fansi].native.server'" -# java-version: 17 -# -# - os: macos-latest -# millargs: "'example.thirdparty[acyclic].native.server'" -# java-version: 11 -# -# - os: macos-13 -# millargs: "'example.thirdparty[jimfs].native.server'" -# java-version: 11 -# steps: -# - uses: actions/checkout@v4 -# with: { fetch-depth: 1 } -# -# - uses: ./.github/actions/pre-build-setup -# with: -# os: ${{ matrix.os }} -# java-version: ${{ matrix.java-version }} -# shell: bash -# -# - uses: ./.github/actions/post-build-setup -# with: -# java-version: ${{ matrix.java-version }} -# os: ${{ matrix.os }} -# -# - uses: ./.github/actions/post-build-selective -# with: -# millargs: ${{ matrix.millargs }} -# coursierarchive: "/tmp" -# shell: bash -# -# linux: -# needs: build-linux -# strategy: -# fail-fast: false -# matrix: -# -# include: -# # For most tests, run them arbitrarily on Java 11 or Java 17 on Linux, and -# # on the opposite version on Windows below, so we get decent coverage of -# # each test on each Java version and each operating system -# # We also try to group tests together to manually balance out the runtimes of each jobs -# - java-version: 17 -# millargs: "'{core,testkit,runner}.__.test'" -# install-android-sdk: false -# install-sbt: true -# -# - java-version: 17 -# millargs: "'libs.{main,scalalib,testrunner}.__.test'" -# install-android-sdk: false -# install-sbt: true -# -# - java-version: 11 -# millargs: "'libs.{scalajslib,scalanativelib,kotlinlib,pythonlib,javascriptlib}.__.test'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "contrib.__.test" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "example.javalib.__.local.server" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "example.kotlinlib.__.local.server" -# install-android-sdk: false -# install-sbt: false -# -# # Run this one using `.native` as a smoketest. Also make sure the java-version -# # is the same as that used in the `build-linux` job to avoid diverging code -# # hashes (https://github.com/com-lihaoyi/mill/pull/4410) -# - java-version: 11 -# millargs: "example.scalalib.__.native.server" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "'example.androidlib.__.local.server'" -# install-android-sdk: true -# install-sbt: false -# -# - java-version: 17 -# millargs: "'example.thirdparty[androidtodo].local.server'" -# install-android-sdk: true -# install-sbt: false -# -# - java-version: 17 -# millargs: "'example.thirdparty[android-endless-tunnel].local.server'" -# install-android-sdk: true -# install-sbt: false -# -# - java-version: 17 -# millargs: "'{example,integration}.migrating.__.local.server'" -# install-android-sdk: false -# install-sbt: true -# -# - java-version: 17 -# millargs: "'example.{pythonlib,javascriptlib}.__.local.server'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 11 -# millargs: "'example.thirdparty[{mockito,commons-io}].local.server'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "'example.thirdparty[{netty,gatling}].local.server'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: '17' -# millargs: "'example.thirdparty[arrow].local.server'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 11 -# millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 11 -# millargs: "'integration.{failure,feature,ide}.__.packaged.server'" -# install-android-sdk: false -# install-sbt: false -# -# # run this specifically in `native` mode to make sure our non-JVM native image -# # launcher is able to bootstrap everything necessary without a JVM installed -# - java-version: 17 -# millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" -# install-android-sdk: false -# -# # These invalidation tests need to be exercised in both execution modes -# # to make sure they work with and without -i/--no-server being passed -# - java-version: 17 -# millargs: "'integration.invalidation.__.packaged.fork'" -# install-android-sdk: false -# install-sbt: false -# -# - java-version: 17 -# millargs: "'integration.invalidation.__.packaged.server'" -# install-android-sdk: false -# install-sbt: false -# -# uses: ./.github/workflows/post-build-selective.yml -# with: -# install-android-sdk: ${{ matrix.install-android-sdk }} -# install-sbt: ${{ matrix.install-sbt || false }} -# java-version: ${{ matrix.java-version }} -# millargs: ${{ matrix.millargs }} -# shell: bash -# -# windows: -# needs: build-windows -# strategy: -# fail-fast: false -# matrix: -# include: -# # just run a subset of examples/ on Windows, because for some reason running -# # the whole suite can take hours on windows v.s. half an hour on linux -# # -# # * One job unit tests, -# # * One job each for local/packaged/native tests -# # * At least one job for each of fork/server tests, and example/integration tests -# - java-version: 11 -# millargs: '"libs.{main,scalalib}.__.test"' -# install-sbt: false -# -# - java-version: 11 -# millargs: '"example.scalalib.{basic,publishing}.__.local.fork"' -# install-sbt: false -# -# - java-version: 11 -# millargs: '"example.migrating.{scalalib,javalib}.__.local.fork"' -# install-sbt: true -# -# - java-version: 17 -# millargs: "'integration.{feature,failure}.__.packaged.fork'" -# install-sbt: false -# -# - java-version: 11 # Run this with Mill native launcher as a smoketest -# millargs: "'integration.invalidation.__.native.server'" -# install-sbt: false -# -# - java-version: 17 -# millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" -# -# uses: ./.github/workflows/post-build-selective.yml -# with: -# os: windows-latest -# java-version: ${{ matrix.java-version }} -# millargs: ${{ matrix.millargs }} -# # Provide a shorter coursier archive folder to avoid hitting path-length bugs when -# # running the graal native image binary on windows -# coursierarchive: "C:/coursier-arc" -# shell: powershell -# install-sbt: ${{ matrix.install-sbt || false }} -# -# itest: -# needs: build-linux -# strategy: -# fail-fast: false -# matrix: -# include: -# # bootstrap tests -# - java-version: 11 # Have one job on oldest JVM -# buildcmd: ci/test-dist-run.sh && ci/test-install-local.sh -# - java-version: 17 # Have one job on default JVM -# buildcmd: ci/test-mill-bootstrap.sh -# -# uses: ./.github/workflows/post-build-raw.yml -# with: -# java-version: ${{ matrix.java-version }} -# buildcmd: ${{ matrix.buildcmd }} -# -# # Scalafmt, Mima, and Scalafix job runs last because it's the least important: -# # usually just an automated or mechanical manual fix to do before merging -# lint-autofix: -# needs: build-linux -# uses: ./.github/workflows/post-build-raw.yml -# with: -# java-version: '17' -# buildcmd: | -# set -eux -# ./mill -i mill.scalalib.scalafmt.ScalafmtModule/scalafmt --check + __.fix --check + mill.javalib.palantirformat.PalantirFormatModule/ --check + mill.kotlinlib.ktlint.KtlintModule/checkFormatAll + millargs: ${{ matrix.millargs }} + coursierarchive: "/tmp" + shell: bash + + linux: + needs: build-linux + strategy: + fail-fast: false + matrix: + + include: + # For most tests, run them arbitrarily on Java 11 or Java 17 on Linux, and + # on the opposite version on Windows below, so we get decent coverage of + # each test on each Java version and each operating system + # We also try to group tests together to manually balance out the runtimes of each jobs + - java-version: 17 + millargs: "'{core,testkit,runner}.__.test'" + install-android-sdk: false + install-sbt: true + + - java-version: 17 + millargs: "'libs.{main,scalalib,testrunner}.__.test'" + install-android-sdk: false + install-sbt: true + + - java-version: 11 + millargs: "'libs.{scalajslib,scalanativelib,kotlinlib,pythonlib,javascriptlib}.__.test'" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "contrib.__.test" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "example.javalib.__.local.server" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "example.kotlinlib.__.local.server" + install-android-sdk: false + install-sbt: false + + # Run this one using `.native` as a smoketest. Also make sure the java-version + # is the same as that used in the `build-linux` job to avoid diverging code + # hashes (https://github.com/com-lihaoyi/mill/pull/4410) + - java-version: 11 + millargs: "example.scalalib.__.native.server" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "'example.androidlib.__.local.server'" + install-android-sdk: true + install-sbt: false + + - java-version: 17 + millargs: "'example.thirdparty[androidtodo].local.server'" + install-android-sdk: true + install-sbt: false + + - java-version: 17 + millargs: "'example.thirdparty[android-endless-tunnel].local.server'" + install-android-sdk: true + install-sbt: false + + - java-version: 17 + millargs: "'{example,integration}.migrating.__.local.server'" + install-android-sdk: false + install-sbt: true + + - java-version: 17 + millargs: "'example.{pythonlib,javascriptlib}.__.local.server'" + install-android-sdk: false + install-sbt: false + + - java-version: 11 + millargs: "'example.thirdparty[{mockito,commons-io}].local.server'" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "'example.thirdparty[{netty,gatling}].local.server'" + install-android-sdk: false + install-sbt: false + + - java-version: '17' + millargs: "'example.thirdparty[arrow].local.server'" + install-android-sdk: false + install-sbt: false + + - java-version: 11 + millargs: "'example.{cli,fundamentals,depth,extending}.__.local.server'" + install-android-sdk: false + install-sbt: false + + - java-version: 11 + millargs: "'integration.{failure,feature,ide}.__.packaged.server'" + install-android-sdk: false + install-sbt: false + + # run this specifically in `native` mode to make sure our non-JVM native image + # launcher is able to bootstrap everything necessary without a JVM installed + - java-version: 17 + millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" + install-android-sdk: false + + # These invalidation tests need to be exercised in both execution modes + # to make sure they work with and without -i/--no-server being passed + - java-version: 17 + millargs: "'integration.invalidation.__.packaged.fork'" + install-android-sdk: false + install-sbt: false + + - java-version: 17 + millargs: "'integration.invalidation.__.packaged.server'" + install-android-sdk: false + install-sbt: false + + uses: ./.github/workflows/post-build-selective.yml + with: + install-android-sdk: ${{ matrix.install-android-sdk }} + install-sbt: ${{ matrix.install-sbt || false }} + java-version: ${{ matrix.java-version }} + millargs: ${{ matrix.millargs }} + shell: bash + + windows: + needs: build-windows + strategy: + fail-fast: false + matrix: + include: + # just run a subset of examples/ on Windows, because for some reason running + # the whole suite can take hours on windows v.s. half an hour on linux + # + # * One job unit tests, + # * One job each for local/packaged/native tests + # * At least one job for each of fork/server tests, and example/integration tests + - java-version: 11 + millargs: '"libs.{main,scalalib}.__.test"' + install-sbt: false + + - java-version: 11 + millargs: '"example.scalalib.{basic,publishing}.__.local.fork"' + install-sbt: false + + - java-version: 11 + millargs: '"example.migrating.{scalalib,javalib}.__.local.fork"' + install-sbt: true + + - java-version: 17 + millargs: "'integration.{feature,failure}.__.packaged.fork'" + install-sbt: false + + - java-version: 11 # Run this with Mill native launcher as a smoketest + millargs: "'integration.invalidation.__.native.server'" + install-sbt: false + + - java-version: 17 + millargs: "'integration.bootstrap[no-java-bootstrap].native.server'" + + uses: ./.github/workflows/post-build-selective.yml + with: + os: windows-latest + java-version: ${{ matrix.java-version }} + millargs: ${{ matrix.millargs }} + # Provide a shorter coursier archive folder to avoid hitting path-length bugs when + # running the graal native image binary on windows + coursierarchive: "C:/coursier-arc" + shell: powershell + install-sbt: ${{ matrix.install-sbt || false }} + + itest: + needs: build-linux + strategy: + fail-fast: false + matrix: + include: + # bootstrap tests + - java-version: 11 # Have one job on oldest JVM + buildcmd: ci/test-dist-run.sh && ci/test-install-local.sh + - java-version: 17 # Have one job on default JVM + buildcmd: ci/test-mill-bootstrap.sh + + uses: ./.github/workflows/post-build-raw.yml + with: + java-version: ${{ matrix.java-version }} + buildcmd: ${{ matrix.buildcmd }} + + # Scalafmt, Mima, and Scalafix job runs last because it's the least important: + # usually just an automated or mechanical manual fix to do before merging + lint-autofix: + needs: build-linux + uses: ./.github/workflows/post-build-raw.yml + with: + java-version: '17' + buildcmd: | + set -eux + ./mill -i mill.scalalib.scalafmt.ScalafmtModule/scalafmt --check + __.fix --check + mill.javalib.palantirformat.PalantirFormatModule/ --check + mill.kotlinlib.ktlint.KtlintModule/checkFormatAll From 46990ff357d6e044083b854b20b285031f793e78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Art=C5=ABras=20=C5=A0lajus?= Date: Mon, 19 May 2025 11:49:19 +0300 Subject: [PATCH 17/18] Code review changes. --- core/api/src/mill/api/Watchable.scala | 15 ++++------ .../src/mill/daemon/MillBuildBootstrap.scala | 2 +- .../src/mill/daemon/MillCliConfig.scala | 2 +- runner/daemon/src/mill/daemon/Watching.scala | 29 +++++++++---------- 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/core/api/src/mill/api/Watchable.scala b/core/api/src/mill/api/Watchable.scala index b47d99bef5a3..4de0944235a5 100644 --- a/core/api/src/mill/api/Watchable.scala +++ b/core/api/src/mill/api/Watchable.scala @@ -8,24 +8,21 @@ package mill.api */ private[mill] sealed trait Watchable private[mill] object Watchable { - - /** A [[Watchable]] that is being watched via polling. */ - private[mill] sealed trait Pollable extends Watchable - - /** A [[Watchable]] that is being watched via a notification system (like inotify). */ - private[mill] sealed trait Notifiable extends Watchable - /** + * Watched path, can be watched via polling or via a notification system. + * * @param p the path to watch * @param quick if true, only watch file attributes * @param signature the initial hash of the path contents */ - case class Path(p: java.nio.file.Path, quick: Boolean, signature: Int) extends Notifiable + case class Path(p: java.nio.file.Path, quick: Boolean, signature: Int) extends Watchable /** + * Watched expression, can only be watched via polling. + * * @param f the expression to watch, returns some sort of hash * @param signature the initial hash from the first invocation of the expression * @param pretty human-readable name */ - case class Value(f: () => Long, signature: Long, pretty: String) extends Pollable + case class Value(f: () => Long, signature: Long, pretty: String) extends Watchable } diff --git a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala index 1c5689ae5b6e..c1a16cc99788 100644 --- a/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala +++ b/runner/daemon/src/mill/daemon/MillBuildBootstrap.scala @@ -280,7 +280,7 @@ class MillBuildBootstrap( // look at the `moduleWatched` of one frame up (`prevOuterFrameOpt`), // and not the `moduleWatched` from the current frame (`prevFrameOpt`) val moduleWatchChanged = - prevOuterFrameOpt.exists(_.moduleWatched.exists(w => !Watching.validateAnyWatchable(w))) + prevOuterFrameOpt.exists(_.moduleWatched.exists(w => !Watching.haveNotChanged(w))) val classLoader = if (runClasspathChanged || moduleWatchChanged) { // Make sure we close the old classloader every time we create a new diff --git a/runner/daemon/src/mill/daemon/MillCliConfig.scala b/runner/daemon/src/mill/daemon/MillCliConfig.scala index 38ee92167e66..0410ffde111a 100644 --- a/runner/daemon/src/mill/daemon/MillCliConfig.scala +++ b/runner/daemon/src/mill/daemon/MillCliConfig.scala @@ -98,7 +98,7 @@ case class MillCliConfig( ) watch: Flag = Flag(), @arg( - name = "watch-via-fs-notify", + name = "notify-watch", doc = "Use filesystem based file watching instead of polling based one (defaults to true)." ) watchViaFsNotify: Boolean = true, diff --git a/runner/daemon/src/mill/daemon/Watching.scala b/runner/daemon/src/mill/daemon/Watching.scala index 18125c5431f5..d6336c4a05aa 100644 --- a/runner/daemon/src/mill/daemon/Watching.scala +++ b/runner/daemon/src/mill/daemon/Watching.scala @@ -94,8 +94,8 @@ object Watching { watched: Seq[Watchable], watchArgs: WatchArgs ): Boolean = { - val (watchedPollables, watchedPathsSeq) = watched.partitionMap { - case w: Watchable.Pollable => Left(w) + val (watchedValues, watchedPathsSeq) = watched.partitionMap { + case v: Watchable.Value => Left(v) case p: Watchable.Path => Right(p) } val watchedPathsSet = watchedPathsSeq.iterator.map(p => os.Path(p.p)).toSet @@ -112,12 +112,12 @@ object Watching { } def doWatch(notifiablesChanged: () => Boolean) = { - val enterKeyPressed = statWatchWait(watchedPollables, stdin, notifiablesChanged) + val enterKeyPressed = statWatchWait(watchedValues, stdin, notifiablesChanged) enterKeyPressed } def doWatchPolling() = - doWatch(notifiablesChanged = () => watchedPathsSeq.exists(p => !validateAnyWatchable(p))) + doWatch(notifiablesChanged = () => watchedPathsSeq.exists(p => !haveNotChanged(p))) def doWatchFsNotify() = { Using.resource(os.write.outputStream(watchArgs.serverDir / "fsNotifyWatchLog")) { watchLog => @@ -159,6 +159,12 @@ object Watching { // root/a/b // root/a // root + // + // We're only setting one `os.watch.watch` on the root, and this makes it sound like + // we're setting multiple. What we're actually doing is choosing the paths we need to watch recursively in + // Linux since inotify is non-recursive by default, since changes in any enclosing folder could result in the + // watched file or folder disappearing (e.g. if the enclosing folder was renamed) and we want to pick up such + // changes. val filterPaths = pathsUnderWorkspaceRoot.flatMap { path => path.relativeTo(workspaceRoot).segments.inits.map(segments => workspaceRoot / segments) } @@ -196,7 +202,7 @@ object Watching { // // We need to do this to prevent any changes from slipping through the gap between the last evaluation and // starting the watch. - val alreadyStale = watched.exists(w => !validateAnyWatchable(w)) + val alreadyStale = watched.exists(w => !haveNotChanged(w)) if (alreadyStale) false else doWatch(notifiablesChanged = () => pathChangesDetected) @@ -214,14 +220,14 @@ object Watching { * @return `true` if enter key is pressed to re-run tasks explicitly, false if changes in watched files occured. */ def statWatchWait( - watched: Seq[Watchable.Pollable], + watchedValues: Seq[Watchable.Value], stdin: InputStream, notifiablesChanged: () => Boolean ): Boolean = { val buffer = new Array[Byte](4 * 1024) @tailrec def statWatchWait0(): Boolean = { - if (!notifiablesChanged() && watched.forall(w => validate(w))) { + if (!notifiablesChanged() && watchedValues.forall(haveNotChanged)) { if (lookForEnterKey()) { true } else { @@ -251,14 +257,7 @@ object Watching { } /** @return true if the watchable did not change. */ - inline def validate(w: Watchable.Pollable): Boolean = validateAnyWatchable(w) - - /** - * As [[validate]] but accepts any [[Watchable]] for the cases when we do not want to use a notification system. - * - * Normally you should use [[validate]] so that types would guide your implementation. - */ - def validateAnyWatchable(w: Watchable): Boolean = poll(w) == signature(w) + def haveNotChanged(w: Watchable): Boolean = poll(w) == signature(w) def poll(w: Watchable): Long = w match { case Watchable.Path(p, quick, sig) => From 5e2d4450cc44d8b735b239157005024f981e3f2e Mon Sep 17 00:00:00 2001 From: "autofix-ci[bot]" <114827586+autofix-ci[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 09:10:31 +0000 Subject: [PATCH 18/18] [autofix.ci] apply automated fixes --- core/api/src/mill/api/Watchable.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/core/api/src/mill/api/Watchable.scala b/core/api/src/mill/api/Watchable.scala index 4de0944235a5..08a4c933330f 100644 --- a/core/api/src/mill/api/Watchable.scala +++ b/core/api/src/mill/api/Watchable.scala @@ -8,6 +8,7 @@ package mill.api */ private[mill] sealed trait Watchable private[mill] object Watchable { + /** * Watched path, can be watched via polling or via a notification system. *