Skip to content

Commit 3208a7c

Browse files
lihaoyilihaoyi
committed
merged
1 parent 76cc83e commit 3208a7c

File tree

17 files changed

+254
-142
lines changed

17 files changed

+254
-142
lines changed

‎core/api/java11/src/mill/api/JsonFormatters.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ trait JsonFormatters {
2525
implicit val pathReadWrite: RW[os.Path] = upickle.readwriter[String]
2626
.bimap[os.Path](
2727
_.toString,
28-
os.Path(_)
28+
os.Path(_, os.pwd)
2929
)
3030

3131
implicit val relPathRW: RW[os.RelPath] = upickle.readwriter[String]

‎core/api/java11/src/mill/api/PathRef.scala‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,19 +130,17 @@ object PathRef {
130130
if (os.exists(path)) {
131131
for (
132132
(path, attrs) <-
133-
os.walk.attrs(path, includeTarget = true, followLinks = true).sortBy(_._1.toString)
133+
os.walk.attrs(path, includeTarget = true, followLinks = false).sortBy(_._1.toString)
134134
) {
135135
val sub = path.subRelativeTo(basePath)
136136
digest.update(sub.toString().getBytes())
137137
if (!attrs.isDir) {
138138
try {
139-
if (isPosix) {
140-
updateWithInt(os.perms(path, followLinks = false).value)
141-
}
139+
if (isPosix) updateWithInt(os.perms(path, followLinks = false).value)
142140
if (quick) {
143141
val value = (attrs.mtime, attrs.size).hashCode()
144142
updateWithInt(value)
145-
} else if (jnio.Files.isReadable(path.toNIO)) {
143+
} else if (!os.isLink(path) && jnio.Files.isReadable(path.toNIO)) {
146144
val is =
147145
try Some(os.read.inputStream(path))
148146
catch {

‎core/api/src/mill/api/Task.scala‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -383,7 +383,8 @@ object Task {
383383

384384
override def evaluate0: (Seq[Any], TaskCtx) => Result[T] =
385385
(_, _) => {
386-
val relPath = os.Path(ctx0.fileName).relativeTo(mill.api.BuildCtx.workspaceRoot)
386+
val relPath = os.Path(ctx0.fileName, mill.api.BuildCtx.workspaceRoot)
387+
.relativeTo(mill.api.BuildCtx.workspaceRoot)
387388
Result.Failure(s"configuration missing in $relPath")
388389
}
389390

@@ -556,7 +557,7 @@ object Task {
556557
val isPrivate: Option[Boolean]
557558
) extends Simple[T] {
558559
val inputs = Nil
559-
override def sideHash: Int = util.Random.nextInt()
560+
override def sideHash: Int = 31337
560561
// FIXME: deprecated return type: Change to Option
561562
override def writerOpt: Some[Writer[?]] = Some(writer)
562563
override private[mill] def isInputTask: Boolean = true

‎core/eval/package.mill‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ import millbuild.*
1010
* codebases.
1111
*/
1212
object `package` extends MillPublishScalaModule {
13-
def moduleDeps = Seq(build.core.api, build.core.exec, build.core.resolve)
13+
def moduleDeps = Seq(build.core.api, build.core.exec, build.core.resolve, build.core.internal)
1414
}

‎core/eval/src/mill/eval/EvaluatorImpl.scala‎

Lines changed: 112 additions & 98 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import mill.api.internal.{ResolveChecker, Resolved, RootModule0}
77
import mill.api.daemon.Watchable
88
import mill.exec.{Execution, PlanImpl}
99
import mill.internal.PrefixLogger
10+
import mill.internal.MillPathSerializer
1011
import mill.resolve.Resolve
1112
import mill.api.internal.ParseArgs
1213

@@ -40,6 +41,17 @@ final class EvaluatorImpl(
4041
new ScriptModuleInit()
4142
)
4243
override val staticBuildOverrides = execution.staticBuildOverrides
44+
private val millPathSerializer = new MillPathSerializer(MillPathSerializer.defaultMapping(workspace))
45+
46+
MillPathSerializer.setupSymlinks(os.pwd, workspace)
47+
48+
private object SetupSymlinkSpawnHook extends (os.Path => Unit) {
49+
def apply(p: os.Path): Unit = MillPathSerializer.setupSymlinks(p, workspace)
50+
override def toString(): String = "SetupSymlinkSpawnHook"
51+
}
52+
53+
private def withPathSerialization[T](t: => T): T =
54+
os.Path.pathSerializer.withValue(millPathSerializer)(t)
4355

4456
def workspace = execution.workspace
4557
def baseLogger = execution.baseLogger
@@ -204,10 +216,10 @@ final class EvaluatorImpl(
204216
}
205217
}
206218

207-
val filePath = os.Path(module.moduleCtx.fileName).relativeTo(workspace)
208-
219+
val normalizedFileName = module.moduleCtx.fileName.replace('\\', '/')
209220
val isRootBuildFile =
210-
filePath == os.sub / "mill-build/build.mill" || filePath == os.sub / "build.mill.yaml"
221+
normalizedFileName.endsWith("/mill-build/build.mill") ||
222+
normalizedFileName.endsWith("/build.mill.yaml")
211223

212224
val millKeys = mill.constants.ConfigConstants.all()
213225
val validKeys =
@@ -291,105 +303,107 @@ final class EvaluatorImpl(
291303
logger: Logger = baseLogger,
292304
serialCommandExec: Boolean = false,
293305
selectiveExecution: Boolean = false
294-
): Evaluator.Result[T] = {
295-
val selectiveExecutionEnabled = selectiveExecution && !tasks.exists(_.isExclusiveCommand)
296-
297-
val (selectedTasks, selectiveResults, maybeNewMetadata) =
298-
if (!selectiveExecutionEnabled) (tasks, Map.empty, None)
299-
else {
300-
val (named, unnamed) =
301-
tasks.partitionMap { case n: Task.Named[?] => Left(n); case t => Right(t) }
302-
val newComputedMetadata = this.selective.computeMetadata(named)
303-
304-
val selectiveExecutionStoredData = for {
305-
_ <- Option.when(os.exists(outPath / OutFiles.millSelectiveExecution))(())
306-
changedTasks <- this.selective.computeChangedTasks0(named, newComputedMetadata)
307-
} yield changedTasks
308-
309-
selectiveExecutionStoredData match {
310-
case None =>
311-
// Ran when previous selective execution metadata is not available, which happens the first time you run
312-
// selective execution.
313-
(tasks, Map.empty, Some(newComputedMetadata.metadata))
314-
case Some(changedTasks) =>
315-
val selectedSet = changedTasks.downstreamTasks.map(_.ctx.segments.render).toSet
316-
317-
(
318-
unnamed ++ named.filter(t =>
319-
t.isExclusiveCommand || selectedSet(t.ctx.segments.render)
320-
),
321-
newComputedMetadata.results,
322-
Some(newComputedMetadata.metadata)
323-
)
306+
): Evaluator.Result[T] = os.ProcessOps.spawnHook.withValue(SetupSymlinkSpawnHook) {
307+
withPathSerialization {
308+
val selectiveExecutionEnabled = selectiveExecution && !tasks.exists(_.isExclusiveCommand)
309+
310+
val (selectedTasks, selectiveResults, maybeNewMetadata) =
311+
if (!selectiveExecutionEnabled) (tasks, Map.empty, None)
312+
else {
313+
val (named, unnamed) =
314+
tasks.partitionMap { case n: Task.Named[?] => Left(n); case t => Right(t) }
315+
val newComputedMetadata = this.selective.computeMetadata(named)
316+
317+
val selectiveExecutionStoredData = for {
318+
_ <- Option.when(os.exists(outPath / OutFiles.millSelectiveExecution))(())
319+
changedTasks <- this.selective.computeChangedTasks0(named, newComputedMetadata)
320+
} yield changedTasks
321+
322+
selectiveExecutionStoredData match {
323+
case None =>
324+
// Ran when previous selective execution metadata is not available, which happens the first time you run
325+
// selective execution.
326+
(tasks, Map.empty, Some(newComputedMetadata.metadata))
327+
case Some(changedTasks) =>
328+
val selectedSet = changedTasks.downstreamTasks.map(_.ctx.segments.render).toSet
329+
330+
(
331+
unnamed ++ named.filter(t =>
332+
t.isExclusiveCommand || selectedSet(t.ctx.segments.render)
333+
),
334+
newComputedMetadata.results,
335+
Some(newComputedMetadata.metadata)
336+
)
337+
}
324338
}
325-
}
326-
327-
val evaluated: ExecutionResults = execution.executeTasks(
328-
goals = selectedTasks,
329-
reporter = reporter,
330-
testReporter = testReporter,
331-
logger = logger,
332-
serialCommandExec = serialCommandExec
333-
)
334339

335-
val allResults = evaluated.transitiveResults ++ selectiveResults
336-
337-
@scala.annotation.nowarn("msg=cannot be checked at runtime")
338-
val watched = allResults.collect {
339-
case (_: Task.Sources, ExecResult.Success(Val(ps: Seq[PathRef]))) =>
340-
ps.map(r => Watchable.Path.from(r))
341-
case (_: Task.Source, ExecResult.Success(Val(p: PathRef))) =>
342-
Seq(Watchable.Path.from(p))
343-
case (t: Task.Input[_], result) =>
344-
345-
val ctx = new mill.api.TaskCtx.Impl(
346-
args = Vector(),
347-
dest0 = () => null,
348-
log = logger,
349-
env = this.execution.env,
350-
reporter = reporter,
351-
testReporter = testReporter,
352-
workspace = workspace,
353-
_systemExitWithReason = (reason, exitCode) =>
354-
throw Exception(s"systemExit called: reason=$reason, exitCode=$exitCode"),
355-
fork = null,
356-
jobs = execution.effectiveThreadCount,
357-
offline = offline,
358-
useFileLocks = useFileLocks
359-
)
360-
val pretty = t.ctx0.fileName + ":" + t.ctx0.lineNum
361-
Seq(Watchable.Value(
362-
() => t.evaluate(ctx).hashCode(),
363-
result.map(_.value).hashCode(),
364-
pretty
365-
))
366-
}.flatten.toVector
367-
368-
for (newMetadata <- maybeNewMetadata) {
369-
val failingTaskNames = allResults
370-
.collect { case (t: Task.Named[_], r) if r.asSuccess.isEmpty => t.ctx.segments.render }
371-
.toSet
340+
val evaluated: ExecutionResults = execution.executeTasks(
341+
goals = selectedTasks,
342+
reporter = reporter,
343+
testReporter = testReporter,
344+
logger = logger,
345+
serialCommandExec = serialCommandExec
346+
)
372347

373-
// For tasks that were not successful, force them to re-run next time even
374-
// if not changed so the user can see that there are still failures remaining
375-
selective.saveMetadata(newMetadata.copy(forceRunTasks = failingTaskNames))
376-
}
348+
val allResults = evaluated.transitiveResults ++ selectiveResults
349+
350+
@scala.annotation.nowarn("msg=cannot be checked at runtime")
351+
val watched = allResults.collect {
352+
case (_: Task.Sources, ExecResult.Success(Val(ps: Seq[PathRef]))) =>
353+
ps.map(r => Watchable.Path.from(r))
354+
case (_: Task.Source, ExecResult.Success(Val(p: PathRef))) =>
355+
Seq(Watchable.Path.from(p))
356+
case (t: Task.Input[_], result) =>
357+
358+
val ctx = new mill.api.TaskCtx.Impl(
359+
args = Vector(),
360+
dest0 = () => null,
361+
log = logger,
362+
env = this.execution.env,
363+
reporter = reporter,
364+
testReporter = testReporter,
365+
workspace = workspace,
366+
_systemExitWithReason = (reason, exitCode) =>
367+
throw Exception(s"systemExit called: reason=$reason, exitCode=$exitCode"),
368+
fork = null,
369+
jobs = execution.effectiveThreadCount,
370+
offline = offline,
371+
useFileLocks = useFileLocks
372+
)
373+
val pretty = t.ctx0.fileName + ":" + t.ctx0.lineNum
374+
Seq(Watchable.Value(
375+
() => t.evaluate(ctx).hashCode(),
376+
result.map(_.value).hashCode(),
377+
pretty
378+
))
379+
}.flatten.toVector
380+
381+
for (newMetadata <- maybeNewMetadata) {
382+
val failingTaskNames = allResults
383+
.collect { case (t: Task.Named[_], r) if r.asSuccess.isEmpty => t.ctx.segments.render }
384+
.toSet
385+
386+
// For tasks that were not successful, force them to re-run next time even
387+
// if not changed so the user can see that there are still failures remaining
388+
selective.saveMetadata(newMetadata.copy(forceRunTasks = failingTaskNames))
389+
}
377390

378-
evaluated.transitiveFailing.size match {
379-
case 0 =>
380-
Evaluator.Result(
381-
watched,
382-
mill.api.Result.Success(evaluated.values.map(_._1.asInstanceOf[T])),
383-
selectedTasks,
384-
evaluated
385-
)
386-
case _ =>
387-
Evaluator.Result(
388-
watched,
389-
mill.internal.Util.formatFailing(evaluated),
390-
selectedTasks,
391-
evaluated
392-
)
391+
evaluated.transitiveFailing.size match {
392+
case 0 =>
393+
Evaluator.Result(
394+
watched,
395+
mill.api.Result.Success(evaluated.values.map(_._1.asInstanceOf[T])),
396+
selectedTasks,
397+
evaluated
398+
)
399+
case _ =>
400+
Evaluator.Result(
401+
watched,
402+
mill.internal.Util.formatFailing(evaluated),
403+
selectedTasks,
404+
evaluated
405+
)
406+
}
393407
}
394408
}
395409

‎core/exec/src/mill/exec/GroupExecution.scala‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -230,13 +230,13 @@ trait GroupExecution {
230230
exclusive: Boolean,
231231
upstreamPathRefs: Seq[PathRef]
232232
): GroupExecution.Results = {
233+
val sideHashes = group.iterator.map(_.sideHash).sum
233234

234235
val inputsHash = {
235236
val externalInputsHash = MurmurHash3.orderedHash(
236237
group.flatMap(_.inputs).filter(!group.contains(_))
237238
.flatMap(results(_).asSuccess.map(_.value._2))
238239
)
239-
val sideHashes = MurmurHash3.orderedHash(group.iterator.map(_.sideHash))
240240
val scriptsHash = MurmurHash3.orderedHash(
241241
group
242242
.iterator
@@ -279,7 +279,9 @@ trait GroupExecution {
279279

280280
// Helper to evaluate the task with full caching support
281281
def evaluateTaskWithCaching(): GroupExecution.Results = {
282-
val cached = loadCachedJson(logger, inputsHash, labelled, paths)
282+
val cached = Option
283+
.when(sideHashes == 0) { loadCachedJson(logger, inputsHash, labelled, paths) }
284+
.flatten
283285

284286
// `cached.isEmpty` means worker metadata file removed by user so recompute the worker
285287
val (multiLogger, _) = resolveLogger(Some(paths).map(_.log), logger)
@@ -298,10 +300,10 @@ trait GroupExecution {
298300
terminal = terminal
299301
)
300302

301-
val cachedValueAndHash =
302-
upToDateWorker.map(w => (w -> Nil, inputsHash))
303+
val cachedValueAndHash: Option[((Val, Seq[PathRef]), Int)] =
304+
upToDateWorker.map(w => ((w, Nil), inputsHash))
303305
.orElse(cached.flatMap { case (_, valOpt, valueHash) =>
304-
valOpt.map((_, valueHash))
306+
valOpt.map(v => (v, valueHash))
305307
})
306308

307309
cachedValueAndHash match {
@@ -370,10 +372,14 @@ trait GroupExecution {
370372
def evaluateBuildOverrideOnly(located: Located[Appendable[BufferedValue]])
371373
: GroupExecution.Results = {
372374

375+
val fileName = labelled.ctx.fileName
376+
val isBuildMill = fileName.replace('\\', '/').endsWith("/mill-build/build.mill")
377+
val displayPath =
378+
scala.util.Try(os.Path(fileName, os.pwd).relativeTo(workspace).toString).getOrElse(fileName)
373379
val (execRes, serializedPaths) =
374-
if (os.Path(labelled.ctx.fileName).endsWith("mill-build/build.mill")) {
380+
if (isBuildMill) {
375381
val msg =
376-
s"Build header config conflicts with task defined in ${os.Path(labelled.ctx.fileName).relativeTo(workspace)}:${labelled.ctx.lineNum}"
382+
s"Build header config conflicts with task defined in ${displayPath}:${labelled.ctx.lineNum}"
377383
(
378384
ExecResult.Failure(
379385
msg,
@@ -706,7 +712,20 @@ trait GroupExecution {
706712
}
707713

708714
def getValueHash(v: Val, task: Task[?], inputsHash: Int): Int = {
709-
if (task.isInstanceOf[Task.Worker[?]]) inputsHash else v.## + invalidateAllHashes
715+
if (task.isInstanceOf[Task.Worker[?]]) inputsHash
716+
else {
717+
task match {
718+
case named: Task.Named[_] =>
719+
named.writerOpt match {
720+
case Some(writer) =>
721+
upickle
722+
.writeJs(v.value)(using writer.asInstanceOf[upickle.Writer[Any]])
723+
.hashCode()
724+
case None => v.##
725+
}
726+
case _ => v.##
727+
}
728+
}
710729
}
711730

712731
private def loadUpToDateWorker(

‎core/exec/test/src/mill/exec/JavaCompileJarTests.scala‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,10 @@ object JavaCompileJarTests extends TestSuite {
139139
val expectedJarContents =
140140
"""META-INF/MANIFEST.MF
141141
|META-INF/
142+
|out/
142143
|test/
144+
|out/mill-home/
145+
|out/mill-workspace/
143146
|test/Bar.class
144147
|test/BarThree.class
145148
|test/BarTwo.class

0 commit comments

Comments
 (0)