Use DoubleLock for mutex on compiler bridge creation to avoid races between MillDaemonMain and ZincWorkerMain processes (#5761)

Hopefully fixes #5743. Will
need to rebootstrap on this to stop the failures in our CI

The previous in-memory lock is no longer sufficient for mutex in this
code path, because now both the main Mill process and any
`ZincWorkerMain` subprocess could both be trying to initialize compiler
bridges in the same `compilerBridge.workspace` folder

Author: lihaoyi

libs/javalib/worker/src/mill/javalib/zinc/ZincWorker.scala

@@ -4,6 +4,7 @@ import mill.api.JsonFormatters.*
 import mill.api.PathRef
 import mill.api.daemon.internal.CompileProblemReporter
 import mill.api.daemon.{Logger, Result}
+import mill.client.lock.*
 import mill.javalib.api.internal.{
   JavaCompilerOptions,
   ZincCompileJava,
@@ -36,7 +37,7 @@ class ZincWorker(
     jobs: Int
 ) extends AutoCloseable { self =>
   private val incrementalCompiler = new sbt.internal.inc.IncrementalCompilerImpl()
-  private val compilerBridgeLocks: mutable.Map[String, Object] = mutable.Map.empty[String, Object]
+  private val compilerBridgeLocks: mutable.Map[String, MemoryLock] = mutable.Map.empty
 
   private val classloaderCache = new RefCountedClassLoaderCache(
     sharedLoader = getClass.getClassLoader,
@@ -547,10 +548,22 @@ class ZincWorker(
       compilerBridge: ZincCompilerBridgeProvider
   ): os.Path = {
     val workingDir = compilerBridge.workspace / s"zinc-${Versions.zinc}" / scalaVersion
-    val lock = synchronized(compilerBridgeLocks.getOrElseUpdate(scalaVersion, new Object()))
+
+    os.makeDir.all(compilerBridge.workspace / "compiler-bridge-locks")
+    val memoryLock = synchronized(
+      compilerBridgeLocks.getOrElseUpdate(scalaVersion, new MemoryLock)
+    )
     val compiledDest = workingDir / "compiled"
     val doneFile = compiledDest / "DONE"
-    lock.synchronized {
+    // Use a double-lock here because we need mutex both between threads within this
+    // process, as well as between different processes since sometimes we are initializing
+    // the compiler bridge inside a separate `ZincWorkerMain` subprocess
+    val doubleLock = new DoubleLock(
+      memoryLock,
+      new FileLock((compilerBridge.workspace / "compiler-bridge-locks" / scalaVersion).toString)
+    )
+    try {
+      doubleLock.lock()
       if (os.exists(doneFile)) compiledDest
       else {
         val acquired =
@@ -572,7 +585,7 @@ class ZincWorker(
             compiledDest
         }
       }
-    }
+    } finally doubleLock.close()
   }
 }
 object ZincWorker {