3-stage compile to support Groovy <-> Java cycles

Author: lostiniceland

libs/groovylib/api/src/mill/groovylib/worker/api/GroovyWorker.scala

@@ -4,9 +4,28 @@ import mill.api.TaskCtx
 import mill.api.Result
 import mill.javalib.api.CompilationResult
 
+/**
+ * Runs the actual compilation.
+ *
+ * Supports 3-stage compilation for Java <-> Groovy
+ * 1. compile Java stubs
+ * 2. compile Java sources (done externally)
+ * 3. compile Groovy sources
+ */
 trait GroovyWorker {
 
-  def compile(sourceFiles: Seq[os.Path], classpath: Seq[os.Path], outputDir: os.Path)(implicit
-      ctx: TaskCtx
-  ): Result[CompilationResult]
+  /**
+   * In a mixed setup this will compile the Groovy sources to Java stubs.
+   */
+  def compileGroovyStubs(sourceFiles: Seq[os.Path], classpath: Seq[os.Path], outputDir: os.Path)
+                        (implicit ctx: TaskCtx)
+  : Result[CompilationResult]
+
+  /**
+   * Compiles the Groovy sources. In a mixed setup this method assumes that the Java stubs
+   * are already present in the outputDir.
+   */
+  def compile(sourceFiles: Seq[os.Path], classpath: Seq[os.Path], outputDir: os.Path)
+             (implicit ctx: TaskCtx)
+  : Result[CompilationResult]
 }

libs/groovylib/src/mill/groovylib/GroovyModule.scala

@@ -90,10 +90,7 @@ trait GroovyModule extends JavaModule with GroovyModuleApi { outer =>
    */
   def groovyCompilerMvnDeps: T[Seq[Dep]] = Task {
     val gv = groovyVersion()
-
     val compilerDep = mvn"org.apache.groovy:groovy-all:$gv"
-//    val annotationDep = mvn"org.codehaus.groovy:groovy-all-annotations:$gv"
-
     Seq(compilerDep)
   }
 
@@ -157,36 +154,39 @@ trait GroovyModule extends JavaModule with GroovyModuleApi { outer =>
           reportOldProblems = zincReportCachedProblems()
         )
       }
+      
+      if(isMixed){
+        ctx.log.info("Compiling Groovy stubs for mixed compilation")
 
-      if (isMixed || isGroovy) {
+        val workerStubResult =
+          GroovyWorkerManager.groovyWorker().withValue(groovyCompilerClasspath()) {
+            _.compileGroovyStubs(groovySourceFiles, compileCp, classes)
+          }
+        workerStubResult match {
+          case Result.Success(_) => compileJava
+          case Result.Failure(reason) => Result.Failure(reason)
+        }
+      }
+      
+      if(isMixed || isGroovy){
         ctx.log.info(
           s"Compiling ${groovySourceFiles.size} Groovy sources to ${classes} ..."
         )
-
-        val compileCp = compileClasspath().map(_.path).filter(os.exists)
-
-        val workerResult =
+        
+        val workerGroovyResult =
           GroovyWorkerManager.groovyWorker().withValue(groovyCompilerClasspath()) {
             _.compile(groovySourceFiles, compileCp, classes)
           }
 
         val analysisFile = dest / "groovy.analysis.dummy" // needed for mills CompilationResult
         os.write(target = analysisFile, data = "", createFolders = true)
 
-        workerResult match {
+        workerGroovyResult match {
           case Result.Success(_) =>
-            val cr = CompilationResult(analysisFile, PathRef(classes))
-            if (!isJava) {
-              // pure Groovy project
-              cr
-            } else {
-              // also run Java compiler and use it's returned result
-              compileJava
-            }
+            CompilationResult(analysisFile, PathRef(classes))
           case Result.Failure(reason) => Result.Failure(reason)
         }
-      } else {
-        // it's Java only
+      }else {
         compileJava
       }
     }

libs/groovylib/test/resources/hello-groovy/groovy-tests/src/test/groovy/HelloMavenTestOnly.groovy

@@ -0,0 +1,12 @@
+package hello.maven.tests
+
+//import hello.Hello
+import org.junit.jupiter.api.Test
+import static org.junit.jupiter.api.Assertions.assertEquals
+
+class HelloMavenTestOnly {
+    @Test
+    void testSuccess() {
+        assertEquals(true, true)
+    }
+}

libs/groovylib/test/src/mill/groovylib/HelloGroovyTests.scala

@@ -1,8 +1,8 @@
 package mill
 package groovylib
 
-import mill.javalib.{JavaModule, MavenModule, TestModule}
-import mill.api.{ExecResult, Task}
+import mill.javalib.{JavaModule, TestModule}
+import mill.api.{ Task}
 import mill.api.Discover
 import mill.testkit.{TestRootModule, UnitTester}
 import utest.*
@@ -17,12 +17,12 @@ object HelloGroovyTests extends TestSuite {
     lazy val millDiscover = Discover[this.type]
 
     // needed for a special test where only the tests are written in Groovy while appcode remains Java
-    object `mixed-compile` extends JavaModule with MavenModule {
+    object `groovy-tests` extends JavaMavenModuleWithGroovyTests {
 
-      object `test` extends TestGroovyMavenModule with TestModule.Junit5 {
+      object `test` extends GroovyMavenTests with TestModule.Junit5 {
 
         override def moduleDeps: Seq[JavaModule] = Seq(
-          HelloGroovy.`mixed-compile`, // TODO improve: TestOnly does not inherit outer deps
+          HelloGroovy.`groovy-tests`, 
         )
 
         override def groovyVersion = groovy4Version
@@ -37,6 +37,7 @@ object HelloGroovyTests extends TestSuite {
 
     object `joint-compile` extends GroovyModule {
       override def groovyVersion: T[String] = groovy4Version
+      override def mainClass = Some("jointcompile.JavaMain")
     }
 
     trait Test extends GroovyModule {
@@ -92,63 +93,16 @@ object HelloGroovyTests extends TestSuite {
   def tests: Tests = Tests {
 
     def m = HelloGroovy.main
-    def mixed = HelloGroovy.`mixed-compile`
+    def mixed = HelloGroovy.`groovy-tests`
+    def joint = HelloGroovy.`joint-compile`
 
-    test("running a Groovy script") {
-      testEval().scoped { eval =>
-        val Right(_) = eval.apply(m.script.run()): @unchecked
-      }
-    }
-
-    test("compile & run Groovy module") {
-      testEval().scoped { eval =>
-        val Right(result) = eval.apply(m.compile): @unchecked
-
-        assert(
-          os.walk(result.value.classes.path).exists(_.last == "Hello.class")
-        )
-
-        val Right(_) = eval.apply(m.run()): @unchecked
-      }
-    }
-
-    test("compile & run Groovy JUnit5 test") {
-      testEval().scoped { eval =>
-
-        val Right(result) = eval.apply(m.test.compile): @unchecked
-
-        assert(
-          os.walk(result.value.classes.path).exists(_.last == "HelloTest.class")
-        )
-
-        val Right(discovered) = eval.apply(m.test.discoveredTestClasses): @unchecked
-        assert(discovered.value == Seq("hello.tests.HelloTest"))
-
-        val Right(_) = eval.apply(m.test.testForked()): @unchecked
-      }
-    }
-
-    test("compiling & running a statically compiled Groovy") {
-      testEval().scoped { eval =>
-        val Right(result) = eval.apply(m.staticcompile.compile): @unchecked
-        assert(
-          os.walk(result.value.classes.path).exists(_.last == "HelloStatic.class")
-        )
-        val Right(_) = eval.apply(m.staticcompile.run()): @unchecked
-      }
-    }
 
-    test("compile & run test-only Maven JUnit5 test") {
+    test("compile & test module (only test uses Groovy)") {
       testEval().scoped { eval =>
 
-        val Right(resultCompile) = eval.apply(mixed.compile): @unchecked
-        assert(
-          os.walk(resultCompile.value.classes.path).exists(_.last == "Greeter.class")
-        )
-
         val Right(_) = eval.apply(mixed.test.compile): @unchecked
         val Right(discovered) = eval.apply(mixed.test.discoveredTestClasses): @unchecked
-        assert(discovered.value == Seq("tests.GreeterTests"))
+        assert(discovered.value == Seq("hello.maven.tests.HelloMavenTestOnly"))
 
         val Right(_) = eval.apply(mixed.test.testForked()): @unchecked
       }
@@ -169,7 +123,20 @@ object HelloGroovyTests extends TestSuite {
       }
     }
 
+    test("compile joint (groovy <-> java cycle) & run") {
+      testEval().scoped { eval =>
+        val Right(result) = eval.apply(joint.compile): @unchecked
 
+        assert(
+          os.walk(result.value.classes.path).exists(_.last == "JavaPrinter.class")
+        )
+        assert(
+          os.walk(result.value.classes.path).exists(_.last == "GroovyGreeter.class")
+        )
+
+        val Right(_) = eval.apply(joint.run()): @unchecked
+      }
+    }
 
   }
 }

libs/groovylib/worker/src/mill/groovylib/worker/impl/GroovyWorkerImpl.scala

@@ -6,12 +6,46 @@ import mill.api.TaskCtx
 import mill.javalib.api.CompilationResult
 import mill.groovylib.worker.api.GroovyWorker
 import org.codehaus.groovy.control.{CompilationUnit, CompilerConfiguration, Phases}
+import org.codehaus.groovy.tools.javac.JavaStubCompilationUnit
+import os.Path
 
 import scala.jdk.CollectionConverters.*
 import scala.util.Try
 
 class GroovyWorkerImpl extends GroovyWorker {
 
+
+  override def compileGroovyStubs(sourceFiles: Seq[Path], classpath: Seq[Path], outputDir: Path)
+                                 (implicit ctx: TaskCtx): Result[CompilationResult] = {
+    val config = new CompilerConfiguration()
+    config.setTargetDirectory(outputDir.toIO)
+    config.setClasspathList(classpath.map(_.toIO.getAbsolutePath).asJava)
+    config.setJointCompilationOptions(Map(
+      "stubDir" -> outputDir.toIO,
+      "keepStubs" -> false
+    ).asJava)
+
+    // we need to set the classloader for groovy to use the worker classloader
+    val parentCl: ClassLoader = this.getClass.getClassLoader
+    // config in the GroovyClassLoader is needed when the CL itself is compiling classes
+    val gcl = new GroovyClassLoader(parentCl, config)
+    // config for actual compilation
+    val stubUnit = JavaStubCompilationUnit(config, gcl)
+
+    sourceFiles.foreach { sourceFile =>
+      stubUnit.addSource(sourceFile.toIO)
+    }
+
+    Try {
+      stubUnit.compile(Phases.CONVERSION)
+      CompilationResult(outputDir, mill.api.PathRef(outputDir))
+    }.fold(
+      exception => Result.Failure(s"Groovy stub generation failed: ${exception.getMessage}"),
+      result => Result.Success(result)
+    )
+
+  }
+
   def compile(
       sourceFiles: Seq[os.Path],
       classpath: Seq[os.Path],
@@ -20,12 +54,13 @@ class GroovyWorkerImpl extends GroovyWorker {
       ctx: TaskCtx
   ): Result[CompilationResult] = {
 
+    val extendedClasspath = classpath :+ outputDir
+
     val config = new CompilerConfiguration()
     config.setTargetDirectory(outputDir.toIO)
-    config.setClasspathList(classpath.map(_.toIO.getAbsolutePath).asJava)
+    config.setClasspathList(extendedClasspath.map(_.toIO.getAbsolutePath).asJava)
     // TODO
 //    config.setDisabledGlobalASTTransformations()
-//    config.setJointCompilationOptions()
 //    config.setSourceEncoding()
 
     // we need to set the classloader for groovy to use the worker classloader
@@ -41,10 +76,20 @@ class GroovyWorkerImpl extends GroovyWorker {
 
     Try {
       unit.compile(Phases.OUTPUT)
+      removeAllJavaFiles(outputDir)
       CompilationResult(outputDir, mill.api.PathRef(outputDir))
     }.fold(
       exception => Result.Failure(s"Groovy compilation failed: ${exception.getMessage}"),
       result => Result.Success(result)
     )
   }
+
+
+  private def removeAllJavaFiles(outputDir: os.Path): Unit = {
+    if (os.exists(outputDir)) {
+      os.walk(outputDir)
+        .filter(_.ext == "java")
+        .foreach(os.remove)
+    }
+  }
 }