[query] fully define mill compile dependencies

Author: ehigham

Makefile

@@ -57,8 +57,10 @@ pylint-hailtop:
 	$(PYTHON) -m pylint --rcfile pylintrc hail/python/hailtop --score=n
 
 .PHONY: check-hail
+check-hail: export HAIL_BUILD_MODE=Dev
+check-hail: HAIL_DEPLOY_TARGET='spark_2.13:3.5.3'
 check-hail: check-hail-fast pylint-hailtop
-	cd hail && HAIL_BUILD_MODE=Dev SCALA_VERSION=2.13 $(MILL) $(MILLOPTS) hail[].__.checkFormat + hail[].__.fix --check
+	cd hail && $(MILL) $(MILLOPTS) hail[].__.checkFormat + hail[].__.fix --check
 
 .PHONY: check-batch
 check-batch: check-batch-fast pylint-batch

build.yaml

@@ -773,23 +773,18 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=Release
-      time retry sh mill --no-daemon --version
-
       # We've encountered the following sporadic error in CI between `mill`
       # finishing assembling the jar and `Make` copying the jar:
       #
       # cp -f out/assembly.dest/out.jar python/hail/backend/hail-all-spark.jar
       # cp: cannot stat 'out/assembly.dest/out.jar': No such file or directory
       #
-      # This was likely from running mill in server mode; mill main exits before
+      # This was caused by running mill in daemon mode; mill main exits before
       # the jar has finished being written.
-      time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
-          echo 'no out.jar found after mill assembly returned. going to sleep'
-          sleep 5
-      fi
+      export MILLOPTS='--no-daemon'
 
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x HAIL_BUILD_MODE=Release       
+      time retry sh mill --no-daemon --version
       time retry make DEV_CLARIFIER=hail-ci-build wheel
 
       # Check wheel size is small enough for pypi (< 200 MiB)
@@ -851,12 +846,12 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x HAIL_BUILD_MODE=CI MILLOPTS='--no-daemon'
       time retry sh mill --no-daemon --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -884,12 +879,12 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
+      export HAIL_DEPLOY_TARGET=dataproc-2.3.x MILLOPTS='--no-daemon' HAIL_BUILD_MODE=CI
       time retry sh mill --no-daemon --version
 
       # See `build_hail_jar_and_wheel`
       time retry make shadowTestJar
-      if [ ! -f out/hail/2.12/test/assembly.dest/out.jar ]; then
+      if [ ! -f "out/hail/${HAIL_DEPLOY_TARGET}/test/assembly.dest/out.jar" ]; then
           echo 'no out.jar found after mill assembly returned. going to sleep'
           sleep 5
       fi
@@ -944,16 +939,11 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      export SPARK_VERSION="3.0.2" SCALA_VERSION="2.12.13" HAIL_BUILD_MODE=Release MILLOPTS='--no-daemon'
+      export SPARK_VERSION="3.3.1" SCALA_VERSION="2.12.13"
+      
+      export MILLOPTS='--no-daemon'
+      export HAIL_DEPLOY_TARGET=hdinsight-5.1 HAIL_BUILD_MODE=Release
       time retry sh mill --no-daemon --version
-
-      # See `build_hail_jar_and_wheel`
-      time retry make shadowJar
-      if [ ! -f out/hail/2.12/assembly.dest/out.jar ]; then
-          echo 'no out.jar found after mill assembly returned. going to sleep'
-          sleep 5
-      fi
-
       time retry make wheel
     inputs:
       - from: /repo
@@ -1081,7 +1071,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /splits.tar.gz
         to: /io/splits.tar.gz
@@ -1691,7 +1681,7 @@ steps:
       - hail_linting_image
       - merge_code
   - kind: runImage
-    name: compile_hail_213
+    name: compile_hail_dataproc_3_0_x
     image:
       valueFrom: base_image.image
     resources:
@@ -1701,8 +1691,8 @@ steps:
       set -ex
       cd /io/repo/hail
 
-      HAIL_BUILD_MODE='CI'
-      time retry sh mill --no-daemon hail[2.13].__.compile
+      export HAIL_BUILD_MODE='CI' 
+      time retry sh mill --no-daemon 'hail[dataproc-3.0.x].__.compile'
     inputs:
       - from: /repo
         to: /io/repo
@@ -2573,7 +2563,7 @@ steps:
           valueFrom: default_ns.name
         mountPath: /batch-gsa-key
     inputs:
-      - from: /derived/release/hail/out/hail/2.12/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
         to: /io/hail.jar
       - from: /repo/hail/env/REVISION
         to: /io/REVISION
@@ -3499,7 +3489,7 @@ steps:
       # #14452 - must build wheel with release `hailtop/hailctl/deploy.yaml`
       # use `-o` to prevent `mill` from rebuilding the "SHADOW_JAR"
       make upload-artifacts HAIL_BUILD_MODE=Release DEPLOY_REMOTE=origin \
-        -o out/hail/2.12/assembly.dest/out.jar
+        -o shadowJar
 
       echo Setting arguments and invoking release.sh.
 
@@ -3524,8 +3514,8 @@ steps:
     inputs:
       - from: /repo
         to: /io/repo
-      - from: /derived/release/hail/out/hail/2.12/assembly.dest/out.jar
-        to: /io/repo/hail/out/hail/2.12/assembly.dest/out.jar
+      - from: /derived/release/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
+        to: /io/repo/hail/out/hail/dataproc-2.3.x/assembly.dest/out.jar
       - from: /azure-wheel
         to: /io/azure-wheel
       - from: /www.tar.gz
@@ -3769,7 +3759,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-fs.xml
         to: /io/testng-fs.xml
@@ -3811,7 +3801,7 @@ steps:
     inputs:
       - from: /resources.tar.gz
         to: /io/resources.tar.gz
-      - from: /derived/debug/hail/out/hail/2.12/test/assembly.dest/out.jar
+      - from: /derived/debug/hail/out/hail/dataproc-2.3.x/test/assembly.dest/out.jar
         to: /io/hail-test.jar
       - from: /repo/hail/testng-services.xml
         to: /io/testng-services.xml

hail/.scalafix.conf

@@ -31,5 +31,6 @@ RemoveUnused {
 }
 
 ForbiddenSymbol.symbols = [
+  "org.sfl4j",
   "scala.collection.compat.immutable.ArraySeq"
 ]

hail/Makefile

@@ -19,8 +19,7 @@ HAIL_PATCH_VERSION := 137
 HAIL_PIP_VERSION := $(HAIL_MAJOR_MINOR_VERSION).$(HAIL_PATCH_VERSION)
 HAIL_VERSION := $(HAIL_PIP_VERSION)-$(SHORT_REVISION)
 
-SCALA_VERSION ?= 2.12
-SPARK_VERSION ?= 3.5.3
+HAIL_DEPLOY_TARGET ?= dataproc-2.3.x
 
 include env_var.mk
 
@@ -64,7 +63,7 @@ CLOUD_HAIL_DOCTEST_DATA_DIR = $(CLOUD_HAIL_TEST_RESOURCES_PREFIX)/doctest/data/
 
 # mill integration
 
-MILL := SCALA_VERSION=$(SCALA_VERSION) bash mill
+MILL := bash mill
 MILLOPTS ?=
 
 .PHONY: mill-clean
@@ -77,7 +76,7 @@ mill-clean:
 # be recomputed.
 FORCE:
 
-SHADOW_JAR := out/hail/$(SCALA_VERSION)/assembly.dest/out.jar
+SHADOW_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_JAR): native-lib-prebuilt
 endif
@@ -87,14 +86,14 @@ $(SHADOW_JAR): FORCE
 .PHONY: shadowJar
 shadowJar: $(SHADOW_JAR)
 
-NON_SHADOW_JAR := out/hail/$(SCALA_VERSION)/jar.dest/out.jar
+NON_SHADOW_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/jar.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(NON_SHADOW_JAR): native-lib-prebuilt
 endif
 $(NON_SHADOW_JAR): FORCE
 	$(MILL) $(MILLOPTS) hail[].jar
 
-SHADOW_TEST_JAR := out/hail/$(SCALA_VERSION)/test/assembly.dest/out.jar
+SHADOW_TEST_JAR := out/hail/$(HAIL_DEPLOY_TARGET)/test/assembly.dest/out.jar
 ifdef HAIL_COMPILE_NATIVES
 $(SHADOW_TEST_JAR): native-lib-prebuilt
 endif
@@ -104,7 +103,7 @@ $(SHADOW_TEST_JAR): FORCE
 .PHONY: shadowTestJar
 shadowTestJar: $(SHADOW_TEST_JAR)
 
-EXTRA_CLASSPATH := out/hail/$(SCALA_VERSION)/writeRunClasspath.dest/runClasspath
+EXTRA_CLASSPATH := out/hail/$(HAIL_DEPLOY_TARGET)/writeRunClasspath.dest/runClasspath
 $(EXTRA_CLASSPATH): FORCE
 	$(MILL) $(MILLOPTS) hail[].writeRunClasspath
 

hail/build.mill

@@ -1,5 +1,5 @@
-//| mill-version: 1.0.4
-//| mill-jvm-version: 11
+//| mill-version: 1.1.0-RC3
+//| mill-jvm-version: 17
 package build
 
 import mill.*
@@ -17,14 +17,7 @@ import org.typelevel.scalacoptions.*
 object Settings:
   val hailMajorMinorVersion = "0.2"
   val hailPatchVersion = "137"
-  val scalaMinorVersions = Map("2.12" -> "2.12.20", "2.12.13" -> "2.12.13", "2.13" -> "2.13.13")
-
-
-object Env extends Module:
-  def sparkVersion: T[String] =
-    Task.Input {
-      Task.env.getOrElse("SPARK_VERSION", "3.5.3")
-    }
+  val scalaMinorVersions = Map("2.12" -> "2.12.20", "2.13" -> "2.13.18")
 
 
 trait HailJavaModule extends JavaModule:
@@ -35,7 +28,7 @@ trait HailJavaModule extends JavaModule:
       else Seq()
 
     val `optim?` =
-      if buildMode == Release
+      if BuildMode == Release
       then Seq("-O")
       else Seq("-g")
 
@@ -85,10 +78,10 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala
         )
 
       val fromBuildMode =
-        buildMode match {
+        BuildMode match {
           case Dev =>
             ScalacOptions.verboseOptions
-  //            + ScalacOptions.checkInit // enable to help debug accessing uninitialized fields
+//              + ScalacOptions.checkInit // enable to help debug accessing uninitialized fields
           case CI =>
             Set(ScalacOptions.fatalWarnings)
           case Release =>
@@ -163,5 +156,6 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala
       richOptions ++ rawOptions
     }
 
-  trait HailScalaTests extends ScalaTests, HailScalaModule
-
+  trait HailScalaTests extends ScalaTests, HailScalaModule:
+    override def enableBsp: Boolean =
+      outer.enableBsp