Skip to content

Commit bae5d57

Browse files
ehighamehigham
committed
[query] use mill multi-file builds
1 parent 733ac4c commit bae5d57

File tree

9 files changed

+898
-444
lines changed

9 files changed

+898
-444
lines changed

‎hail/build.mill‎

Lines changed: 129 additions & 401 deletions
Large diffs are not rendered by default.

‎hail/hail/ir-gen/package.mill‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package build.hail.`ir-gen`
2+
3+
import mill.{Args, DefaultTaskModule, PathRef, T, Task}
4+
import mill.scalalib.Dep
5+
6+
import millbuild.MvnCoordinate.*
7+
8+
object `package` extends build.HailScalaModule, DefaultTaskModule:
9+
10+
override def scalaVersion: T[String] =
11+
"2.13.18"
12+
13+
override def defaultTask(): String =
14+
"generate"
15+
16+
override def mvnDeps: T[Seq[Dep]] =
17+
Seq[Dep](
18+
`mainargs` :: "0.7.7",
19+
`os-lib` :: "0.10.7",
20+
`sourcecode` :: "0.4.2",
21+
`scala-collection-compat`,
22+
)
23+
24+
def generate: T[PathRef] =
25+
Task {
26+
runner().run(Args("--path", Task.dest).value)
27+
PathRef(Task.dest)
28+
}

‎hail/hail/memory/package.mill‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package build.hail.memory
2+
3+
import mill.{Cross, T}
4+
5+
object `package` extends Cross[Memory]("debug", "release")
6+
trait Memory extends build.HailJavaModule, Cross.Module[String]:
7+
8+
override def moduleDir: os.Path =
9+
super.moduleDir / crossValue
10+
11+
override def javacOptions: T[Seq[String]] =
12+
super.javacOptions() ++ (
13+
if scala.util.Properties.versionString.startsWith("1.8")
14+
then Seq("-XDenableSunApiLintControl", "-Xlint:-sunapi")
15+
else Seq()
16+
)

‎hail/hail/package.mill‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
package build.hail
2+
3+
import mill.*
4+
import mill.api.{BuildCtx, Result}
5+
import mill.scalalib.*
6+
import mill.scalalib.Assembly.*
7+
import mill.scalalib.TestModule.TestNg
8+
import mill.util.{Jvm, VcsVersion}
9+
10+
import build.Env
11+
import build.Settings
12+
import build.HailScalaModule
13+
14+
import millbuild.BuildConfig.*
15+
import millbuild.BuildMode.*
16+
import millbuild.MvnCoordinate.*
17+
18+
object `package` extends Cross[RootHailModule](enabledScalaVersions)
19+
20+
trait RootHailModule extends CrossScalaModule, HailScalaModule:
21+
outer =>
22+
23+
override def crossScalaVersion: String =
24+
Settings.scalaMinorVersions(crossValue)
25+
26+
def buildInfo: T[PathRef] = Task {
27+
val revision = VcsVersion.vcsState().currentRevision
28+
os.write(
29+
Task.dest / "build-info.properties",
30+
s"""[Build Metadata]
31+
|revision=$revision
32+
|sparkVersion=${Env.sparkVersion()}
33+
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
34+
|hailBuildConfiguration=${buildMode.toString.toLowerCase}
35+
|""".stripMargin,
36+
)
37+
PathRef(Task.dest)
38+
}
39+
40+
override def moduleDeps: Seq[JavaModule] =
41+
Seq(
42+
build.hail.memory(if buildMode == CI then "debug" else "release"),
43+
)
44+
45+
override def resources: T[Seq[PathRef]] =
46+
super.resources() ++ Seq(
47+
BuildCtx.withFilesystemCheckerDisabled {
48+
PathRef(BuildCtx.workspaceRoot / "prebuilt" / "lib")
49+
},
50+
buildInfo(),
51+
)
52+
53+
override def generatedSources: T[Seq[PathRef]] =
54+
Task {
55+
Seq(build.hail.`ir-gen`.generate())
56+
}
57+
58+
override def unmanagedClasspath: T[Seq[PathRef]] =
59+
Seq(build.hail.shadeazure.assembly())
60+
61+
// omit unmanagedClasspath from the jar
62+
override def jar: T[PathRef] =
63+
Task {
64+
val jar = Task.dest / "out.jar"
65+
Jvm.createJar(jar, (resources() ++ Seq(compile().classes)).map(_.path).filter(os.exists), manifest())
66+
PathRef(jar)
67+
}
68+
69+
override def bomMvnDeps: T[Seq[Dep]] =
70+
Seq(
71+
`asm-bom` :: "9.9",
72+
`libraries-bom` :: "26.66.0",
73+
`log4j-bom` :: "2.22.0",
74+
)
75+
76+
override def compileMvnDeps: T[Seq[Dep]] =
77+
Seq[Dep](
78+
// WARNING WARNING WARNING
79+
// Before changing the breeze version review:
80+
// - https://hail.zulipchat.com/#narrow/stream/123011-Hail-Query-Dev/topic/new.20spark.20ndarray.20failures/near/41645
81+
// - https://github.com/hail-is/hail/pull/11555
82+
`breeze` :: "1.1",
83+
`avro` :: "1.11.2",
84+
`commons-codec` :: "1.16.1",
85+
`commons-io` :: "2.16.1",
86+
`commons-lang3` :: "3.12.0",
87+
`commons-math3` :: "3.6.1",
88+
`hadoop-client-api` :: "3.3.4",
89+
`httpclient` :: "4.5.14",
90+
`jackson-core` :: "2.15.2",
91+
`json4s-jackson` :: "3.7.0-M11",
92+
`lz4-java` :: "1.8.0",
93+
`log4j-api`,
94+
`log4j-core`,
95+
`zstd-jni` :: "1.5.5-4",
96+
`scalac-compat-annotation` :: "0.1.4",
97+
(`spark-core` :: Env.sparkVersion()).excludeOrg("org.scalanlp"),
98+
(`spark-mllib` :: Env.sparkVersion()).excludeOrg("org.scalanlp"),
99+
)
100+
101+
override def mvnDeps: T[Seq[Dep]] =
102+
Seq[Dep](
103+
`all` :: "1.1.2",
104+
`asm`,
105+
`asm-analysis`,
106+
`asm-util`,
107+
`google-auth-library-oauth2-http`
108+
.mvn
109+
.excludeOrg(
110+
"commons-codec",
111+
"org.apache.httpcomponents",
112+
"org.slf4j",
113+
),
114+
`google-cloud-storage`
115+
.mvn
116+
.excludeOrg(
117+
"com.fasterxml.jackson.core",
118+
"commons-codec",
119+
"org.slf4j",
120+
),
121+
(`elasticsearch-spark` :: "9.2.1")
122+
.excludeOrg(
123+
"commons-logging",
124+
"org.apache.spark",
125+
"org.slf4j",
126+
),
127+
`freemarker` :: "2.3.31",
128+
(`htsjdk` :: "3.0.5").excludeOrg("*"),
129+
(`jdistlib` :: "0.4.5").excludeOrg("*"),
130+
`jna` :: "5.13.0",
131+
(`log4j-api-scala` :: "13.1.0").excludeOrg("*"),
132+
`scala-collection-compat`,
133+
`sourcecode` :: "0.4.2",
134+
)
135+
136+
override def runMvnDeps: T[Seq[Dep]] =
137+
Seq(
138+
(`breeze-natives` :: "1.1")
139+
.excludeOrg(
140+
"org.apache.commons",
141+
"org.slf4j",
142+
),
143+
`junixsocket-core` :: "2.6.1",
144+
)
145+
146+
override def assemblyRules: Seq[Rule] =
147+
super.assemblyRules ++ Seq(
148+
Rule.Exclude("META-INF/INDEX.LIST"),
149+
Rule.ExcludePattern("^scala/(?!collection/compat).*"),
150+
Rule.AppendPattern("META-INF/services/.*", "\n"),
151+
Rule.Relocate("breeze.**", "is.hail.relocated.@0"),
152+
Rule.Relocate("com.google.**", "is.hail.relocated.@0"),
153+
Rule.Relocate("scala.collection.compat.**", "is.hail.relocated.@0"),
154+
)
155+
156+
override def scalacPluginMvnDeps: T[Seq[Dep]] =
157+
Seq(
158+
`better-monadic-for` :: "0.3.1",
159+
)
160+
161+
def writeRunClasspath: T[PathRef] =
162+
Task {
163+
os.write(
164+
Task.dest / "runClasspath",
165+
runClasspath().map(_.path).mkString(":"),
166+
)
167+
PathRef(Task.dest)
168+
}
169+
170+
object test extends HailScalaTests, TestNg, CrossValue:
171+
override def forkArgs: T[Seq[String]] =
172+
Seq("-Xss4m", "-Xmx4096M")
173+
174+
override def compileMvnDeps: T[Seq[Dep]] =
175+
outer.compileMvnDeps()
176+
177+
override def mvnDeps: T[Seq[Dep]] =
178+
outer.mvnDeps() ++ Seq(
179+
`guice` :: "5.1.0",
180+
`mockito-scala` :: "1.17.31",
181+
`scalacheck` :: "1.18.1",
182+
`scalacheck-1-18` :: "3.2.19.0",
183+
`scalatest` :: "3.2.19",
184+
`scalatest-shouldmatchers` :: "3.2.19",
185+
`testng-7-10` :: "3.2.19.0",
186+
)
187+
188+
override def runMvnDeps: T[Seq[Dep]] =
189+
outer.runMvnDeps()
190+
191+
override def assemblyRules: Seq[Rule] =
192+
outer.assemblyRules

‎hail/hail/shadeazure/package.mill‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package build.hail.shadeazure
2+
3+
import mill.T
4+
import mill.javalib.{Dep, JavaModule}
5+
import mill.javalib.Assembly.Rule
6+
7+
import millbuild.MvnCoordinate.*
8+
9+
object `package` extends JavaModule:
10+
11+
override def mvnDeps: T[Seq[Dep]] =
12+
Seq(
13+
`azure-core-http-netty` :: "1.13.7",
14+
`azure-identity` :: "1.8.3",
15+
`azure-storage-blob` :: "12.22.0",
16+
)
17+
18+
override def assemblyRules: Seq[Rule] =
19+
Seq(
20+
Rule.ExcludePattern("META-INF/*.RSA"),
21+
Rule.ExcludePattern("META-INF/*.SF"),
22+
Rule.ExcludePattern("META-INF/*.DSA"),
23+
Rule.Relocate("com.azure.**", "is.hail.shadedazure.@0"),
24+
Rule.Relocate("com.ctc.**", "is.hail.shadedazure.@0"),
25+
Rule.Relocate("com.fasterxml.**", "is.hail.shadedazure.@0"),
26+
Rule.Relocate("com.microsoft.**", "is.hail.shadedazure.@0"),
27+
Rule.Relocate("com.nimbusds.**", "is.hail.shadedazure.@0"),
28+
Rule.Relocate("com.sun.**", "is.hail.shadedazure.@0"),
29+
Rule.Relocate("io.netty.**", "is.hail.shadedazure.@0"),
30+
Rule.Relocate("is.hail.**", "is.hail.shadedazure.@0"),
31+
Rule.Relocate("net.jcip.**", "is.hail.shadedazure.@0"),
32+
Rule.Relocate("net.minidev.**", "is.hail.shadedazure.@0"),
33+
Rule.Relocate("org.apache.**", "is.hail.shadedazure.@0"),
34+
Rule.Relocate("org.codehaus.**", "is.hail.shadedazure.@0"),
35+
Rule.Relocate("org.objectweb.**", "is.hail.shadedazure.@0"),
36+
Rule.Relocate("org.reactivestreams.**", "is.hail.shadedazure.@0"),
37+
Rule.Relocate("org.slf4j.**", "is.hail.shadedazure.@0"),
38+
Rule.Relocate("reactor.adapter.**", "is.hail.shadedazure.@0"),
39+
Rule.Relocate("reactor.core.**", "is.hail.shadedazure.@0"),
40+
Rule.Relocate("reactor.netty.**", "is.hail.shadedazure.@0"),
41+
Rule.Relocate("reactor.util.**", "is.hail.shadedazure.@0"),
42+
)

0 commit comments

Comments
 (0)