Skip to content

Commit b294921

Browse files
ehighamehigham
committed
[query] use mill multi-file builds
1 parent 733ac4c commit b294921

File tree

9 files changed

+894
-444
lines changed

9 files changed

+894
-444
lines changed

‎hail/build.mill‎

Lines changed: 129 additions & 401 deletions
Large diffs are not rendered by default.

‎hail/hail/ir-gen/package.mill‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package build.hail.`ir-gen`
2+
3+
import mill.{Args, DefaultTaskModule, PathRef, T, Task}
4+
import mill.scalalib.Dep
5+
6+
import millbuild.MvnCoordinate.*
7+
8+
object `package` extends build.HailScalaModule, DefaultTaskModule:
9+
10+
override def scalaVersion: T[String] =
11+
"2.13.18"
12+
13+
override def defaultTask(): String =
14+
"generate"
15+
16+
override def mvnDeps: T[Seq[Dep]] =
17+
Seq[Dep](
18+
`mainargs` :: "0.7.7",
19+
`os-lib` :: "0.10.7",
20+
`sourcecode` :: "0.4.2",
21+
`scala-collection-compat`,
22+
)
23+
24+
def generate: T[PathRef] =
25+
Task {
26+
runner().run(Args("--path", Task.dest).value)
27+
PathRef(Task.dest)
28+
}

‎hail/hail/memory/package.mill‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package build.hail.memory
2+
3+
import mill.{Cross, T}
4+
5+
object `package` extends Cross[Memory]("debug", "release")
6+
trait Memory extends build.HailJavaModule, Cross.Module[String]:
7+
8+
override def moduleDir: os.Path =
9+
super.moduleDir / crossValue
10+
11+
override def javacOptions: T[Seq[String]] =
12+
super.javacOptions() ++ (
13+
if scala.util.Properties.versionString.startsWith("1.8")
14+
then Seq("-XDenableSunApiLintControl", "-Xlint:-sunapi")
15+
else Seq()
16+
)

‎hail/hail/package.mill‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
package build.hail
2+
3+
import mill.*
4+
import mill.api.{BuildCtx, Result}
5+
import mill.scalalib.*
6+
import mill.scalalib.Assembly.*
7+
import mill.scalalib.TestModule.TestNg
8+
import mill.util.{Jvm, VcsVersion}
9+
10+
import build.Env
11+
import build.Settings
12+
import build.HailScalaModule
13+
import millbuild.BuildConfig.*
14+
import millbuild.BuildMode.*
15+
import millbuild.MvnCoordinate.*
16+
17+
object `package` extends Cross[RootHailModule](enabledScalaVersions)
18+
19+
trait RootHailModule extends CrossScalaModule, HailScalaModule:
20+
outer =>
21+
22+
override def crossScalaVersion: String =
23+
Settings.scalaMinorVersions(crossValue)
24+
25+
def buildInfo: T[PathRef] = Task {
26+
val revision = VcsVersion.vcsState().currentRevision
27+
os.write(
28+
Task.dest / "build-info.properties",
29+
s"""[Build Metadata]
30+
|revision=$revision
31+
|sparkVersion=${Env.sparkVersion()}
32+
|hailPipVersion=${Settings.hailMajorMinorVersion}.${Settings.hailPatchVersion}
33+
|hailBuildConfiguration=${buildMode}
34+
|""".stripMargin,
35+
)
36+
PathRef(Task.dest)
37+
}
38+
39+
override def moduleDeps: Seq[JavaModule] =
40+
Seq(
41+
build.hail.memory(if buildMode == CI then "debug" else "release"),
42+
)
43+
44+
override def resources: T[Seq[PathRef]] =
45+
super.resources() ++ Seq(
46+
BuildCtx.withFilesystemCheckerDisabled {
47+
PathRef(BuildCtx.workspaceRoot / "prebuilt" / "lib")
48+
},
49+
buildInfo(),
50+
)
51+
52+
override def generatedSources: T[Seq[PathRef]] =
53+
Task {
54+
Seq(build.hail.`ir-gen`.generate())
55+
}
56+
57+
override def unmanagedClasspath: T[Seq[PathRef]] =
58+
Seq(build.hail.shadeazure.assembly())
59+
60+
// omit unmanagedClasspath from the jar
61+
override def jar: T[PathRef] =
62+
Task {
63+
val jar = Task.dest / "out.jar"
64+
Jvm.createJar(jar, (resources() ++ Seq(compile().classes)).map(_.path).filter(os.exists), manifest())
65+
PathRef(jar)
66+
}
67+
68+
override def bomMvnDeps: T[Seq[Dep]] =
69+
Seq(
70+
`asm-bom` :: "9.9",
71+
`libraries-bom` :: "26.66.0",
72+
`log4j-bom` :: "2.22.0",
73+
)
74+
75+
override def compileMvnDeps: T[Seq[Dep]] =
76+
Seq[Dep](
77+
// WARNING WARNING WARNING
78+
// Before changing the breeze version review:
79+
// - https://hail.zulipchat.com/#narrow/stream/123011-Hail-Query-Dev/topic/new.20spark.20ndarray.20failures/near/41645
80+
// - https://github.com/hail-is/hail/pull/11555
81+
`breeze` :: "1.1",
82+
`avro` :: "1.11.2",
83+
`commons-codec` :: "1.16.1",
84+
`commons-io` :: "2.16.1",
85+
`commons-lang3` :: "3.12.0",
86+
`commons-math3` :: "3.6.1",
87+
`hadoop-client-api` :: "3.3.4",
88+
`httpclient` :: "4.5.14",
89+
`jackson-core` :: "2.15.2",
90+
`json4s-jackson` :: "3.7.0-M11",
91+
`lz4-java` :: "1.8.0",
92+
`log4j-api`,
93+
`log4j-core`,
94+
`zstd-jni` :: "1.5.5-4",
95+
`scalac-compat-annotation` :: "0.1.4",
96+
(`spark-core` :: Env.sparkVersion()).excludeOrg("org.scalanlp"),
97+
(`spark-mllib` :: Env.sparkVersion()).excludeOrg("org.scalanlp"),
98+
)
99+
100+
override def mvnDeps: T[Seq[Dep]] =
101+
Seq[Dep](
102+
`all` :: "1.1.2",
103+
`asm`,
104+
`asm-analysis`,
105+
`asm-util`,
106+
`google-auth-library-oauth2-http`
107+
.mvn
108+
.excludeOrg(
109+
"commons-codec",
110+
"org.apache.httpcomponents",
111+
"org.slf4j",
112+
),
113+
`google-cloud-storage`
114+
.mvn
115+
.excludeOrg(
116+
"com.fasterxml.jackson.core",
117+
"commons-codec",
118+
"org.slf4j",
119+
),
120+
(`elasticsearch-spark` :: "9.2.1")
121+
.excludeOrg(
122+
"commons-logging",
123+
"org.apache.spark",
124+
"org.slf4j",
125+
),
126+
`freemarker` :: "2.3.31",
127+
(`htsjdk` :: "3.0.5").excludeOrg("*"),
128+
(`jdistlib` :: "0.4.5").excludeOrg("*"),
129+
`jna` :: "5.13.0",
130+
(`log4j-api-scala` :: "13.1.0").excludeOrg("*"),
131+
`scala-collection-compat`,
132+
`sourcecode` :: "0.4.2",
133+
)
134+
135+
override def runMvnDeps: T[Seq[Dep]] =
136+
Seq(
137+
(`breeze-natives` :: "1.1")
138+
.excludeOrg(
139+
"org.apache.commons",
140+
"org.slf4j",
141+
),
142+
`junixsocket-core` :: "2.6.1",
143+
)
144+
145+
override def assemblyRules: Seq[Rule] =
146+
super.assemblyRules ++ Seq(
147+
Rule.Exclude("META-INF/INDEX.LIST"),
148+
Rule.ExcludePattern("^scala/(?!collection/compat).*"),
149+
Rule.AppendPattern("META-INF/services/.*", "\n"),
150+
Rule.Relocate("breeze.**", "is.hail.relocated.@0"),
151+
Rule.Relocate("com.google.**", "is.hail.relocated.@0"),
152+
Rule.Relocate("scala.collection.compat.**", "is.hail.relocated.@0"),
153+
)
154+
155+
override def scalacPluginMvnDeps: T[Seq[Dep]] =
156+
Seq(
157+
`better-monadic-for` :: "0.3.1",
158+
)
159+
160+
def writeRunClasspath: T[PathRef] =
161+
Task {
162+
os.write(
163+
Task.dest / "runClasspath",
164+
runClasspath().map(_.path).mkString(":"),
165+
)
166+
PathRef(Task.dest)
167+
}
168+
169+
object test extends HailScalaTests, TestNg, CrossValue:
170+
override def forkArgs: T[Seq[String]] =
171+
Seq("-Xss4m", "-Xmx4096M")
172+
173+
override def compileMvnDeps: T[Seq[Dep]] =
174+
outer.compileMvnDeps()
175+
176+
override def mvnDeps: T[Seq[Dep]] =
177+
outer.mvnDeps() ++ Seq(
178+
`guice` :: "5.1.0",
179+
`mockito-scala` :: "1.17.31",
180+
`scalacheck` :: "1.18.1",
181+
`scalacheck-1-18` :: "3.2.19.0",
182+
`scalatest` :: "3.2.19",
183+
`scalatest-shouldmatchers` :: "3.2.19",
184+
`testng-7-10` :: "3.2.19.0",
185+
)
186+
187+
override def runMvnDeps: T[Seq[Dep]] =
188+
outer.runMvnDeps()
189+
190+
override def assemblyRules: Seq[Rule] =
191+
outer.assemblyRules

‎hail/hail/shadeazure/package.mill‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package build.hail.shadeazure
2+
3+
import mill.T
4+
import mill.javalib.{Dep, JavaModule}
5+
import mill.javalib.Assembly.Rule
6+
7+
import millbuild.MvnCoordinate.*
8+
9+
object `package` extends JavaModule:
10+
11+
override def mvnDeps: T[Seq[Dep]] =
12+
Seq(
13+
`azure-core-http-netty` :: "1.13.7",
14+
`azure-identity` :: "1.8.3",
15+
`azure-storage-blob` :: "12.22.0",
16+
)
17+
18+
override def assemblyRules: Seq[Rule] =
19+
Seq(
20+
Rule.ExcludePattern("META-INF/*.RSA"),
21+
Rule.ExcludePattern("META-INF/*.SF"),
22+
Rule.ExcludePattern("META-INF/*.DSA"),
23+
Rule.Relocate("com.azure.**", "is.hail.shadedazure.@0"),
24+
Rule.Relocate("com.ctc.**", "is.hail.shadedazure.@0"),
25+
Rule.Relocate("com.fasterxml.**", "is.hail.shadedazure.@0"),
26+
Rule.Relocate("com.microsoft.**", "is.hail.shadedazure.@0"),
27+
Rule.Relocate("com.nimbusds.**", "is.hail.shadedazure.@0"),
28+
Rule.Relocate("com.sun.**", "is.hail.shadedazure.@0"),
29+
Rule.Relocate("io.netty.**", "is.hail.shadedazure.@0"),
30+
Rule.Relocate("is.hail.**", "is.hail.shadedazure.@0"),
31+
Rule.Relocate("net.jcip.**", "is.hail.shadedazure.@0"),
32+
Rule.Relocate("net.minidev.**", "is.hail.shadedazure.@0"),
33+
Rule.Relocate("org.apache.**", "is.hail.shadedazure.@0"),
34+
Rule.Relocate("org.codehaus.**", "is.hail.shadedazure.@0"),
35+
Rule.Relocate("org.objectweb.**", "is.hail.shadedazure.@0"),
36+
Rule.Relocate("org.reactivestreams.**", "is.hail.shadedazure.@0"),
37+
Rule.Relocate("org.slf4j.**", "is.hail.shadedazure.@0"),
38+
Rule.Relocate("reactor.adapter.**", "is.hail.shadedazure.@0"),
39+
Rule.Relocate("reactor.core.**", "is.hail.shadedazure.@0"),
40+
Rule.Relocate("reactor.netty.**", "is.hail.shadedazure.@0"),
41+
Rule.Relocate("reactor.util.**", "is.hail.shadedazure.@0"),
42+
)

0 commit comments

Comments
 (0)