[query] extract utils into new module

hail/build.mill

@@ -132,7 +132,7 @@ trait HailScalaModule extends ScalaModule, HailJavaModule, ScalafmtModule, Scala
           "-Wconf:cat=deprecation&msg=symbol literal is deprecated:s",
           raw"-Wconf:cat=unused-imports&origin=scala\.collection\.compat\._:s",
           raw"-Wconf:cat=unused-imports&origin=scala\.Option\.option2Iterable:s",
-          raw"-Wconf:cat=unused-imports&origin=is\.hail\.utils\.compat\._:s",
+          raw"-Wconf:cat=unused-imports&origin=is\.hail\.collection\.compat\._:s",
           raw"-Wconf:msg=unused value of type org\.scalatest\..*Assertion:s",
           raw"-Wconf:msg=unused value of type org\.mockito\.stubbing\.ScalaOngoingStubbing:s",
         )

hail/hail/ir-gen/src/Main.scala

@@ -1203,9 +1203,10 @@ object Main {
     val pack = "package is.hail.expr.ir.defs"
     val imports = Seq(
       "is.hail.annotations.Annotation",
+      "is.hail.collection.FastSeq",
       "is.hail.io.{AbstractTypedCodecSpec, BufferSpec}",
       "is.hail.types.virtual.{Type, TArray, TStream, TVoid, TStruct, TTuple}",
-      "is.hail.utils.{FastSeq, StringEscapeUtils}",
+      "is.hail.utils.StringEscapeUtils",
       "is.hail.expr.ir.{AggOp, BaseIR, IR, TableIR, MatrixIR, BlockMatrixIR, Name, UnaryOp, BinaryOp, " +
         "ComparisonOp, CanEmit, EmitParamType, TableWriter, " +
         "WrappedMatrixNativeMultiWriter, MatrixWriter, MatrixNativeMultiWriter, BlockMatrixWriter, " +

hail/hail/package.mill

@@ -25,6 +25,7 @@ trait RootHailModule extends CrossScalaModule, HailScalaModule:
 
   override def moduleDeps: Seq[JavaModule] =
     Seq(
+      build.hail.utils(crossValue),
       build.hail.memory(if buildMode == CI then "debug" else "release"),
     )
 

hail/hail/src/is/hail/HailFeatureFlags.scala

@@ -2,10 +2,10 @@ package is.hail
 
 import is.hail.backend.ExecutionCache
 import is.hail.backend.spark.SparkBackend
+import is.hail.collection.implicits.toRichIterable
 import is.hail.expr.ir.{agg, Optimize}
 import is.hail.io.fs.RequesterPaysConfig
 import is.hail.types.encoded.EType
-import is.hail.utils._
 
 import scala.collection.mutable
 

hail/hail/src/is/hail/annotations/Annotation.scala

@@ -1,5 +1,6 @@
 package is.hail.annotations
 
+import is.hail.collection.implicits.toRichIterable
 import is.hail.types.virtual._
 import is.hail.utils._
 

hail/hail/src/is/hail/annotations/BroadcastValue.scala

@@ -6,8 +6,9 @@ import is.hail.expr.ir.defs.EncodedLiteral
 import is.hail.io.{BufferSpec, Decoder, TypedCodecSpec}
 import is.hail.types.physical.{PArray, PStruct, PType}
 import is.hail.types.virtual.{TBaseStruct, TStruct}
-import is.hail.utils.{formatSpace, ArrayOfByteArrayOutputStream, Logging}
-import is.hail.utils.prettyPrint.ArrayOfByteArrayInputStream
+import is.hail.utils.{
+  formatSpace, ArrayOfByteArrayInputStream, ArrayOfByteArrayOutputStream, Logging,
+}
 
 import java.io.InputStream
 

hail/hail/src/is/hail/annotations/ChunkCache.scala

@@ -1,6 +1,6 @@
 package is.hail.annotations
 
-import is.hail.expr.ir.LongArrayBuilder
+import is.hail.collection.LongArrayBuilder
 
 import scala.collection.mutable
 

hail/hail/src/is/hail/annotations/ExtendedOrdering.scala

@@ -1,5 +1,6 @@
 package is.hail.annotations
 
+import is.hail.collection.implicits.toRichIterable
 import is.hail.utils._
 
 import org.apache.spark.sql.Row

hail/hail/src/is/hail/annotations/OrderedRVIterator.scala

@@ -1,10 +1,12 @@
 package is.hail.annotations
 
 import is.hail.backend.HailStateManager
+import is.hail.collection.{FlipbookIterator, StagingIterator, StateMachine}
+import is.hail.collection.compat.mutable.Growable
+import is.hail.collection.implicits.toRichIterator
 import is.hail.rvd.{RVDContext, RVDType}
 import is.hail.types.physical.PInterval
 import is.hail.utils._
-import is.hail.utils.compat.mutable.Growable
 
 import scala.collection.mutable
 

hail/hail/src/is/hail/annotations/RegionMemory.scala

@@ -1,7 +1,7 @@
 package is.hail.annotations
 
-import is.hail.expr.ir.{AnyRefArrayBuilder, LongArrayBuilder}
-import is.hail.utils._
+import is.hail.collection.{AnyRefArrayBuilder, LongArrayBuilder}
+import is.hail.utils.implicits.toRichBoolean
 
 final class RegionMemory(pool: RegionPool) extends AutoCloseable {
   private[this] val usedBlocks = new LongArrayBuilder(4)

hail/hail/src/is/hail/annotations/RegionPool.scala

@@ -1,6 +1,7 @@
 package is.hail.annotations
 
-import is.hail.expr.ir.LongArrayBuilder
+import is.hail.collection.LongArrayBuilder
+import is.hail.io.fs.readableBytes
 import is.hail.utils._
 
 import scala.collection.mutable

hail/hail/src/is/hail/annotations/RegionValueBuilder.scala

@@ -1,9 +1,11 @@
 package is.hail.annotations
 
 import is.hail.backend.HailStateManager
+import is.hail.collection.{IntArrayStack, LongArrayStack, ObjectArrayStack}
 import is.hail.types.physical._
 import is.hail.types.virtual._
 import is.hail.utils._
+import is.hail.utils.implicits.toRichBoolean
 
 class RegionValueBuilder(sm: HailStateManager, var region: Region) {
   def this(sm: HailStateManager) = this(sm, null)

hail/hail/src/is/hail/annotations/WritableRegionValue.scala

@@ -1,9 +1,9 @@
 package is.hail.annotations
 
 import is.hail.backend.HailStateManager
+import is.hail.collection.compat.mutable.GrowableCompat
 import is.hail.rvd.RVDContext
 import is.hail.types.physical.{PStruct, PType}
-import is.hail.utils.compat.mutable.GrowableCompat
 
 import scala.collection.mutable.{ArrayBuffer, PriorityQueue}
 

hail/hail/src/is/hail/asm4s/ClassBuilder.scala

@@ -1,8 +1,10 @@
 package is.hail.asm4s
 
+import is.hail.collection.FastSeq
 import is.hail.expr.ir.EmitCodeBuilder
 import is.hail.lir
 import is.hail.utils._
+import is.hail.utils.implicits.toRichBoolean
 
 import scala.collection.mutable
 

hail/hail/src/is/hail/asm4s/Code.scala

@@ -1,9 +1,10 @@
 package is.hail.asm4s
 
+import is.hail.collection.FastSeq
+import is.hail.collection.implicits.toRichIterable
 import is.hail.expr.ir.EmitCodeBuilder
 import is.hail.lir
 import is.hail.lir.{Block, ControlX, ValueX}
-import is.hail.utils._
 
 import scala.reflect.ClassTag
 

hail/hail/src/is/hail/asm4s/CodeBuilder.scala

@@ -1,7 +1,8 @@
 package is.hail.asm4s
 
 import is.hail.{lir, EmitTracebackInAssertions}
-import is.hail.utils.{toRichIterable, Traceback}
+import is.hail.collection.implicits.toRichIterable
+import is.hail.utils.Traceback
 
 import scala.collection.compat._
 

hail/hail/src/is/hail/utils/richUtils/RichCodeInputBuffer.scala → hail/hail/src/is/hail/asm4s/implicits/RichCodeInputBuffer.scala

@@ -1,4 +1,4 @@
-package is.hail.utils.richUtils
+package is.hail.asm4s.implicits
 
 import is.hail.annotations.Region
 import is.hail.asm4s.{Code, _}
@@ -7,7 +7,6 @@ import is.hail.io.InputBuffer
 import is.hail.types.physical.stypes.SValue
 import is.hail.types.physical.stypes.interfaces._
 import is.hail.types.virtual._
-import is.hail.utils._
 
 class RichCodeInputBuffer(
   val ib: Value[InputBuffer]

hail/hail/src/is/hail/utils/richUtils/RichCodeIterator.scala → hail/hail/src/is/hail/asm4s/implicits/RichCodeIterator.scala

@@ -1,4 +1,4 @@
-package is.hail.utils.richUtils
+package is.hail.asm4s.implicits
 
 import is.hail.asm4s.{Code, TypeInfo}
 

hail/hail/src/is/hail/utils/richUtils/RichCodeOutputBuffer.scala → hail/hail/src/is/hail/asm4s/implicits/RichCodeOutputBuffer.scala

@@ -1,4 +1,4 @@
-package is.hail.utils.richUtils
+package is.hail.asm4s.implicits
 
 import is.hail.annotations.Region
 import is.hail.asm4s._

hail/hail/src/is/hail/utils/richUtils/RichCodeRegion.scala → hail/hail/src/is/hail/asm4s/implicits/RichCodeRegion.scala

@@ -1,4 +1,4 @@
-package is.hail.utils.richUtils
+package is.hail.asm4s.implicits
 
 import is.hail.annotations.{Region, RegionMemory, RegionPool}
 import is.hail.asm4s._

hail/hail/src/is/hail/asm4s/implicits/package.scala

@@ -0,0 +1,24 @@
+package is.hail.asm4s
+
+import is.hail.annotations.Region
+import is.hail.io.{InputBuffer, OutputBuffer}
+
+package object implicits {
+  implicit def valueToRichCodeInputBuffer(in: Value[InputBuffer]): RichCodeInputBuffer =
+    new RichCodeInputBuffer(in)
+
+  implicit def valueToRichCodeOutputBuffer(out: Value[OutputBuffer]): RichCodeOutputBuffer =
+    new RichCodeOutputBuffer(out)
+
+  implicit def toRichCodeIterator[T](it: Code[Iterator[T]]): RichCodeIterator[T] =
+    new RichCodeIterator[T](it)
+
+  implicit def valueToRichCodeIterator[T](it: Value[Iterator[T]]): RichCodeIterator[T] =
+    new RichCodeIterator[T](it)
+
+  implicit def codeToRichCodeRegion(region: Code[Region]): RichCodeRegion =
+    new RichCodeRegion(region)
+
+  implicit def valueToRichCodeRegion(region: Value[Region]): RichCodeRegion =
+    new RichCodeRegion(region)
+}

hail/hail/src/is/hail/backend/BackendUtils.scala

@@ -2,6 +2,7 @@ package is.hail.backend
 
 import is.hail.annotations.Region
 import is.hail.asm4s._
+import is.hail.collection.implicits.toRichIndexedSeq
 import is.hail.expr.ir.analyses.SemanticHash
 import is.hail.expr.ir.lowering.TableStageDependency
 import is.hail.io.fs._

hail/hail/src/is/hail/backend/HailTaskContext.scala

@@ -30,12 +30,6 @@ abstract class HailTaskContext extends AutoCloseable with Logging {
 
   def getRegionPool(): RegionPool = thePool
 
-  def partSuffix(): String = {
-    val rng = new java.security.SecureRandom()
-    val fileUUID = new java.util.UUID(rng.nextLong(), rng.nextLong())
-    s"${stageId()}-${partitionId()}-${attemptNumber()}-$fileUUID"
-  }
-
   val finalizers = mutable.ArrayBuffer.empty[TaskFinalizer]
 
   def newFinalizer(): TaskFinalizer = {

hail/hail/src/is/hail/backend/driver/BackendRpc.scala

@@ -1,6 +1,7 @@
 package is.hail.backend.driver
 
 import is.hail.backend.{Backend, ExecuteContext}
+import is.hail.collection.FastSeq
 import is.hail.expr.ir.IRParser
 import is.hail.expr.ir.functions.IRFunctionRegistry
 import is.hail.expr.ir.functions.IRFunctionRegistry.UserDefinedFnKey
@@ -9,7 +10,7 @@ import is.hail.io.plink.LoadPlink
 import is.hail.io.vcf.LoadVCF
 import is.hail.types.virtual.{Kind, TFloat64}
 import is.hail.types.virtual.Kinds._
-import is.hail.utils.{jsonToBytes, using, ExecutionTimer, FastSeq}
+import is.hail.utils.{jsonToBytes, using, ExecutionTimer}
 import is.hail.utils.ExecutionTimer.Timings
 import is.hail.variant.ReferenceGenome
 

hail/hail/src/is/hail/backend/driver/BatchQueryDriver.scala

@@ -5,6 +5,8 @@ import is.hail.annotations.Memory
 import is.hail.asm4s.HailClassLoader
 import is.hail.backend.{Backend, ExecuteContext, OwningTempFileManager}
 import is.hail.backend.service._
+import is.hail.collection.ImmutableMap
+import is.hail.collection.implicits.toRichIterable
 import is.hail.expr.ir.lowering.IrMetadata
 import is.hail.io.fs.{CloudStorageFSConfig, FS, RouterFS}
 import is.hail.io.reference.{IndexedFastaSequenceFile, LiftOver}

hail/hail/src/is/hail/backend/driver/Py4JQueryDriver.scala

@@ -4,6 +4,8 @@ import is.hail.{linalg, HailFeatureFlags}
 import is.hail.asm4s.HailClassLoader
 import is.hail.backend._
 import is.hail.backend.spark.SparkBackend
+import is.hail.collection.FastSeq
+import is.hail.collection.implicits.toRichIterable
 import is.hail.expr.{JSONAnnotationImpex, SparkAnnotationImpex}
 import is.hail.expr.ir._
 import is.hail.expr.ir.IRParser.parseType
@@ -13,11 +15,13 @@ import is.hail.expr.ir.functions.IRFunctionRegistry
 import is.hail.expr.ir.lowering.IrMetadata
 import is.hail.io.fs._
 import is.hail.io.reference.{IndexedFastaSequenceFile, LiftOver}
+import is.hail.sparkextras.implicits._
 import is.hail.types.physical.PStruct
 import is.hail.types.virtual.{TArray, TInterval}
 import is.hail.types.virtual.Kinds.{BlockMatrix, Matrix, Table, Value}
 import is.hail.utils._
 import is.hail.utils.ExecutionTimer.Timings
+import is.hail.utils.implicits.toRichString
 import is.hail.variant.ReferenceGenome
 
 import scala.annotation.nowarn

hail/hail/src/is/hail/backend/local/LocalBackend.scala

@@ -2,14 +2,14 @@ package is.hail.backend.local
 
 import is.hail.backend._
 import is.hail.backend.Backend.PartitionFn
+import is.hail.collection.compat.immutable.ArraySeq
 import is.hail.expr.Validate
 import is.hail.expr.ir._
 import is.hail.expr.ir.analyses.SemanticHash
 import is.hail.expr.ir.lowering._
 import is.hail.types._
 import is.hail.types.physical.PTuple
 import is.hail.utils._
-import is.hail.utils.compat.immutable.ArraySeq
 
 import scala.reflect.ClassTag
 import scala.util.control.NonFatal

hail/hail/src/is/hail/backend/service/ServiceBackend.scala

@@ -5,6 +5,8 @@ import is.hail.backend._
 import is.hail.backend.Backend.PartitionFn
 import is.hail.backend.local.LocalTaskContext
 import is.hail.backend.service.ServiceBackend.MaxAvailableGcsConnections
+import is.hail.collection.FastSeq
+import is.hail.collection.compat.immutable.ArraySeq
 import is.hail.expr.Validate
 import is.hail.expr.ir.{
   CompileAndEvaluate, IR, IRSize, LoweringAnalyses, SortField, TableIR, TableReader, TypeCheck,
@@ -17,7 +19,6 @@ import is.hail.services.oauth2.{CloudCredentials, HailCredentials}
 import is.hail.types._
 import is.hail.types.physical._
 import is.hail.utils._
-import is.hail.utils.compat.immutable.ArraySeq
 
 import scala.collection.compat._
 import scala.concurrent.{Await, CancellationException, ExecutionContext, Future}

hail/hail/src/is/hail/backend/spark/SparkBackend.scala

@@ -4,6 +4,7 @@ import is.hail.annotations._
 import is.hail.asm4s._
 import is.hail.backend._
 import is.hail.backend.Backend.PartitionFn
+import is.hail.collection.compat.immutable.ArraySeq
 import is.hail.expr.Validate
 import is.hail.expr.ir._
 import is.hail.expr.ir.analyses.SemanticHash
@@ -14,7 +15,6 @@ import is.hail.rvd.RVD
 import is.hail.types._
 import is.hail.types.physical.{PStruct, PTuple}
 import is.hail.utils._
-import is.hail.utils.compat.immutable.ArraySeq
 
 import scala.collection.mutable
 import scala.concurrent.{CancellationException, ExecutionException}

hail/hail/src/is/hail/compatibility/LegacyEncodedTypeParser.scala

@@ -1,10 +1,10 @@
 package is.hail.compatibility
 
+import is.hail.collection.FastSeq
 import is.hail.expr.ir.{IRParser, PunctuationToken, TokenIterator}
 import is.hail.expr.ir.IRParser._
 import is.hail.types.encoded._
 import is.hail.types.virtual._
-import is.hail.utils.FastSeq
 
 object LegacyEncodedTypeParser {
 

hail/hail/src/is/hail/compatibility/LegacyRVDSpecs.scala

@@ -1,12 +1,13 @@
 package is.hail.compatibility
 
 import is.hail.backend.HailStateManager
+import is.hail.collection.FastSeq
 import is.hail.expr.JSONAnnotationImpex
 import is.hail.io._
 import is.hail.rvd.{AbstractRVDSpec, IndexSpec2, IndexedRVDSpec2, RVDPartitioner}
 import is.hail.types.encoded._
 import is.hail.types.virtual._
-import is.hail.utils.{FastSeq, Interval}
+import is.hail.utils.Interval
 
 import org.json4s.JValue