Initial port to Geotrellis 3.0

Author: metasim

bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala

@@ -46,7 +46,7 @@ class CatalystSerializerBench extends SparkEnv {
 
   @Setup(Level.Trial)
   def setupData(): Unit = {
-    crsEnc = StandardEncoders.crsEncoder.resolveAndBind()
+    crsEnc = StandardEncoders.crsSparkEncoder.resolveAndBind()
   }
 
   @Benchmark

bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala

@@ -23,6 +23,7 @@ package org.locationtech.rasterframes.bench
 
 import java.util.concurrent.TimeUnit
 
+import geotrellis.raster.Dimensions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.rf.TileUDT
 import org.locationtech.rasterframes.tiles.InternalRowTile
@@ -56,7 +57,7 @@ class TileCellScanBench extends SparkEnv {
   @Benchmark
   def deserializeRead(): Double  = {
     val tile = TileType.deserialize(tileRow)
-    val (cols, rows) = tile.dimensions
+    val Dimensions(cols, rows) = tile.dimensions
     tile.getDouble(cols - 1, rows - 1) +
       tile.getDouble(cols/2, rows/2) +
       tile.getDouble(0, 0)

build.sbt

@@ -50,17 +50,20 @@ lazy val core = project
   .settings(
     moduleName := "rasterframes",
     libraryDependencies ++= Seq(
+      `slf4j-api`,
       shapeless,
       `jts-core`,
+      `spray-json`,
       geomesa("z3").value,
       geomesa("spark-jts").value,
-      `geotrellis-contrib-vlm`,
-      `geotrellis-contrib-gdal`,
+//      `geotrellis-contrib-vlm`,
+//      `geotrellis-contrib-gdal`,
       spark("core").value % Provided,
       spark("mllib").value % Provided,
       spark("sql").value % Provided,
       geotrellis("spark").value,
       geotrellis("raster").value,
+      geotrellis("gdal").value,
       geotrellis("s3").value,
       geotrellis("spark-testkit").value % Test excludeAll (
         ExclusionRule(organization = "org.scalastic"),

core/src/main/scala/org/locationtech/rasterframes/PairRDDConverter.scala

@@ -23,7 +23,7 @@ package org.locationtech.rasterframes
 
 import org.locationtech.rasterframes.util._
 import geotrellis.raster.{MultibandTile, Tile, TileFeature}
-import geotrellis.spark.{SpaceTimeKey, SpatialKey}
+import geotrellis.layer._
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql._
 import org.apache.spark.sql.rf.TileUDT

core/src/main/scala/org/locationtech/rasterframes/StandardColumns.scala

@@ -25,7 +25,7 @@ import java.sql.Timestamp
 
 import geotrellis.proj4.CRS
 import geotrellis.raster.Tile
-import geotrellis.spark.{SpatialKey, TemporalKey}
+import geotrellis.layer._
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.functions.col
 import org.locationtech.jts.geom.{Point => jtsPoint, Polygon => jtsPolygon}

core/src/main/scala/org/locationtech/rasterframes/encoders/ProjectedExtentEncoder.scala

@@ -32,6 +32,6 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
  */
 object ProjectedExtentEncoder {
   def apply(): ExpressionEncoder[ProjectedExtent] = {
-    DelegatingSubfieldEncoder("extent" -> extentEncoder, "crs" -> crsEncoder)
+    DelegatingSubfieldEncoder("extent" -> extentEncoder, "crs" -> crsSparkEncoder)
   }
 }

core/src/main/scala/org/locationtech/rasterframes/encoders/StandardEncoders.scala

@@ -28,8 +28,7 @@ import org.locationtech.rasterframes.stats.{CellHistogram, CellStatistics, Local
 import org.locationtech.jts.geom.Envelope
 import geotrellis.proj4.CRS
 import geotrellis.raster.{CellSize, CellType, Raster, Tile, TileLayout}
-import geotrellis.spark.tiling.LayoutDefinition
-import geotrellis.spark.{KeyBounds, SpaceTimeKey, SpatialKey, TemporalKey, TemporalProjectedExtent, TileLayerMetadata}
+import geotrellis.layer._
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.{Encoder, Encoders}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
@@ -53,7 +52,7 @@ trait StandardEncoders extends SpatialEncoders {
   implicit def singlebandTileEncoder: ExpressionEncoder[Tile] = ExpressionEncoder()
   implicit def rasterEncoder: ExpressionEncoder[Raster[Tile]] = ExpressionEncoder()
   implicit def tileLayerMetadataEncoder[K: TypeTag]: ExpressionEncoder[TileLayerMetadata[K]] = TileLayerMetadataEncoder()
-  implicit def crsEncoder: ExpressionEncoder[CRS] = CRSEncoder()
+  implicit def crsSparkEncoder: ExpressionEncoder[CRS] = CRSEncoder()
   implicit def projectedExtentEncoder: ExpressionEncoder[ProjectedExtent] = ProjectedExtentEncoder()
   implicit def temporalProjectedExtentEncoder: ExpressionEncoder[TemporalProjectedExtent] = TemporalProjectedExtentEncoder()
   implicit def cellTypeEncoder: ExpressionEncoder[CellType] = CellTypeEncoder()

core/src/main/scala/org/locationtech/rasterframes/encoders/StandardSerializers.scala

@@ -24,8 +24,8 @@ package org.locationtech.rasterframes.encoders
 import com.github.blemale.scaffeine.Scaffeine
 import geotrellis.proj4.CRS
 import geotrellis.raster._
-import geotrellis.spark._
-import geotrellis.spark.tiling.LayoutDefinition
+import geotrellis.layer._
+
 import geotrellis.vector._
 import org.apache.spark.sql.types._
 import org.locationtech.jts.geom.Envelope
@@ -71,19 +71,19 @@ trait StandardSerializers {
     )
   }
 
-  implicit val gridBoundsSerializer: CatalystSerializer[GridBounds] = new CatalystSerializer[GridBounds] {
+  implicit val gridBoundsSerializer: CatalystSerializer[GridBounds[Int]] = new CatalystSerializer[GridBounds[Int]] {
     override val schema: StructType = StructType(Seq(
       StructField("colMin", IntegerType, false),
       StructField("rowMin", IntegerType, false),
       StructField("colMax", IntegerType, false),
       StructField("rowMax", IntegerType, false)
     ))
 
-    override protected def to[R](t: GridBounds, io: CatalystIO[R]): R = io.create(
+    override protected def to[R](t: GridBounds[Int], io: CatalystIO[R]): R = io.create(
       t.colMin, t.rowMin, t.colMax, t.rowMax
     )
 
-    override protected def from[R](t: R, io: CatalystIO[R]): GridBounds = GridBounds(
+    override protected def from[R](t: R, io: CatalystIO[R]): GridBounds[Int] = GridBounds[Int](
       io.getInt(t, 0),
       io.getInt(t, 1),
       io.getInt(t, 2),

core/src/main/scala/org/locationtech/rasterframes/encoders/TemporalProjectedExtentEncoder.scala

@@ -22,7 +22,7 @@
 package org.locationtech.rasterframes.encoders
 
 import org.locationtech.rasterframes._
-import geotrellis.spark.TemporalProjectedExtent
+import geotrellis.layer._
 import org.apache.spark.sql.Encoders
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 
@@ -34,9 +34,10 @@ import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
  */
 object TemporalProjectedExtentEncoder {
   def apply(): ExpressionEncoder[TemporalProjectedExtent] = {
+    import StandardEncoders.crsSparkEncoder
     DelegatingSubfieldEncoder(
       "extent" -> extentEncoder,
-      "crs" -> crsEncoder,
+      "crs" -> crsSparkEncoder,
       "instant" -> Encoders.scalaLong.asInstanceOf[ExpressionEncoder[Long]]
     )
   }

core/src/main/scala/org/locationtech/rasterframes/encoders/TileLayerMetadataEncoder.scala

@@ -21,7 +21,7 @@
 
 package org.locationtech.rasterframes.encoders
 
-import geotrellis.spark.{KeyBounds, TileLayerMetadata}
+import geotrellis.layer._
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 
 import scala.reflect.runtime.universe._
@@ -39,7 +39,7 @@ object TileLayerMetadataEncoder {
     "cellType" -> cellTypeEncoder,
     "layout" -> layoutDefinitionEncoder,
     "extent" -> extentEncoder,
-    "crs" -> crsEncoder
+    "crs" -> crsSparkEncoder
   )
 
   def apply[K: TypeTag](): ExpressionEncoder[TileLayerMetadata[K]] = {