Incremental updates toward GT 3.0 compatibility.

Author: metasim

core/src/main/scala/org/locationtech/rasterframes/tiles/FixedDelegatingTile.scala

@@ -0,0 +1,39 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.tiles
+import geotrellis.raster.{ArrayTile, DelegatingTile, Tile}
+
+/**
+ * Workaround for case where `combine` is invoked on two delegating tiles.
+ * @since 8/22/18
+ */
+trait FixedDelegatingTile extends DelegatingTile {
+  override def combine(r2: Tile)(f: (Int, Int) ⇒ Int): Tile = (delegate, r2) match {
+    case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combine(r2.toArrayTile())(f)
+    case _ ⇒ delegate.combine(r2)(f)
+  }
+
+  override def combineDouble(r2: Tile)(f: (Double, Double) ⇒ Double): Tile = (delegate, r2) match {
+    case (del: ArrayTile, r2: DelegatingTile) ⇒ del.combineDouble(r2.toArrayTile())(f)
+    case _ ⇒ delegate.combineDouble(r2)(f)
+  }
+}

core/src/main/scala/org/locationtech/rasterframes/tiles/InternalRowTile.scala

@@ -23,9 +23,9 @@ package org.locationtech.rasterframes.tiles
 
 import java.nio.ByteBuffer
 
-import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO
 import geotrellis.raster._
 import org.apache.spark.sql.catalyst.InternalRow
+import org.locationtech.rasterframes.encoders.CatalystSerializer.CatalystIO
 import org.locationtech.rasterframes.model.{Cells, TileDataContext}
 
 /**
@@ -34,7 +34,7 @@ import org.locationtech.rasterframes.model.{Cells, TileDataContext}
  *
  * @since 11/29/17
  */
-class InternalRowTile(val mem: InternalRow) extends DelegatingTile {
+class InternalRowTile(val mem: InternalRow) extends FixedDelegatingTile {
   import InternalRowTile._
 
   override def toArrayTile(): ArrayTile = realizedTile.toArrayTile()

core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala

@@ -23,7 +23,7 @@ package org.locationtech.rasterframes.tiles
 
 import geotrellis.proj4.CRS
 import geotrellis.raster.io.geotiff.SinglebandGeoTiff
-import geotrellis.raster.{CellType, DelegatingTile, ProjectedRaster, Tile}
+import geotrellis.raster.{CellType, ProjectedRaster, Tile}
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.rf.TileUDT
@@ -40,7 +40,7 @@ import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
  *
  * @since 9/5/18
  */
-trait ProjectedRasterTile extends DelegatingTile with ProjectedRasterLike {
+trait ProjectedRasterTile extends FixedDelegatingTile with ProjectedRasterLike {
   def extent: Extent
   def crs: CRS
   def projectedExtent: ProjectedExtent = ProjectedExtent(extent, crs)

core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala

@@ -21,9 +21,9 @@
 
 package org.locationtech.rasterframes.tiles
 import org.locationtech.rasterframes._
-import geotrellis.raster.{DelegatingTile, Tile, isNoData}
+import geotrellis.raster.{Tile, isNoData}
 
-class ShowableTile(val delegate: Tile) extends DelegatingTile {
+class ShowableTile(val delegate: Tile) extends FixedDelegatingTile {
   override def equals(obj: Any): Boolean = obj match {
     case st: ShowableTile => delegate.equals(st.delegate)
     case o => delegate.equals(o)

core/src/test/resources/log4j.properties

@@ -45,4 +45,5 @@ log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
 log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
 log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
 
-log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=ERROR
+log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=ERROR
+log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=ERROR

core/src/test/scala/org/locationtech/rasterframes/GeometryFunctionsSpec.scala

@@ -25,7 +25,6 @@ import geotrellis.proj4.{LatLng, Sinusoidal, WebMercator}
 import geotrellis.raster.Dimensions
 import geotrellis.vector._
 import org.locationtech.jts.geom.{Coordinate, GeometryFactory}
-import spray.json.JsNumber
 
 /**
  * Test rig for operations providing interop with JTS types.

core/src/test/scala/org/locationtech/rasterframes/RasterJoinSpec.scala

@@ -24,6 +24,8 @@ package org.locationtech.rasterframes
 import geotrellis.raster.resample.Bilinear
 import geotrellis.raster.testkit.RasterMatchers
 import geotrellis.raster.{IntConstantNoDataCellType, Raster, Tile}
+import geotrellis.vector.Extent
+import org.apache.spark.SparkConf
 import org.apache.spark.sql.functions._
 import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate
 import org.locationtech.rasterframes.expressions.aggregates.TileRasterizerAggregate.ProjectedRasterDefinition
@@ -74,14 +76,16 @@ class RasterJoinSpec extends TestEnvironment with TestData with RasterMatchers {
     }
 
     it("should join same scene in two projections, same tile size") {
-
       // b4warpedRf source data is gdal warped b4nativeRf data; join them together.
       val joined = b4nativeRf.rasterJoin(b4warpedRf)
       // create a Raster from tile2 which should be almost equal to b4nativeTif
-      val result = joined.agg(TileRasterizerAggregate(
+      val agg = joined.agg(TileRasterizerAggregate(
         ProjectedRasterDefinition(b4nativeTif.cols, b4nativeTif.rows, b4nativeTif.cellType, b4nativeTif.crs, b4nativeTif.extent, Bilinear),
         $"crs", $"extent", $"tile2") as "raster"
-      ).select(col("raster").as[Raster[Tile]]).first()
+      ).select(col("raster").as[Raster[Tile]])
+
+      agg.printSchema()
+      val result = agg.first()
 
       result.extent shouldBe b4nativeTif.extent
 
@@ -165,4 +169,6 @@ class RasterJoinSpec extends TestEnvironment with TestData with RasterMatchers {
       joined.columns should contain allElementsOf Seq("left_id", "right_id_agg")
     }
   }
+
+  override def additionalConf: SparkConf = super.additionalConf.set("spark.sql.codegen.comments", "true")
 }

core/src/test/scala/org/locationtech/rasterframes/encoders/EncodingSpec.scala

@@ -26,7 +26,7 @@ import java.net.URI
 
 import geotrellis.layer._
 import geotrellis.proj4._
-import geotrellis.raster.{CellType, Tile}
+import geotrellis.raster.{ArrayTile, CellType, Raster, Tile}
 import geotrellis.vector.{Extent, ProjectedExtent}
 import org.apache.spark.SparkConf
 import org.apache.spark.sql.Row
@@ -145,6 +145,16 @@ class EncodingSpec extends TestEnvironment with TestData {
       write(ds)
       assert(ds.first === env)
     }
+
+    it("should code RDD[Raster[Tile]]") {
+      import spark.implicits._
+      val t: Tile = ArrayTile(Array.emptyDoubleArray, 0, 0)
+      val e = Extent(1, 2 ,3, 4)
+      val r = Raster(t, e)
+      val ds = Seq(r).toDS()
+      println(ds.first())
+    }
+
   }
   describe("Dataframe encoding ops on spatial types") {
 

project/RFProjectPlugin.scala

@@ -28,7 +28,7 @@ object RFProjectPlugin extends AutoPlugin {
       "-Ywarn-unused-import"
     ),
     scalacOptions in (Compile, doc) ++= Seq("-no-link-warnings"),
-    console / scalacOptions := Seq("-feature"),
+    Compile / console / scalacOptions := Seq("-feature"),
     javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
     cancelable in Global := true,
     publishTo in ThisBuild := sonatypePublishTo.value,