Merge pull request #554 from echeipesh/scala-2.12

Scala 2.12 and Spark 3 support

Author: metasim

.gitignore

@@ -1,7 +1,13 @@
+# Operating System Files
+
+*.DS_Store
+Thumbs.db
+
 *.class
 *.log
 
 # sbt specific
+.bsp
 .cache
 .history
 .lib/

.java-version

@@ -1,16 +1,11 @@
-maxColumn = 138
+version = 3.0.3
+runner.dialect = scala212
+indent.main = 2
+indent.significant = 2
+maxColumn = 150
 continuationIndent.defnSite = 2
-binPack.parentConstructors = true
-binPack.literalArgumentLists = false
-newlines.penalizeSingleSelectMultiArgList = false
-newlines.sometimesBeforeColonInMethodReturnType = false
-align.openParenCallSite = false
-align.openParenDefnSite = false
-docstrings = JavaDoc
-rewriteTokens {
-  "⇒" = "=>"
-  "←" = "<-"
-}
-optIn.selfAnnotationNewline = false
-optIn.breakChainOnFirstMethodDot = true
-importSelectors = BinPack
+assumeStandardLibraryStripMargin = true
+danglingParentheses.preset = true
+rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers]
+docstrings.style = Asterisk
+# align.preset = more

.scalafmt.conf

@@ -0,0 +1,3 @@
+# Enable auto-env through the sdkman_auto_env config
+# Add key=value pairs of SDKs to use below
+java=11.0.11.hs-adpt

.sdkmanrc

@@ -11,7 +11,7 @@ libraryDependencies ++= Seq(
 jmhIterations := Some(5)
 jmhWarmupIterations := Some(8)
 jmhTimeUnit := None
-javaOptions in Jmh := Seq("-Xmx4g")
+Jmh / javaOptions := Seq("-Xmx4g")
 
 // To enable profiling:
 // jmhExtraOptions := Some("-prof jmh.extras.JFR")

bench/build.sbt

@@ -21,10 +21,9 @@
 
 package org.locationtech.rasterframes.bench
 import java.util.concurrent.TimeUnit
-
 import geotrellis.raster.{CellType, DoubleUserDefinedNoDataCellType, IntUserDefinedNoDataCellType}
 import org.apache.spark.sql.catalyst.InternalRow
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.StandardEncoders
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -37,16 +36,12 @@ class CellTypeBench {
   def setupData(): Unit = {
     ct = IntUserDefinedNoDataCellType(scala.util.Random.nextInt())
     val o: CellType = DoubleUserDefinedNoDataCellType(scala.util.Random.nextDouble())
-    row = o.toInternalRow
+    row = StandardEncoders.cellTypeEncoder.createSerializer()(o)
   }
 
   @Benchmark
-  def fromRow(): CellType = {
-    row.to[CellType]
-  }
+  def fromRow(): CellType = StandardEncoders.cellTypeEncoder.createDeserializer()(row)
 
   @Benchmark
-  def intoRow(): InternalRow = {
-    ct.toInternalRow
-  }
+  def intoRow(): InternalRow = StandardEncoders.cellTypeEncoder.createSerializer()(ct)
 }

bench/src/main/scala/org/locationtech/rasterframes/bench/CatalystSerializerBench.scala

@@ -26,7 +26,6 @@ import java.util.concurrent.TimeUnit
 import geotrellis.raster.Dimensions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.rf.TileUDT
-import org.locationtech.rasterframes.tiles.InternalRowTile
 import org.openjdk.jmh.annotations._
 
 @BenchmarkMode(Array(Mode.AverageTime))
@@ -62,15 +61,4 @@ class TileCellScanBench extends SparkEnv {
       tile.getDouble(cols/2, rows/2) +
       tile.getDouble(0, 0)
   }
-
-  @Benchmark
-  def internalRowRead(): Double  = {
-    val tile = new InternalRowTile(tileRow)
-    val cols = tile.cols
-    val rows = tile.rows
-    tile.getDouble(cols - 1, rows - 1) +
-      tile.getDouble(cols/2, rows/2) +
-      tile.getDouble(0, 0)
-  }
 }
-

bench/src/main/scala/org/locationtech/rasterframes/bench/CellTypeBench.scala

@@ -24,8 +24,6 @@ package org.locationtech.rasterframes.bench
 import java.net.URI
 import java.util.concurrent.TimeUnit
 
-import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
-import org.locationtech.rasterframes.ref.RasterRef
 import geotrellis.raster.Tile
 import geotrellis.vector.Extent
 import org.apache.spark.sql.catalyst.InternalRow
@@ -53,24 +51,23 @@ class TileEncodeBench extends SparkEnv {
   @Setup(Level.Trial)
   def setupData(): Unit = {
     cellTypeName match {
-      case "rasterRef" ⇒
+      case "rasterRef" =>
         val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF"
         val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0)
-        tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None))
-      case _ ⇒
+        tile = RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None)
+      case _ =>
         tile = randomTile(tileSize, tileSize, cellTypeName)
     }
   }
 
   @Benchmark
   def encode(): InternalRow = {
-    tileEncoder.toRow(tile)
+    tileEncoder.createSerializer.apply(tile)
   }
 
   @Benchmark
   def roundTrip(): Tile = {
-    val row = tileEncoder.toRow(tile)
-    boundEncoder.fromRow(row)
+    val row = tileEncoder.createSerializer().apply(tile)
+    boundEncoder.createDeserializer().apply(row)
   }
 }
-

bench/src/main/scala/org/locationtech/rasterframes/bench/TileCellScanBench.scala

@@ -37,10 +37,10 @@ package object bench {
     val cellType = CellType.fromName(cellTypeName)
     val tile = ArrayTile.alloc(cellType, cols, rows)
     if(cellType.isFloatingPoint) {
-      tile.mapDouble(_ ⇒ rnd.nextGaussian())
+      tile.mapDouble(_ => rnd.nextGaussian())
     }
     else {
-      tile.map(_ ⇒ {
+      tile.map(_ => {
         var c = NODATA
         do {
           c = rnd.nextInt(255)