Release 0.5.11.

Signed-off-by: Simeon H.K. Fitch <[email protected]>

Author: metasim

bench/archive/jmh-results-20171130120731.json

@@ -1,9 +1,17 @@
 enablePlugins(BenchmarkPlugin)
 
+libraryDependencies ++= Seq(
+  spark("core"),
+  spark("sql"),
+  geotrellis("spark"),
+  geotrellis("raster")
+)
+
 jmhIterations := Some(5)
 jmhTimeUnit := None
 javaOptions in Jmh := Seq("-Xmx4g")
 
 // To enable profiling:
 // jmhExtraOptions := Some("-prof jmh.extras.JFR")
+// jmhExtraOptions := Some("-prof gc")
 

bench/archive/jmh-results-20171130135353.json

@@ -0,0 +1,73 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2017 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+
+package astraea.spark.rasterframes.bench
+
+import java.util.concurrent.TimeUnit
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
+import org.apache.spark.sql.gt.InternalRowTile
+import org.apache.spark.sql.gt.types.TileUDT
+import org.openjdk.jmh.annotations._
+
+@BenchmarkMode(Array(Mode.AverageTime))
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+/**
+ * @author sfitch
+ * @since 9/29/17
+ */
+class TileCellScanBench extends SparkEnv {
+
+  @Param(Array("uint8", "int32", "float32", "float64"))
+  var cellTypeName: String = _
+
+  @Param(Array("512"))
+  var tileSize: Int = _
+
+  @transient
+  var tileRow: InternalRow = _
+
+  @Setup(Level.Trial)
+  def setupData(): Unit = {
+    tileRow = TileUDT.serialize(randomTile(tileSize, tileSize, cellTypeName))
+  }
+
+  @Benchmark
+  def deserializeRead(): Double  = {
+    val tile = TileUDT.deserialize(tileRow)
+    val (cols, rows) = tile.dimensions
+    tile.getDouble(cols - 1, rows - 1) +
+      tile.getDouble(cols/2, rows/2) +
+      tile.getDouble(0, 0)
+  }
+
+  @Benchmark
+  def internalRowRead(): Double  = {
+    val tile = new InternalRowTile(tileRow)
+    val cols = tile.cols
+    val rows = tile.rows
+    tile.getDouble(cols - 1, rows - 1) +
+      tile.getDouble(cols/2, rows/2) +
+      tile.getDouble(0, 0)
+  }
+}
+

bench/archive/jmh-results-20171130145409.json

@@ -0,0 +1,66 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2017 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ */
+package astraea.spark.rasterframes.bench
+
+import java.util.concurrent.TimeUnit
+
+import astraea.spark.rasterframes._
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+import org.openjdk.jmh.annotations._
+
+/**
+ *
+ * @author sfitch
+ * @since 10/4/17
+ */
+@BenchmarkMode(Array(Mode.AverageTime))
+@State(Scope.Benchmark)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+class TileExplodeBench extends SparkEnv {
+  import spark.implicits._
+
+  @Param(Array("uint8", "uint16ud255", "float32", "float64"))
+  var cellTypeName: String = _
+
+  @Param(Array("256"))
+  var tileSize: Int = _
+
+  @Param(Array("100"))
+  var numTiles: Int = _
+
+  @transient
+  var tiles: DataFrame = _
+
+  @Setup(Level.Trial)
+  def setupData(): Unit = {
+    tiles = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
+      .toDF("tile").repartition(10)
+  }
+
+  @Benchmark
+  def arrayExplode() = {
+    tiles.select(posexplode(tileToArray[Double]($"tile"))).count()
+  }
+
+  @Benchmark
+  def tileExplode() = {
+    tiles.select(explodeTiles($"tile")).count()
+  }
+}

bench/archive/jmh-results-20171201123901.json

@@ -26,10 +26,7 @@ object ProjectPlugin extends AutoPlugin {
     "spark" -> "2.1.0"
   )
 
-  private def geotrellis(module: String) =
-    "org.locationtech.geotrellis" %% s"geotrellis-$module" % versions("geotrellis")
-  private def spark(module: String) =
-    "org.apache.spark" %% s"spark-$module" % versions("spark")
+  import autoImport._
 
   override def projectSettings = Seq(
     organization := "io.astraea",
@@ -79,6 +76,12 @@ object ProjectPlugin extends AutoPlugin {
   )
 
   object autoImport {
+
+    def geotrellis(module: String) =
+      "org.locationtech.geotrellis" %% s"geotrellis-$module" % versions("geotrellis")
+    def spark(module: String) =
+      "org.apache.spark" %% s"spark-$module" % versions("spark")
+
     def releaseSettings: Seq[Def.Setting[_]] = {
       val buildSite: (State) ⇒ State = releaseStepTask(makeSite)
       val publishSite: (State) ⇒ State = releaseStepTask(ghpagesPushSite)
@@ -124,9 +127,9 @@ object ProjectPlugin extends AutoPlugin {
       apiURL := Some(url("http://rasterframes.io/latest/api")),
       autoAPIMappings := false,
       paradoxProperties in Paradox ++= Map(
-        "github.base_url" -> "https://github.com/s22s/raster-frames"//,
-        //"scaladoc.org.apache.spark.sql.gt" -> "http://rasterframes.io/latest" //,
-        //"scaladoc.geotrellis.base_url" -> "https://geotrellis.github.io/scaladocs/latest"
+        "github.base_url" -> "https://github.com/s22s/raster-frames",
+        "scaladoc.org.apache.spark.sql.gt" -> "http://rasterframes.io/latest",
+        "scaladoc.geotrellis.base_url" -> "https://geotrellis.github.io/scaladocs/latest"
       ),
       sourceDirectory in Paradox := tutTargetDirectory.value,
       sourceDirectory in Paradox in paradoxTheme := sourceDirectory.value / "main" / "paradox" / "_template",

bench/build.sbt

@@ -1 +1 @@
-sbt.version=1.0.2
+sbt.version=1.0.4

bench/src/main/scala/astraea/spark/rasterframes/bench/TileCellScanBench.scala

@@ -10,7 +10,7 @@ addSbtPlugin("de.heikoseeberger" % "sbt-header" % "3.0.2")
 
 addSbtPlugin("com.lucidchart" % "sbt-scalafmt" % "1.10")
 
-addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.6.0")
+addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.6.2")
 
 addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.3.1")