Merge branch 'develop' into feature/xz2

* develop:
  Fixed image link in README.md
  PR feedback.
  Removed `GeoTiffCollectionRelation` due to usage limitation and overlap with `RasterSourceDataSource` functionality.
  Bump pytest to 4.x series
  In rf_ipython, honor users pandas max_colwidth on Geometry objects
  Additional python version wrangling.
  merge pyspark-notebook:spark-2.3.4-hadoop-2.7 Dockerfile into this one
  Dropping pandas version requirement.
  Updated requirements.txt for Spark.
  Fixes to `UserDefinedAggregateFunction` implementations so they run in the Databricks environment. Not sure what the cause is, but this change set seems to fix it.
  Added shading of Typesafe config.
  Upgraded to Spark 2.4.4
  Bumped dev version.
  Set release version.
  Reworking of logging usage to fix #293
  Eyecandy for index page.
  Bumping pytest version.
  Added liblzma-dev to build environment.

Author: metasim

README.md

@@ -1,4 +1,4 @@
-<img src="docs/src/main/paradox/_template/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&reg;</sup>
+<img src="docs/src/main/paradox/_template/assets/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&reg;</sup>
 
  [![Join the chat at https://gitter.im/locationtech/rasterframes](https://badges.gitter.im/locationtech/rasterframes.svg)](https://gitter.im/locationtech/rasterframes?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 

bench/src/main/scala/org/locationtech/rasterframes/bench/TileExplodeBench.scala

@@ -22,12 +22,11 @@ package org.locationtech.rasterframes.bench
 
 import java.util.concurrent.TimeUnit
 
-import org.apache.spark.sql.catalyst.InternalRow
-import org.apache.spark.sql.catalyst.expressions.BoundReference
-import org.apache.spark.sql.rf.TileUDT
 import org.locationtech.rasterframes._
-import org.locationtech.rasterframes.expressions.generators.ExplodeTiles
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
 import org.openjdk.jmh.annotations._
+
 /**
  *
  * @author sfitch
@@ -37,32 +36,33 @@ import org.openjdk.jmh.annotations._
 @State(Scope.Benchmark)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 class TileExplodeBench extends SparkEnv {
+  import spark.implicits._
 
-  //@Param(Array("uint8", "uint16ud255", "float32", "float64"))
-  @Param(Array("uint16ud255"))
+  @Param(Array("uint8", "uint16ud255", "float32", "float64"))
   var cellTypeName: String = _
 
   @Param(Array("256"))
   var tileSize: Int = _
 
-  @Param(Array("2000"))
+  @Param(Array("100"))
   var numTiles: Int = _
 
   @transient
-  var tiles: Array[InternalRow] = _
-
-  var exploder: ExplodeTiles = _
+  var tiles: DataFrame = _
 
   @Setup(Level.Trial)
   def setupData(): Unit = {
-    tiles = Array.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
-        .map(t => InternalRow(TileUDT.tileSerializer.toInternalRow(t)))
-    val expr = BoundReference(0, TileType, true)
-    exploder = new ExplodeTiles(1.0, None, Seq(expr))
+    tiles = Seq.fill(numTiles)(randomTile(tileSize, tileSize, cellTypeName))
+      .toDF("tile").repartition(10)
+  }
+
+  @Benchmark
+  def arrayExplode() = {
+    tiles.select(posexplode(rf_tile_to_array_double($"tile"))).count()
   }
+
   @Benchmark
   def tileExplode() = {
-    for(t <- tiles)
-      exploder.eval(t)
+    tiles.select(rf_explode_tiles($"tile")).count()
   }
 }

build.sbt

@@ -32,7 +32,10 @@ lazy val root = project
   .withId("RasterFrames")
   .aggregate(core, datasource, pyrasterframes, experimental)
   .enablePlugins(RFReleasePlugin)
-  .settings(publish / skip := true)
+  .settings(
+    publish / skip := true,
+    clean := clean.dependsOn(`rf-notebook`/clean).value
+  )
 
 lazy val `rf-notebook` = project
   .dependsOn(pyrasterframes)

build/circleci/Dockerfile

@@ -14,16 +14,18 @@ RUN sudo apt-get update && \
       pandoc \
       wget \
       gcc g++ build-essential \
-      libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
-            libcurl4-gnutls-dev \
-            libproj-dev \
-            libgeos-dev \
-            libhdf4-alt-dev \
-            bash-completion \
-            cmake \
-            imagemagick \
-            libpng-dev \
-            libffi-dev \
+      libreadline-gplv2-dev libncursesw5-dev \
+      libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
+      liblzma-dev \
+      libcurl4-gnutls-dev \
+      libproj-dev \
+      libgeos-dev \
+      libhdf4-alt-dev \
+      bash-completion \
+      cmake \
+      imagemagick \
+      libpng-dev \
+      libffi-dev \
     && sudo apt autoremove \
     && sudo apt-get clean all
 #    && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
@@ -73,8 +75,7 @@ RUN cd /tmp && \
         --with-threads \
         --without-jp2mrsid \
         --without-netcdf \
-        --without-ecw \
-    && \
+        --without-ecw && \
     make -j 8 && \
     sudo make install && \
     sudo ldconfig && \

core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryLocalRasterOp.scala

@@ -21,18 +21,22 @@
 
 package org.locationtech.rasterframes.expressions
 
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import com.typesafe.scalalogging.LazyLogging
+import com.typesafe.scalalogging.Logger
 import geotrellis.raster.Tile
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.BinaryExpression
 import org.apache.spark.sql.rf.TileUDT
 import org.apache.spark.sql.types.DataType
+import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.expressions.DynamicExtractors._
+import org.slf4j.LoggerFactory
 
 /** Operation combining two tiles or a tile and a scalar into a new tile. */
-trait BinaryLocalRasterOp extends BinaryExpression with LazyLogging {
+trait BinaryLocalRasterOp extends BinaryExpression {
+
+  @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
+
 
   override def dataType: DataType = left.dataType
 

core/src/main/scala/org/locationtech/rasterframes/expressions/BinaryRasterOp.scala

@@ -21,18 +21,20 @@
 
 package org.locationtech.rasterframes.expressions
 
-import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import com.typesafe.scalalogging.LazyLogging
+import com.typesafe.scalalogging.Logger
 import geotrellis.raster.Tile
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.BinaryExpression
 import org.apache.spark.sql.rf.TileUDT
 import org.apache.spark.sql.types.DataType
+import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
+import org.slf4j.LoggerFactory
 
 /** Operation combining two tiles into a new tile. */
-trait BinaryRasterOp extends BinaryExpression with LazyLogging {
+trait BinaryRasterOp extends BinaryExpression {
+  @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
 
   override def dataType: DataType = left.dataType
 

core/src/main/scala/org/locationtech/rasterframes/expressions/UnaryLocalRasterOp.scala

@@ -21,18 +21,20 @@
 
 package org.locationtech.rasterframes.expressions
 
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import com.typesafe.scalalogging.LazyLogging
+import com.typesafe.scalalogging.Logger
 import geotrellis.raster.Tile
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions.UnaryExpression
 import org.apache.spark.sql.rf.TileUDT
 import org.apache.spark.sql.types.DataType
+import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.expressions.DynamicExtractors._
+import org.slf4j.LoggerFactory
 
 /** Operation on a tile returning a tile. */
-trait UnaryLocalRasterOp extends UnaryExpression with LazyLogging {
+trait UnaryLocalRasterOp extends UnaryExpression {
+  @transient protected lazy val logger = Logger(LoggerFactory.getLogger(getClass.getName))
 
   override def dataType: DataType = child.dataType
 

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/CellStatsAggregate.scala

@@ -123,8 +123,8 @@ object CellStatsAggregate {
   import org.locationtech.rasterframes.encoders.StandardEncoders.cellStatsEncoder
 
   def apply(col: Column): TypedColumn[Any, CellStatistics] =
-    new Column(new CellStatsAggregateUDAF(col.expr))
-      .as(s"rf_agg_stats($col)") // node renaming in class doesn't seem to propogate
+    new CellStatsAggregate()(ExtractTile(col))
+      .as(s"rf_agg_stats($col)")
       .as[CellStatistics]
 
   /** Adapter hack to allow UserDefinedAggregateFunction to be referenced as an expression. */

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/HistogramAggregate.scala

@@ -98,8 +98,8 @@ object HistogramAggregate {
   import org.locationtech.rasterframes.encoders.StandardEncoders.cellHistEncoder
 
   def apply(col: Column): TypedColumn[Any, CellHistogram] =
-    new Column(new HistogramAggregateUDAF(col.expr))
-      .as(s"rf_agg_approx_histogram($col)") // node renaming in class doesn't seem to propogate
+    new HistogramAggregate()(ExtractTile(col))
+      .as(s"rf_agg_approx_histogram($col)")
       .as[CellHistogram]
 
   /** Adapter hack to allow UserDefinedAggregateFunction to be referenced as an expression. */

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalCountAggregate.scala

@@ -92,7 +92,7 @@ object LocalCountAggregate {
   object LocalDataCellsUDAF {
     def apply(child: Expression): LocalDataCellsUDAF = new LocalDataCellsUDAF(child)
     def apply(tile: Column): TypedColumn[Any, Tile] =
-      new Column(new LocalDataCellsUDAF(tile.expr))
+      new LocalCountAggregate(true)(ExtractTile(tile))
         .as(s"rf_agg_local_data_cells($tile)")
         .as[Tile]
   }
@@ -107,7 +107,7 @@ object LocalCountAggregate {
   object LocalNoDataCellsUDAF {
     def apply(child: Expression): LocalNoDataCellsUDAF = new LocalNoDataCellsUDAF(child)
     def apply(tile: Column): TypedColumn[Any, Tile] =
-      new Column(new LocalNoDataCellsUDAF(tile.expr))
+      new LocalCountAggregate(false)(ExtractTile(tile))
         .as(s"rf_agg_local_no_data_cells($tile)")
         .as[Tile]
   }