Merge commit '1ea29f2ccdc74eb3e900fbb075eae6e9421ea11a' into feature/ipython-display-tweaks

* commit '1ea29f2ccdc74eb3e900fbb075eae6e9421ea11a':
  Bug was with changes to `rf_with_no_data` acceptance of either `Double` or `Int`, and then incorrectly changing floating point tiles into a float cell type. Fixes #304.
  Updated ShowableTile to render NoData values differently.
  Moved array function tests to RasterFunctionsSpec.

# Conflicts:
#	pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py

Author: metasim

core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala

@@ -57,7 +57,7 @@ case class SetCellType(tile: Expression, cellType: Expression)
   extends BinaryExpression with CodegenFallback {
   def left = tile
   def right = cellType
-  override def nodeName: String = "set_cell_type"
+  override def nodeName: String = "rf_convert_cell_type"
   override def dataType: DataType = left.dataType
 
   override def checkInputDataTypes(): TypeCheckResult = {

core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala

@@ -22,7 +22,6 @@
 package org.locationtech.rasterframes.expressions.transformers
 
 import com.typesafe.scalalogging.LazyLogging
-import geotrellis.raster.Tile
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
@@ -33,7 +32,7 @@ import org.apache.spark.sql.rf.TileUDT
 import org.apache.spark.sql.types._
 import org.locationtech.rasterframes.encoders.CatalystSerializer._
 import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.expressions.{fpTile, row}
+import org.locationtech.rasterframes.expressions.row
 
 @ExpressionDescription(
   usage = "_FUNC_(tile, value) - Set the NoData value for the given tile.",
@@ -64,20 +63,19 @@ case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExp
   override protected def nullSafeEval(input1: Any, input2: Any): Any = {
     implicit val tileSer = TileUDT.tileSerializer
     val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
+
     val result = numberArgExtractor(right.dataType)(input2) match {
-      case DoubleArg(d) => op(fpTile(leftTile), d)
-      case IntegerArg(i) => op(leftTile, i)
+      case DoubleArg(d) => leftTile.withNoData(Some(d))
+      case IntegerArg(i) => leftTile.withNoData(Some(i.toDouble))
     }
 
     leftCtx match {
       case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
       case None => result.toInternalRow
     }
   }
-
-  protected def op(left: Tile, right: Double): Tile = left.withNoData(Some(right))
-  protected def op(left: Tile, right: Int): Tile = left.withNoData(Some(right))
 }
+
 object SetNoDataValue {
   def apply(left: Column, right: Column): Column =
     new Column(SetNoDataValue(left.expr, right.expr))

core/src/main/scala/org/locationtech/rasterframes/functions/package.scala

@@ -90,8 +90,6 @@ package object functions {
     }
   }
 
-
-
   /** Alias for constant tiles of zero */
   private[rasterframes] val tileZeros: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒
     makeConstantTile(0, cols, rows, cellTypeName)

core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala

@@ -21,7 +21,7 @@
 
 package org.locationtech.rasterframes.tiles
 import org.locationtech.rasterframes._
-import geotrellis.raster.Tile
+import geotrellis.raster.{Tile, isNoData}
 
 class ShowableTile(val delegate: Tile) extends FixedDelegatingTile {
   override def equals(obj: Any): Boolean = obj match {
@@ -39,8 +39,14 @@ object ShowableTile {
     val dims = tile.dimensions
 
     val data = if (tile.cellType.isFloatingPoint)
-      tile.toArrayDouble()
-    else tile.toArray()
+      tile.toArrayDouble().map {
+        case c if isNoData(c) => "--"
+        case c => c.toString
+      }
+    else tile.toArray().map {
+      case c if isNoData(c) => "--"
+      case c => c.toString
+    }
 
     val cells = if(tile.size <= maxCells) {
       data.mkString("[", ",", "]")

core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala

@@ -110,45 +110,6 @@ class ExplodeSpec extends TestEnvironment with TestData {
       cells.count(_.isNaN) should be(tiles.size)
     }
 
-    it("should convert tile into array") {
-      val query = sql(
-        """select rf_tile_to_array_int(
-          |  rf_make_constant_tile(1, 10, 10, 'int8raw')
-          |) as intArray
-          |""".stripMargin)
-      query.as[Array[Int]].first.sum should be (100)
-
-      val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType)
-      val df = Seq[Tile](tile).toDF("tile")
-      val arrayDF = df.select(rf_tile_to_array_double($"tile").as[Array[Double]])
-      arrayDF.first().sum should be (110.0 +- 0.0001)
-
-      checkDocs("rf_tile_to_array_int")
-    }
-
-    it("should convert an array into a tile") {
-      val tile = TestData.randomTile(10, 10, FloatCellType)
-      val df = Seq[Tile](tile, null).toDF("tile")
-      val arrayDF = df.withColumn("tileArray", rf_tile_to_array_double($"tile"))
-
-      val back = arrayDF.withColumn("backToTile", rf_array_to_tile($"tileArray", 10, 10))
-
-      val result = back.select($"backToTile".as[Tile]).first
-
-      assert(result.toArrayDouble() === tile.toArrayDouble())
-
-      // Same round trip, but with SQL expression for rf_array_to_tile
-      val resultSql = arrayDF.selectExpr("rf_array_to_tile(tileArray, 10, 10) as backToTile").as[Tile].first
-
-      assert(resultSql.toArrayDouble() === tile.toArrayDouble())
-
-      val hasNoData = back.withColumn("withNoData", rf_with_no_data($"backToTile", 0))
-
-      val result2 = hasNoData.select($"withNoData".as[Tile]).first
-
-      assert(result2.cellType.asInstanceOf[UserDefinedNoData[_]].noDataValue === 0)
-    }
-
     it("should reassemble single exploded tile") {
       val tile = TestData.randomTile(10, 10, FloatCellType)
       val df = Seq[Tile](tile).toDF("tile")

core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala

@@ -120,17 +120,20 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
     it("should change NoData value") {
       val df = Seq((TestData.injectND(7)(three), TestData.injectND(12)(two))).toDF("three", "two")
 
-      val ct = df.select(
+      val ndCT = df.select(
         rf_with_no_data($"three", 3) as "three",
-        rf_with_no_data($"two", 2) as "two"
+        rf_with_no_data($"two", 2.0) as "two"
       )
 
-      val (cnt3, cnt2) = ct.select(rf_no_data_cells($"three"), rf_no_data_cells($"two")).as[(Long, Long)].first()
+      val (cnt3, cnt2) = ndCT.select(rf_no_data_cells($"three"), rf_no_data_cells($"two")).as[(Long, Long)].first()
 
       cnt3 should be ((cols * rows) - 7)
       cnt2 should be ((cols * rows) - 12)
 
       checkDocs("rf_with_no_data")
+
+      // Should maintain original cell type.
+      ndCT.select(rf_cell_type($"two")).first().withDefaultNoData() should be(ct.withDefaultNoData())
     }
   }
 
@@ -568,6 +571,48 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
     }
   }
 
+  describe("array operations") {
+    it("should convert tile into array") {
+      val query = sql(
+        """select rf_tile_to_array_int(
+          |  rf_make_constant_tile(1, 10, 10, 'int8raw')
+          |) as intArray
+          |""".stripMargin)
+      query.as[Array[Int]].first.sum should be (100)
+
+      val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType)
+      val df = Seq[Tile](tile).toDF("tile")
+      val arrayDF = df.select(rf_tile_to_array_double($"tile").as[Array[Double]])
+      arrayDF.first().sum should be (110.0 +- 0.0001)
+
+      checkDocs("rf_tile_to_array_int")
+      checkDocs("rf_tile_to_array_double")
+    }
+
+    it("should convert an array into a tile") {
+      val tile = TestData.randomTile(10, 10, FloatCellType)
+      val df = Seq[Tile](tile, null).toDF("tile")
+      val arrayDF = df.withColumn("tileArray", rf_tile_to_array_double($"tile"))
+
+      val back = arrayDF.withColumn("backToTile", rf_array_to_tile($"tileArray", 10, 10))
+
+      val result = back.select($"backToTile".as[Tile]).first
+
+      assert(result.toArrayDouble() === tile.toArrayDouble())
+
+      // Same round trip, but with SQL expression for rf_array_to_tile
+      val resultSql = arrayDF.selectExpr("rf_array_to_tile(tileArray, 10, 10) as backToTile").as[Tile].first
+
+      assert(resultSql.toArrayDouble() === tile.toArrayDouble())
+
+      val hasNoData = back.withColumn("withNoData", rf_with_no_data($"backToTile", 0))
+
+      val result2 = hasNoData.select($"withNoData".as[Tile]).first
+
+      assert(result2.cellType.asInstanceOf[UserDefinedNoData[_]].noDataValue === 0)
+    }
+  }
+
   describe("analytical transformations") {
     it("should compute rf_normalized_difference") {
       val df = Seq((three, two)).toDF("three", "two")

core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala

@@ -20,7 +20,8 @@
  */
 
 package org.locationtech.rasterframes
-import geotrellis.raster.{CellType, Tile}
+import geotrellis.raster
+import geotrellis.raster.{CellType, NoNoData, Tile}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.rf._
 import org.apache.spark.sql.types.StringType
@@ -98,6 +99,12 @@ class TileUDTSpec extends TestEnvironment with TestData with Inspectors {
       forEveryConfig { tile =>
         val stringified = Seq(tile).toDF("tile").select($"tile".cast(StringType)).as[String].first()
         stringified should be(ShowableTile.show(tile))
+
+        if(!tile.cellType.isInstanceOf[NoNoData]) {
+          val withNd = tile.mutable
+          withNd.update(0, raster.NODATA)
+          ShowableTile.show(withNd) should include("--")
+        }
       }
     }
   }

docs/src/main/paradox/release-notes.md

@@ -4,6 +4,7 @@
 
 ### 0.8.1
 
+* Added `toMarkdown()` and `toHTML()` extension methods for `DataFrame`, and registered them with the IPython formatter system when `rf_ipython` is imported.  
 * Fixed: Removed false return type garauntee in cases where an `Expression` accepts either `Tile` or `ProjectedRasterTile` [(#295)](https://github.com/locationtech/rasterframes/issues/295)
 
 ### 0.8.0

pyrasterframes/src/main/python/docs/raster-catalogs.pymd

@@ -55,7 +55,7 @@ one_d_cat_csv = '\n'.join(['B01', scene1_B01, scene2_B01])
 This is what it looks like in DataFrame form:
 
 ```python, view_spark_cat_1
-one_d_cat_df.show(truncate=False)
+one_d_cat_df
 ```
 
 ### Two-D
@@ -87,7 +87,7 @@ tow_d_cat_csv = '\n'.join(['B01,B02', scene1_B01 + "," + scene1_B02, scene2_B01
 This is what it looks like in DataFrame form:
 
 ```python, view_spark_cat_1
-two_d_cat_df.show(truncate=False)
+two_d_cat_df
 ```
 
 ## Using External Catalogs
@@ -104,11 +104,12 @@ scene_list = spark.read \
     .format("csv") \
     .option("header", "true") \
     .load(SparkFiles.get("2018-07-04_scenes.txt"))
-display(scene_list)
+scene_list
 ```
 
 Observe the scenes list file has URIs to `index.html` files in the download_url column. The image URI's are in the same directory. The filenames are of the form `${gid}_B${band}.TIF`. The next code chunk builds these URIs, which completes our catalog.
 
+
 ```python, show_remote_catalog
 modis_catalog = scene_list \
     .withColumn('base_url',
@@ -117,7 +118,7 @@ modis_catalog = scene_list \
     .withColumn('B01' , F.concat('base_url', F.lit("_B01.TIF"))) \
     .withColumn('B02' , F.concat('base_url', F.lit("_B02.TIF"))) \
     .withColumn('B03' , F.concat('base_url', F.lit("_B03.TIF")))
-display(modis_catalog)
+modis_catalog
 ```
 
 ## Using Built-in Catalogs