Merge pull request #305 from s22s/fix/304

vpipkt · web-flow · commit 1ea29f2ccdc7 · 2019-08-21T11:22:16.000-04:00
Fixes 304; improves ShowableTile.
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala
@@ -57,7 +57,7 @@ case class SetCellType(tile: Expression, cellType: Expression)
   extends BinaryExpression with CodegenFallback {
   def left = tile
   def right = cellType
-  override def nodeName: String = "set_cell_type"
+  override def nodeName: String = "rf_convert_cell_type"
   override def dataType: DataType = left.dataType
 
   override def checkInputDataTypes(): TypeCheckResult = {
diff --git a/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala b/core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetNoDataValue.scala
@@ -22,7 +22,6 @@
 package org.locationtech.rasterframes.expressions.transformers
 
 import com.typesafe.scalalogging.LazyLogging
-import geotrellis.raster.Tile
 import org.apache.spark.sql.Column
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
@@ -33,7 +32,7 @@ import org.apache.spark.sql.rf.TileUDT
 import org.apache.spark.sql.types._
 import org.locationtech.rasterframes.encoders.CatalystSerializer._
 import org.locationtech.rasterframes.expressions.DynamicExtractors._
-import org.locationtech.rasterframes.expressions.{fpTile, row}
+import org.locationtech.rasterframes.expressions.row
 
 @ExpressionDescription(
   usage = "_FUNC_(tile, value) - Set the NoData value for the given tile.",
@@ -64,20 +63,19 @@ case class SetNoDataValue(left: Expression, right: Expression) extends BinaryExp
   override protected def nullSafeEval(input1: Any, input2: Any): Any = {
     implicit val tileSer = TileUDT.tileSerializer
     val (leftTile, leftCtx) = tileExtractor(left.dataType)(row(input1))
+
     val result = numberArgExtractor(right.dataType)(input2) match {
-      case DoubleArg(d) => op(fpTile(leftTile), d)
-      case IntegerArg(i) => op(leftTile, i)
+      case DoubleArg(d) => leftTile.withNoData(Some(d))
+      case IntegerArg(i) => leftTile.withNoData(Some(i.toDouble))
     }
 
     leftCtx match {
       case Some(ctx) => ctx.toProjectRasterTile(result).toInternalRow
       case None => result.toInternalRow
     }
   }
-
-  protected def op(left: Tile, right: Double): Tile = left.withNoData(Some(right))
-  protected def op(left: Tile, right: Int): Tile = left.withNoData(Some(right))
 }
+
 object SetNoDataValue {
   def apply(left: Column, right: Column): Column =
     new Column(SetNoDataValue(left.expr, right.expr))
diff --git a/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala b/core/src/main/scala/org/locationtech/rasterframes/functions/package.scala
@@ -90,8 +90,6 @@ package object functions {
     }
   }
 
-
-
   /** Alias for constant tiles of zero */
   private[rasterframes] val tileZeros: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒
     makeConstantTile(0, cols, rows, cellTypeName)
diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ShowableTile.scala
@@ -21,7 +21,7 @@
 
 package org.locationtech.rasterframes.tiles
 import org.locationtech.rasterframes._
-import geotrellis.raster.Tile
+import geotrellis.raster.{Tile, isNoData}
 
 class ShowableTile(val delegate: Tile) extends FixedDelegatingTile {
   override def equals(obj: Any): Boolean = obj match {
@@ -39,8 +39,14 @@ object ShowableTile {
     val dims = tile.dimensions
 
     val data = if (tile.cellType.isFloatingPoint)
-      tile.toArrayDouble()
-    else tile.toArray()
+      tile.toArrayDouble().map {
+        case c if isNoData(c) => "--"
+        case c => c.toString
+      }
+    else tile.toArray().map {
+      case c if isNoData(c) => "--"
+      case c => c.toString
+    }
 
     val cells = if(tile.size <= maxCells) {
       data.mkString("[", ",", "]")
diff --git a/core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ExplodeSpec.scala
@@ -110,45 +110,6 @@ class ExplodeSpec extends TestEnvironment with TestData {
       cells.count(_.isNaN) should be(tiles.size)
     }
 
-    it("should convert tile into array") {
-      val query = sql(
-        """select rf_tile_to_array_int(
-          |  rf_make_constant_tile(1, 10, 10, 'int8raw')
-          |) as intArray
-          |""".stripMargin)
-      query.as[Array[Int]].first.sum should be (100)
-
-      val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType)
-      val df = Seq[Tile](tile).toDF("tile")
-      val arrayDF = df.select(rf_tile_to_array_double($"tile").as[Array[Double]])
-      arrayDF.first().sum should be (110.0 +- 0.0001)
-
-      checkDocs("rf_tile_to_array_int")
-    }
-
-    it("should convert an array into a tile") {
-      val tile = TestData.randomTile(10, 10, FloatCellType)
-      val df = Seq[Tile](tile, null).toDF("tile")
-      val arrayDF = df.withColumn("tileArray", rf_tile_to_array_double($"tile"))
-
-      val back = arrayDF.withColumn("backToTile", rf_array_to_tile($"tileArray", 10, 10))
-
-      val result = back.select($"backToTile".as[Tile]).first
-
-      assert(result.toArrayDouble() === tile.toArrayDouble())
-
-      // Same round trip, but with SQL expression for rf_array_to_tile
-      val resultSql = arrayDF.selectExpr("rf_array_to_tile(tileArray, 10, 10) as backToTile").as[Tile].first
-
-      assert(resultSql.toArrayDouble() === tile.toArrayDouble())
-
-      val hasNoData = back.withColumn("withNoData", rf_with_no_data($"backToTile", 0))
-
-      val result2 = hasNoData.select($"withNoData".as[Tile]).first
-
-      assert(result2.cellType.asInstanceOf[UserDefinedNoData[_]].noDataValue === 0)
-    }
-
     it("should reassemble single exploded tile") {
       val tile = TestData.randomTile(10, 10, FloatCellType)
       val df = Seq[Tile](tile).toDF("tile")
diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala
@@ -120,17 +120,20 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
     it("should change NoData value") {
       val df = Seq((TestData.injectND(7)(three), TestData.injectND(12)(two))).toDF("three", "two")
 
-      val ct = df.select(
+      val ndCT = df.select(
         rf_with_no_data($"three", 3) as "three",
-        rf_with_no_data($"two", 2) as "two"
+        rf_with_no_data($"two", 2.0) as "two"
       )
 
-      val (cnt3, cnt2) = ct.select(rf_no_data_cells($"three"), rf_no_data_cells($"two")).as[(Long, Long)].first()
+      val (cnt3, cnt2) = ndCT.select(rf_no_data_cells($"three"), rf_no_data_cells($"two")).as[(Long, Long)].first()
 
       cnt3 should be ((cols * rows) - 7)
       cnt2 should be ((cols * rows) - 12)
 
       checkDocs("rf_with_no_data")
+
+      // Should maintain original cell type.
+      ndCT.select(rf_cell_type($"two")).first().withDefaultNoData() should be(ct.withDefaultNoData())
     }
   }
 
@@ -568,6 +571,48 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
     }
   }
 
+  describe("array operations") {
+    it("should convert tile into array") {
+      val query = sql(
+        """select rf_tile_to_array_int(
+          |  rf_make_constant_tile(1, 10, 10, 'int8raw')
+          |) as intArray
+          |""".stripMargin)
+      query.as[Array[Int]].first.sum should be (100)
+
+      val tile = FloatConstantTile(1.1f, 10, 10, FloatCellType)
+      val df = Seq[Tile](tile).toDF("tile")
+      val arrayDF = df.select(rf_tile_to_array_double($"tile").as[Array[Double]])
+      arrayDF.first().sum should be (110.0 +- 0.0001)
+
+      checkDocs("rf_tile_to_array_int")
+      checkDocs("rf_tile_to_array_double")
+    }
+
+    it("should convert an array into a tile") {
+      val tile = TestData.randomTile(10, 10, FloatCellType)
+      val df = Seq[Tile](tile, null).toDF("tile")
+      val arrayDF = df.withColumn("tileArray", rf_tile_to_array_double($"tile"))
+
+      val back = arrayDF.withColumn("backToTile", rf_array_to_tile($"tileArray", 10, 10))
+
+      val result = back.select($"backToTile".as[Tile]).first
+
+      assert(result.toArrayDouble() === tile.toArrayDouble())
+
+      // Same round trip, but with SQL expression for rf_array_to_tile
+      val resultSql = arrayDF.selectExpr("rf_array_to_tile(tileArray, 10, 10) as backToTile").as[Tile].first
+
+      assert(resultSql.toArrayDouble() === tile.toArrayDouble())
+
+      val hasNoData = back.withColumn("withNoData", rf_with_no_data($"backToTile", 0))
+
+      val result2 = hasNoData.select($"withNoData".as[Tile]).first
+
+      assert(result2.cellType.asInstanceOf[UserDefinedNoData[_]].noDataValue === 0)
+    }
+  }
+
   describe("analytical transformations") {
     it("should compute rf_normalized_difference") {
       val df = Seq((three, two)).toDF("three", "two")
diff --git a/core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/TileUDTSpec.scala
@@ -20,7 +20,8 @@
  */
 
 package org.locationtech.rasterframes
-import geotrellis.raster.{CellType, Tile}
+import geotrellis.raster
+import geotrellis.raster.{CellType, NoNoData, Tile}
 import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
 import org.apache.spark.sql.rf._
 import org.apache.spark.sql.types.StringType
@@ -98,6 +99,12 @@ class TileUDTSpec extends TestEnvironment with TestData with Inspectors {
       forEveryConfig { tile =>
         val stringified = Seq(tile).toDF("tile").select($"tile".cast(StringType)).as[String].first()
         stringified should be(ShowableTile.show(tile))
+
+        if(!tile.cellType.isInstanceOf[NoNoData]) {
+          val withNd = tile.mutable
+          withNd.update(0, raster.NODATA)
+          ShowableTile.show(withNd) should include("--")
+        }
       }
     }
   }
diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py b/pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py
@@ -153,7 +153,7 @@ def _safe_tile_to_html(t):
     return return_html
 
 
-def spark_df_to_markdown(df, num_rows=5, truncate=True, vertical=False):
+def spark_df_to_markdown(df, num_rows=5, truncate=False, vertical=False):
     from pyrasterframes import RFContext
     return RFContext.active().call("_dfToMarkdown", df._jdf, num_rows, truncate)
 

Original file line number	Diff line number	Diff line change
`@@ -90,8 +90,6 @@ package object functions {`
`90`	`90`	`}`
`91`	`91`	`}`
`92`	`92`
`93`		`-`
`94`		`-`
`95`	`93`	`/** Alias for constant tiles of zero */`
`96`	`94`	`private[rasterframes] val tileZeros: (Int, Int, String) ⇒ Tile = (cols, rows, cellTypeName) ⇒`
`97`	`95`	`makeConstantTile(0, cols, rows, cellTypeName)`