Merge remote-tracking branch 'locationtech/develop' into feature/python-raster-reader-arg-refactor

Signed-off-by: Jason T. Brown <[email protected]>

Author: vpipkt

build.sbt

@@ -129,20 +129,28 @@ lazy val experimental = project
 
 lazy val docs = project
   .dependsOn(core, datasource, pyrasterframes)
-  .enablePlugins(SiteScaladocPlugin, ParadoxPlugin, GhpagesPlugin, ScalaUnidocPlugin)
+  .enablePlugins(SiteScaladocPlugin, ParadoxPlugin, ParadoxMaterialThemePlugin, GhpagesPlugin, ScalaUnidocPlugin)
   .settings(
     apiURL := Some(url("http://rasterframes.io/latest/api")),
     autoAPIMappings := true,
     ghpagesNoJekyll := true,
     ScalaUnidoc / siteSubdirName := "latest/api",
     paradox / siteSubdirName := ".",
     paradoxProperties ++= Map(
-      "github.base_url" -> "https://github.com/locationtech/rasterframes",
       "version" -> version.value,
-      "scaladoc.org.apache.spark.sql.rf" -> "http://rasterframes.io/latest"
+      "scaladoc.org.apache.spark.sql.rf" -> "http://rasterframes.io/latest",
+      "github.base_url" -> ""
     ),
     paradoxNavigationExpandDepth := Some(3),
-    paradoxTheme := Some(builtinParadoxTheme("generic")),
+    Compile / paradoxMaterialTheme ~= { _
+      .withRepository(uri("https://github.com/locationtech/rasterframes"))
+      .withCustomStylesheet("assets/custom.css")
+      .withCopyright("""&copy; 2017-2019 <a href="https://astraea.earth">Astraea</a>, Inc. All rights reserved.""")
+      .withLogo("assets/images/RF-R.svg")
+      .withFavicon("assets/images/RasterFrames_32x32.ico")
+      .withColor("blue-grey", "light-blue")
+      .withGoogleAnalytics("UA-106630615-1")
+    },
     makeSite := makeSite
       .dependsOn(Compile / unidoc)
       .dependsOn((Compile / paradox)
@@ -157,6 +165,8 @@ lazy val docs = project
     addMappingsToSiteDir(Compile / paradox / mappings, paradox / siteSubdirName)
   )
 
+//ParadoxMaterialThemePlugin.paradoxMaterialThemeSettings(Paradox)
+
 lazy val bench = project
   .dependsOn(core % "compile->test")
   .settings(publish / skip := true)

core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala

@@ -22,6 +22,7 @@
 package org.locationtech.rasterframes
 import geotrellis.proj4.CRS
 import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
+import geotrellis.raster.render.ColorRamp
 import geotrellis.raster.{CellType, Tile}
 import geotrellis.vector.Extent
 import org.apache.spark.annotation.Experimental
@@ -34,6 +35,7 @@ import org.locationtech.rasterframes.expressions.aggregates._
 import org.locationtech.rasterframes.expressions.generators._
 import org.locationtech.rasterframes.expressions.localops._
 import org.locationtech.rasterframes.expressions.tilestats._
+import org.locationtech.rasterframes.expressions.transformers.RenderPNG.{RenderCompositePNG, RenderColorRampPNG}
 import org.locationtech.rasterframes.expressions.transformers._
 import org.locationtech.rasterframes.model.TileDimensions
 import org.locationtech.rasterframes.stats._
@@ -81,6 +83,10 @@ trait RasterFunctions {
   def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int, ct: CellType): TypedColumn[Any, Tile] =
     rf_convert_cell_type(TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows)), ct).as(cellData.columnName).as[Tile](singlebandTileEncoder)
 
+  /** Create a Tile from a column of cell data with location indexes and perform cell conversion. */
+  def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Int, tileRows: Int): TypedColumn[Any, Tile] =
+    TileAssembler(columnIndex, rowIndex, cellData, lit(tileCols), lit(tileRows))
+
   /** Create a Tile from  a column of cell data with location indexes. */
   def rf_assemble_tile(columnIndex: Column, rowIndex: Column, cellData: Column, tileCols: Column, tileRows: Column): TypedColumn[Any, Tile] =
     TileAssembler(columnIndex, rowIndex, cellData, tileCols, tileRows)
@@ -94,6 +100,18 @@ trait RasterFunctions {
   /** Change the Tile's cell type */
   def rf_convert_cell_type(col: Column, cellTypeName: String): Column = SetCellType(col, cellTypeName)
 
+  /** Change the Tile's cell type */
+  def rf_convert_cell_type(col: Column, cellType: Column): Column = SetCellType(col, cellType)
+
+  /** Change the interpretation of the Tile's cell values according to specified CellType */
+  def rf_interpret_cell_type_as(col: Column, cellType: CellType): Column = InterpretAs(col, cellType)
+
+  /** Change the interpretation of the Tile's cell values according to specified CellType */
+  def rf_interpret_cell_type_as(col: Column, cellTypeName: String): Column = InterpretAs(col, cellTypeName)
+
+  /** Change the interpretation of the Tile's cell values according to specified CellType */
+  def rf_interpret_cell_type_as(col: Column, cellType: Column): Column = InterpretAs(col, cellType)
+
   /** Resample tile to different size based on scalar factor or tile whose dimension to match. Scalar less
     * than one will downsample tile; greater than one will upsample. Uses nearest-neighbor. */
   def rf_resample[T: Numeric](tileCol: Column, factorValue: T) = Resample(tileCol, factorValue)
@@ -317,12 +335,24 @@ trait RasterFunctions {
     ReprojectGeometry(sourceGeom, srcCRSCol, dstCRSCol)
 
   /** Render Tile as ASCII string, for debugging purposes. */
-  def rf_render_ascii(col: Column): TypedColumn[Any, String] =
-    DebugRender.RenderAscii(col)
+  def rf_render_ascii(tile: Column): TypedColumn[Any, String] =
+    DebugRender.RenderAscii(tile)
 
   /** Render Tile cell values as numeric values, for debugging purposes. */
-  def rf_render_matrix(col: Column): TypedColumn[Any, String] =
-    DebugRender.RenderMatrix(col)
+  def rf_render_matrix(tile: Column): TypedColumn[Any, String] =
+    DebugRender.RenderMatrix(tile)
+
+  /** Converts tiles in a column into PNG encoded byte array, using given ColorRamp to assign values to colors. */
+  def rf_render_png(tile: Column, colors: ColorRamp): TypedColumn[Any, Array[Byte]] =
+    RenderColorRampPNG(tile, colors)
+
+  /** Converts columns of tiles representing RGB channels into a PNG encoded byte array. */
+  def rf_render_png(red: Column, green: Column, blue: Column): TypedColumn[Any, Array[Byte]] =
+    RenderCompositePNG(red, green, blue)
+
+  /** Converts columns of tiles representing RGB channels into a single RGB packaged tile. */
+  def rf_rgb_composite(red: Column, green: Column, blue: Column): Column =
+    RGBComposite(red, green, blue)
 
   /** Cellwise less than value comparison between two tiles. */
   def rf_local_less(left: Column, right: Column): Column = Less(left, right)
@@ -359,6 +389,12 @@ trait RasterFunctions {
   /** Cellwise inequality comparison between a tile and a scalar. */
   def rf_local_unequal[T: Numeric](tileCol: Column, value: T): Column = Unequal(tileCol, value)
 
+  /** Return a tile with ones where the input is NoData, otherwise zero */
+  def rf_local_no_data(tileCol: Column): Column = Undefined(tileCol)
+
+  /** Return a tile with zeros where the input is NoData, otherwise one*/
+  def rf_local_data(tileCol: Column): Column = Defined(tileCol)
+
   /** Round cell values to nearest integer without chaning cell type. */
   def rf_round(tileCol: Column): Column = Round(tileCol)
 

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala

@@ -0,0 +1,48 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.expressions.localops
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
+import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
+
+@ExpressionDescription(
+  usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.",
+  arguments = """
+  Arguments:
+    * tile - tile column to inspect""",
+  examples = """
+  Examples:
+    > SELECT  _FUNC_(tile);
+       ..."""
+)
+case class Defined(child: Expression) extends UnaryLocalRasterOp
+  with NullToValue with CodegenFallback {
+  override def nodeName: String = "rf_local_data"
+  override def na: Any = null
+  override protected def op(child: Tile): Tile = child.localDefined()
+}
+object Defined{
+  def apply(tile: Column): Column = new Column(Defined(tile.expr))
+}

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala

@@ -0,0 +1,48 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.expressions.localops
+
+import geotrellis.raster.Tile
+import org.apache.spark.sql.Column
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
+import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
+
+@ExpressionDescription(
+  usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.",
+  arguments = """
+  Arguments:
+    * tile - tile column to inspect""",
+  examples = """
+  Examples:
+    > SELECT  _FUNC_(tile);
+       ..."""
+)
+case class Undefined(child: Expression) extends UnaryLocalRasterOp
+  with NullToValue with CodegenFallback {
+  override def nodeName: String = "rf_local_no_data"
+  override def na: Any = null
+  override protected def op(child: Tile): Tile = child.localUndefined()
+}
+object Undefined{
+  def apply(tile: Column): Column = new Column(Undefined(tile.expr))
+}

core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala

@@ -68,6 +68,7 @@ package object expressions {
     registry.registerExpression[ExplodeTiles]("rf_explode_tiles")
     registry.registerExpression[GetCellType]("rf_cell_type")
     registry.registerExpression[SetCellType]("rf_convert_cell_type")
+    registry.registerExpression[InterpretAs]("rf_interpret_cell_type_as")
     registry.registerExpression[SetNoDataValue]("rf_with_no_data")
     registry.registerExpression[GetDimensions]("rf_dimensions")
     registry.registerExpression[ExtentToGeometry]("st_geometry")
@@ -86,6 +87,8 @@ package object expressions {
     registry.registerExpression[GreaterEqual]("rf_local_greater_equal")
     registry.registerExpression[Equal]("rf_local_equal")
     registry.registerExpression[Unequal]("rf_local_unequal")
+    registry.registerExpression[Undefined]("rf_local_no_data")
+    registry.registerExpression[Defined]("rf_local_data")
     registry.registerExpression[Sum]("rf_tile_sum")
     registry.registerExpression[Round]("rf_round")
     registry.registerExpression[Abs]("rf_abs")
@@ -128,6 +131,9 @@ package object expressions {
 
     registry.registerExpression[DebugRender.RenderAscii]("rf_render_ascii")
     registry.registerExpression[DebugRender.RenderMatrix]("rf_render_matrix")
+    registry.registerExpression[RenderPNG.RenderCompositePNG]("rf_render_png")
+    registry.registerExpression[RGBComposite]("rf_rgb_composite")
+
     registry.registerExpression[transformers.ReprojectGeometry]("st_reproject")
   }
 }

core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/DebugRender.scala

@@ -21,28 +21,29 @@
 
 package org.locationtech.rasterframes.expressions.transformers
 
-import org.locationtech.rasterframes.expressions.UnaryRasterOp
-import org.locationtech.rasterframes.util.TileAsMatrix
-import geotrellis.raster.Tile
 import geotrellis.raster.render.ascii.AsciiArtEncoder
+import geotrellis.raster.{Tile, isNoData}
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
 import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
 import org.apache.spark.sql.types.{DataType, StringType}
 import org.apache.spark.sql.{Column, TypedColumn}
 import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.rasterframes.expressions.UnaryRasterOp
 import org.locationtech.rasterframes.model.TileContext
+import spire.syntax.cfor.cfor
 
 abstract class DebugRender(asciiArt: Boolean) extends UnaryRasterOp
-   with CodegenFallback with Serializable {
-   override def dataType: DataType = StringType
+  with CodegenFallback with Serializable {
+  import org.locationtech.rasterframes.expressions.transformers.DebugRender.TileAsMatrix
+  override def dataType: DataType = StringType
 
-   override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
-     UTF8String.fromString(if (asciiArt)
-       s"\n${tile.renderAscii(AsciiArtEncoder.Palette.NARROW)}\n"
-     else
-       s"\n${tile.renderMatrix(6)}\n"
-     )
-   }
+  override protected def eval(tile: Tile, ctx: Option[TileContext]): Any = {
+    UTF8String.fromString(if (asciiArt)
+      s"\n${tile.renderAscii(AsciiArtEncoder.Palette.NARROW)}\n"
+    else
+      s"\n${tile.renderMatrix(6)}\n"
+    )
+  }
 }
 
 object DebugRender {
@@ -75,4 +76,29 @@ object DebugRender {
     def apply(tile: Column): TypedColumn[Any, String] =
       new Column(RenderMatrix(tile.expr)).as[String]
   }
+
+  implicit class TileAsMatrix(val tile: Tile) extends AnyVal {
+    def renderMatrix(significantDigits: Int): String = {
+      val ND = s"%${significantDigits+5}s".format(Double.NaN)
+      val fmt = s"% ${significantDigits+5}.${significantDigits}g"
+      val buf = new StringBuilder("[")
+      cfor(0)(_ < tile.rows, _ + 1) { row =>
+        if(row > 0) buf.append(' ')
+        buf.append('[')
+        cfor(0)(_ < tile.cols, _ + 1) { col =>
+          val v = tile.getDouble(col, row)
+          if (isNoData(v)) buf.append(ND)
+          else buf.append(fmt.format(v))
+
+          if (col < tile.cols - 1)
+            buf.append(',')
+        }
+        buf.append(']')
+        if (row < tile.rows - 1)
+          buf.append(",\n")
+      }
+      buf.append("]")
+      buf.toString()
+    }
+  }
 }