Fixed Add expression to use GT behavior; created new BiasedAdd for Aggregate use.

Author: metasim

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/LocalMeanAggregate.scala

@@ -22,7 +22,7 @@
 package org.locationtech.rasterframes.expressions.aggregates
 
 import org.locationtech.rasterframes.expressions.UnaryRasterAggregate
-import org.locationtech.rasterframes.expressions.localops.{Add => AddTiles, Divide => DivideTiles}
+import org.locationtech.rasterframes.expressions.localops.{BiasedAdd, Divide => DivideTiles}
 import org.locationtech.rasterframes.expressions.transformers.SetCellType
 import geotrellis.raster.Tile
 import geotrellis.raster.mapalgebra.local
@@ -59,16 +59,16 @@ case class LocalMeanAggregate(child: Expression) extends UnaryRasterAggregate {
   override lazy val updateExpressions: Seq[Expression] = Seq(
     If(IsNull(count),
       SetCellType(Defined(child), Literal("int32")),
-      If(IsNull(child), count, AddTiles(count, Defined(child)))
+      If(IsNull(child), count, BiasedAdd(count, Defined(child)))
     ),
     If(IsNull(sum),
       SetCellType(child, Literal("float64")),
-      If(IsNull(child), sum, AddTiles(sum, child))
+      If(IsNull(child), sum, BiasedAdd(sum, child))
     )
   )
   override val mergeExpressions: Seq[Expression] = Seq(
-    AddTiles(count.left, count.right),
-    AddTiles(sum.left, sum.right)
+    BiasedAdd(count.left, count.right),
+    BiasedAdd(sum.left, sum.right)
   )
   override lazy val evaluateExpression: Expression = DivideTiles(sum, count)
 }

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Add.scala

@@ -30,7 +30,6 @@ import org.apache.spark.sql.{Column, TypedColumn}
 import org.locationtech.rasterframes._
 import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
 import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
-import org.locationtech.rasterframes.util.DataBiasedOp.BiasedAdd
 
 @ExpressionDescription(
   usage = "_FUNC_(tile, rhs) - Performs cell-wise addition between two tiles or a tile and a scalar.",
@@ -48,9 +47,9 @@ import org.locationtech.rasterframes.util.DataBiasedOp.BiasedAdd
 case class Add(left: Expression, right: Expression) extends BinaryLocalRasterOp
   with CodegenFallback {
   override val nodeName: String = "rf_local_add"
-  override protected def op(left: Tile, right: Tile): Tile = BiasedAdd(left, right)
-  override protected def op(left: Tile, right: Double): Tile = BiasedAdd(left, right)
-  override protected def op(left: Tile, right: Int): Tile = BiasedAdd(left, right)
+  override protected def op(left: Tile, right: Tile): Tile = left.localAdd(right)
+  override protected def op(left: Tile, right: Double): Tile = left.localAdd(right)
+  override protected def op(left: Tile, right: Int): Tile = left.localAdd(right)
 
   override def eval(input: InternalRow): Any = {
     if(input == null) null

core/src/main/scala/org/locationtech/rasterframes/expressions/localops/BiasedAdd.scala

@@ -0,0 +1,74 @@
+/*
+ * This software is licensed under the Apache 2 license, quoted below.
+ *
+ * Copyright 2019 Astraea, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *     [http://www.apache.org/licenses/LICENSE-2.0]
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ */
+
+package org.locationtech.rasterframes.expressions.localops
+import geotrellis.raster.Tile
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
+import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
+import org.apache.spark.sql.functions.lit
+import org.apache.spark.sql.{Column, TypedColumn}
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.expressions.BinaryLocalRasterOp
+import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
+import org.locationtech.rasterframes.util.DataBiasedOp
+
+@ExpressionDescription(
+  usage = "_FUNC_(tile, rhs) - Performs cell-wise addition between two tiles or a tile and a scalar. " +
+    "Unlike a regular 'add', this considers `<data> + <nodata> = <data>.",
+  arguments = """
+  Arguments:
+    * tile - left-hand-side tile
+    * rhs  - a tile or scalar value to add to each cell""",
+  examples = """
+  Examples:
+    > SELECT _FUNC_(tile, 1.5);
+       ...
+    > SELECT _FUNC_(tile1, tile2);
+       ..."""
+)
+case class BiasedAdd(left: Expression, right: Expression) extends BinaryLocalRasterOp
+  with CodegenFallback {
+  override val nodeName: String = "rf_local_biased_add"
+  override protected def op(left: Tile, right: Tile): Tile = DataBiasedOp.BiasedAdd(left, right)
+  override protected def op(left: Tile, right: Double): Tile = DataBiasedOp.BiasedAdd(left, right)
+  override protected def op(left: Tile, right: Int): Tile = DataBiasedOp.BiasedAdd(left, right)
+
+  override def eval(input: InternalRow): Any = {
+    if(input == null) null
+    else {
+      val l = left.eval(input)
+      val r = right.eval(input)
+      if (l == null && r == null) null
+      else if (l == null) r
+      else if (r == null && tileExtractor.isDefinedAt(right.dataType)) l
+      else if (r == null) null
+      else nullSafeEval(l, r)
+    }
+  }
+}
+object BiasedAdd {
+  def apply(left: Column, right: Column): TypedColumn[Any, Tile] =
+    new Column(BiasedAdd(left.expr, right.expr)).as[Tile]
+
+  def apply[N: Numeric](tile: Column, value: N): TypedColumn[Any, Tile] =
+    new Column(BiasedAdd(tile.expr, lit(value).expr)).as[Tile]
+}

pyrasterframes/src/main/python/setup.py

@@ -152,7 +152,8 @@ def initialize_options(self):
         'numpy>=1.7',
         'shapely',
         'pandas',
-        'rasterio'
+        'rasterio',
+        'boto3'
     ],
     packages=[
         'pyrasterframes',