Skip to content

Commit 7689ace

Browse files
vpipktvpipkt
authored
Additional raster functions: rf_interpret_cell_type_as, rf_local_data, rf_local_no_data (#309)
* Add expression and raster functions around Tile.interpretAs called rf_interpret_cell_type_as * Add rf_local_data and rf_local_no_data #244 Signed-off-by: Jason T. Brown <[email protected]>
2 parents dc050ad + 7623383 commit 7689ace

File tree

11 files changed

+326
-4
lines changed

11 files changed

+326
-4
lines changed

‎core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,18 @@ trait RasterFunctions {
100100
/** Change the Tile's cell type */
101101
def rf_convert_cell_type(col: Column, cellTypeName: String): Column = SetCellType(col, cellTypeName)
102102

103+
/** Change the Tile's cell type */
104+
def rf_convert_cell_type(col: Column, cellType: Column): Column = SetCellType(col, cellType)
105+
106+
/** Change the interpretation of the Tile's cell values according to specified CellType */
107+
def rf_interpret_cell_type_as(col: Column, cellType: CellType): Column = InterpretAs(col, cellType)
108+
109+
/** Change the interpretation of the Tile's cell values according to specified CellType */
110+
def rf_interpret_cell_type_as(col: Column, cellTypeName: String): Column = InterpretAs(col, cellTypeName)
111+
112+
/** Change the interpretation of the Tile's cell values according to specified CellType */
113+
def rf_interpret_cell_type_as(col: Column, cellType: Column): Column = InterpretAs(col, cellType)
114+
103115
/** Resample tile to different size based on scalar factor or tile whose dimension to match. Scalar less
104116
* than one will downsample tile; greater than one will upsample. Uses nearest-neighbor. */
105117
def rf_resample[T: Numeric](tileCol: Column, factorValue: T) = Resample(tileCol, factorValue)
@@ -377,6 +389,12 @@ trait RasterFunctions {
377389
/** Cellwise inequality comparison between a tile and a scalar. */
378390
def rf_local_unequal[T: Numeric](tileCol: Column, value: T): Column = Unequal(tileCol, value)
379391

392+
/** Return a tile with ones where the input is NoData, otherwise zero */
393+
def rf_local_no_data(tileCol: Column): Column = Undefined(tileCol)
394+
395+
/** Return a tile with zeros where the input is NoData, otherwise one*/
396+
def rf_local_data(tileCol: Column): Column = Defined(tileCol)
397+
380398
/** Round cell values to nearest integer without chaning cell type. */
381399
def rf_round(tileCol: Column): Column = Round(tileCol)
382400

‎core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Defined.scala

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.expressions.localops
23+
24+
import geotrellis.raster.Tile
25+
import org.apache.spark.sql.Column
26+
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
27+
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
28+
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
29+
30+
@ExpressionDescription(
31+
usage = "_FUNC_(tile) - Return a tile with zeros where the input is NoData, otherwise one.",
32+
arguments = """
33+
Arguments:
34+
* tile - tile column to inspect""",
35+
examples = """
36+
Examples:
37+
> SELECT _FUNC_(tile);
38+
..."""
39+
)
40+
case class Defined(child: Expression) extends UnaryLocalRasterOp
41+
with NullToValue with CodegenFallback {
42+
override def nodeName: String = "rf_local_data"
43+
override def na: Any = null
44+
override protected def op(child: Tile): Tile = child.localDefined()
45+
}
46+
object Defined{
47+
def apply(tile: Column): Column = new Column(Defined(tile.expr))
48+
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/localops/Undefined.scala

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.expressions.localops
23+
24+
import geotrellis.raster.Tile
25+
import org.apache.spark.sql.Column
26+
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
27+
import org.apache.spark.sql.catalyst.expressions.{Expression, ExpressionDescription}
28+
import org.locationtech.rasterframes.expressions.{NullToValue, UnaryLocalRasterOp}
29+
30+
@ExpressionDescription(
31+
usage = "_FUNC_(tile) - Return a tile with ones where the input is NoData, otherwise zero.",
32+
arguments = """
33+
Arguments:
34+
* tile - tile column to inspect""",
35+
examples = """
36+
Examples:
37+
> SELECT _FUNC_(tile);
38+
..."""
39+
)
40+
case class Undefined(child: Expression) extends UnaryLocalRasterOp
41+
with NullToValue with CodegenFallback {
42+
override def nodeName: String = "rf_local_no_data"
43+
override def na: Any = null
44+
override protected def op(child: Tile): Tile = child.localUndefined()
45+
}
46+
object Undefined{
47+
def apply(tile: Column): Column = new Column(Undefined(tile.expr))
48+
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ package object expressions {
6868
registry.registerExpression[ExplodeTiles]("rf_explode_tiles")
6969
registry.registerExpression[GetCellType]("rf_cell_type")
7070
registry.registerExpression[SetCellType]("rf_convert_cell_type")
71+
registry.registerExpression[InterpretAs]("rf_interpret_cell_type_as")
7172
registry.registerExpression[SetNoDataValue]("rf_with_no_data")
7273
registry.registerExpression[GetDimensions]("rf_dimensions")
7374
registry.registerExpression[ExtentToGeometry]("st_geometry")
@@ -86,6 +87,8 @@ package object expressions {
8687
registry.registerExpression[GreaterEqual]("rf_local_greater_equal")
8788
registry.registerExpression[Equal]("rf_local_equal")
8889
registry.registerExpression[Unequal]("rf_local_unequal")
90+
registry.registerExpression[Undefined]("rf_local_no_data")
91+
registry.registerExpression[Defined]("rf_local_data")
8992
registry.registerExpression[Sum]("rf_tile_sum")
9093
registry.registerExpression[Round]("rf_round")
9194
registry.registerExpression[Abs]("rf_abs")

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/InterpretAs.scala

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.expressions.transformers
23+
24+
import geotrellis.raster.CellType
25+
import org.apache.spark.sql.Column
26+
import org.apache.spark.sql.catalyst.InternalRow
27+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
28+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
29+
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
30+
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
31+
import org.apache.spark.sql.functions.lit
32+
import org.apache.spark.sql.rf.{TileUDT}
33+
import org.apache.spark.sql.types._
34+
import org.apache.spark.unsafe.types.UTF8String
35+
import org.locationtech.rasterframes.encoders.CatalystSerializer._
36+
import org.locationtech.rasterframes.expressions.DynamicExtractors.tileExtractor
37+
import org.locationtech.rasterframes.expressions.row
38+
39+
@ExpressionDescription(
40+
usage = "_FUNC_(tile, value) - Change the interpretation of the Tile's cell values according to specified CellType",
41+
arguments = """
42+
Arguments:
43+
* tile - left-hand-side tile
44+
* rhs - a cell type definition""",
45+
examples = """
46+
Examples:
47+
> SELECT _FUNC_(tile, 'int16ud0');
48+
..."""
49+
)
50+
case class InterpretAs(tile: Expression, cellType: Expression)
51+
extends BinaryExpression with CodegenFallback {
52+
def left = tile
53+
def right = cellType
54+
override def nodeName: String = "rf_interpret_cell_type_as"
55+
override def dataType: DataType = left.dataType
56+
57+
override def checkInputDataTypes(): TypeCheckResult = {
58+
if (!tileExtractor.isDefinedAt(left.dataType))
59+
TypeCheckFailure(s"Input type '${left.dataType}' does not conform to a raster type.")
60+
else
61+
right.dataType match {
62+
case StringType => TypeCheckSuccess
63+
case t if t.conformsTo[CellType] => TypeCheckSuccess
64+
case _ =>
65+
TypeCheckFailure(s"Expected CellType but received '${right.dataType.simpleString}'")
66+
}
67+
}
68+
69+
private def toCellType(datum: Any): CellType = {
70+
right.dataType match {
71+
case StringType =>
72+
val text = datum.asInstanceOf[UTF8String].toString
73+
CellType.fromName(text)
74+
case st if st.conformsTo[CellType] =>
75+
row(datum).to[CellType]
76+
}
77+
}
78+
79+
override protected def nullSafeEval(tileInput: Any, ctInput: Any): InternalRow = {
80+
implicit val tileSer = TileUDT.tileSerializer
81+
82+
val (tile, ctx) = tileExtractor(left.dataType)(row(tileInput))
83+
val ct = toCellType(ctInput)
84+
val result = tile.interpretAs(ct)
85+
86+
ctx match {
87+
case Some(c) => c.toProjectRasterTile(result).toInternalRow
88+
case None => result.toInternalRow
89+
}
90+
}
91+
}
92+
93+
object InterpretAs{
94+
def apply(tile: Column, cellType: CellType): Column =
95+
new Column(new InterpretAs(tile.expr, lit(cellType.name).expr))
96+
def apply(tile: Column, cellType: String): Column =
97+
new Column(new InterpretAs(tile.expr, lit(cellType).expr))
98+
def apply(tile: Column, cellType: Column): Column =
99+
new Column(new InterpretAs(tile.expr, cellType.expr))
100+
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/SetCellType.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,4 +101,6 @@ object SetCellType {
101101
new Column(new SetCellType(tile.expr, lit(cellType.name).expr))
102102
def apply(tile: Column, cellType: String): Column =
103103
new Column(new SetCellType(tile.expr, lit(cellType).expr))
104+
def apply(tile: Column, cellType: Column): Column =
105+
new Column(new SetCellType(tile.expr, cellType.expr))
104106
}

‎core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
5656
lazy val six = ProjectedRasterTile(three * two, three.extent, three.crs)
5757
lazy val nd = TestData.projectedRasterTile(cols, rows, -2, extent, crs, ct)
5858
lazy val randPRT = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextInt(), extent, crs, ct)
59-
lazy val randNDPRT = TestData.injectND(numND)(randPRT)
59+
lazy val randNDPRT: Tile = TestData.injectND(numND)(randPRT)
6060

6161
lazy val randDoubleTile = TestData.projectedRasterTile(cols, rows, scala.util.Random.nextGaussian(), extent, crs, DoubleConstantNoDataCellType)
6262
lazy val randDoubleNDTile = TestData.injectND(numND)(randDoubleTile)
@@ -948,4 +948,38 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
948948
image.getWidth should be(red.cols)
949949
image.getHeight should be(red.rows)
950950
}
951+
it("should interpret cell values with a specified cell type") {
952+
checkDocs("rf_interpret_cell_type_as")
953+
val df = Seq(randNDPRT).toDF("t")
954+
.withColumn("tile", rf_interpret_cell_type_as($"t", "int8raw"))
955+
val resultTile = df.select("tile").as[Tile].first()
956+
957+
resultTile.cellType should be (CellType.fromName("int8raw"))
958+
// should have same number of values that are -2 the old ND
959+
val countOldNd = df.select(
960+
rf_tile_sum(rf_local_equal($"tile", ct.noDataValue)),
961+
rf_no_data_cells($"t")
962+
).first()
963+
countOldNd._1 should be (countOldNd._2)
964+
965+
// should not have no data any more (raw type)
966+
val countNewNd = df.select(rf_no_data_cells($"tile")).first()
967+
countNewNd should be (0L)
968+
969+
}
970+
971+
it("should return local data and nodata"){
972+
checkDocs("rf_local_data")
973+
checkDocs("rf_local_no_data")
974+
975+
val df = Seq(randNDPRT).toDF("t")
976+
.withColumn("ld", rf_local_data($"t"))
977+
.withColumn("lnd", rf_local_no_data($"t"))
978+
979+
val ndResult = df.select($"lnd").as[Tile].first()
980+
ndResult should be (randNDPRT.localUndefined())
981+
982+
val dResult = df.select($"ld").as[Tile].first()
983+
dResult should be (randNDPRT.localDefined())
984+
}
951985
}

‎docs/src/main/paradox/release-notes.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@
44

55
### 0.8.1
66

7+
* Added `rf_local_no_data`, `rf_local_data` and `rf_interpret_cell_type_as` raster functions.
78
* Added: `rf_rgb_composite` and `rf_render_png`.
89
* Added `toMarkdown` and `toHTML` extension methods for `DataFrame`, and registered them with the IPython formatter system when `rf_ipython` is imported.
910
* New documentation theme (thanks [@jonas](https://github.com/jonas)!).
10-
* Fixed: Removed false return type garauntee in cases where an `Expression` accepts either `Tile` or `ProjectedRasterTile` [(#295)](https://github.com/locationtech/rasterframes/issues/295)
11+
* Fixed: Removed false return type guarantee in cases where an `Expression` accepts either `Tile` or `ProjectedRasterTile` [(#295)](https://github.com/locationtech/rasterframes/issues/295)
1112

1213
### 0.8.0
1314

‎pyrasterframes/src/main/python/docs/reference.pymd

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,17 @@ Example: `SELECT rf_mk_crs('EPSG:4326')`
115115

116116
### rf_convert_cell_type
117117

118-
Tile rf_convert_cell_type(Tile tileCol, String cellType)
118+
Tile rf_convert_cell_type(Tile tile_col, CellType cell_type)
119+
Tile rf_convert_cell_type(Tile tile_col, String cell_type)
119120

120-
Convert `tileCol` to a different cell type. Available cell types can be retrieved with the @ref:[rf_cell_types](reference.md#rf-cell-types) function.
121+
Convert `tile_col` to a different cell type. In Python you can pass a CellType object to `cell_type`.
122+
123+
### rf_interpret_cell_type_as
124+
125+
Tile rf_interpret_cell_type_as(Tile tile_col, CellType cell_type)
126+
Tile rf_interpret_cell_type_as(Tile tile_col, String cell_type)
127+
128+
Change the interpretation of the `tile_col`'s cell values according to specified `cell_type`. In Python you can pass a CellType object to `cell_type`.
121129

122130
### rf_resample
123131

@@ -224,6 +232,20 @@ Generate a `tile` with the values from `data_tile`, with NoData in cells where t
224232

225233
Returns true if `tile` contains only NoData. By definition returns false if cell type does not support NoData. To count NoData cells or data cells, see @ref:[`rf_no_data_cells`](reference.md#rf-no-data-cells), @ref:[`rf_data_cells`](reference.md#rf-data-cells), @ref:[`rf_agg_no_data_cells`](reference.md#rf-agg-no-data-cells), @ref:[`rf_agg_data_cells`](reference.md#rf-agg-data-cells), @ref:[`rf_agg_local_no_data_cells`](reference.md#rf-agg-local-no-data-cells), and @ref:[`rf_agg_local_data_cells`](reference.md#rf-agg-local-data-cells). This function is distinguished from @ref:[`rf_for_all`](reference.md#rf-for-all), which tests that values are not NoData and nonzero.
226234

235+
### rf_local_no_data
236+
237+
Tile rf_local_no_data(Tile tile)
238+
239+
Returns a tile with values of 1 in each cell where the input tile contains NoData. Otherwise values are 0.
240+
241+
### rf_local_data
242+
243+
Tile rf_local_no_data(Tile tile)
244+
245+
Returns a tile with values of 0 in each cell where the input tile contains NoData. Otherwise values are 1.
246+
247+
### rf_local_data
248+
227249
### rf_with_no_data
228250

229251
Tile rf_with_no_data(Tile tile, Double no_data_value)

‎pyrasterframes/src/main/python/pyrasterframes/rasterfunctions.py

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,11 @@ def rf_convert_cell_type(tile_col, cell_type):
8686
jfcn = RFContext.active().lookup('rf_convert_cell_type')
8787
return Column(jfcn(_to_java_column(tile_col), _parse_cell_type(cell_type)))
8888

89+
def rf_interpret_cell_type_as(tile_col, cell_type):
90+
"""Change the interpretation of the tile_col's cell values according to specified cell_type"""
91+
jfcn = RFContext.active().lookup('rf_interpret_cell_type_as')
92+
return Column(jfcn(_to_java_column(tile_col), _parse_cell_type(cell_type)))
93+
8994

9095
def rf_make_constant_tile(scalar_value, num_cols, num_rows, cell_type=CellType.float64()):
9196
"""Constructor for constant tile column"""
@@ -255,6 +260,13 @@ def rf_local_unequal_int(tile_col, scalar):
255260
"""Return a Tile with values equal 1 if the cell is not equal to a scalar, otherwise 0"""
256261
return _apply_scalar_to_tile('rf_local_unequal_int', tile_col, scalar)
257262

263+
def rf_local_no_data(tile_col):
264+
"""Return a tile with ones where the input is NoData, otherwise zero."""
265+
return _apply_column_function('rf_local_no_data', tile_col)
266+
267+
def rf_local_data(tile_col):
268+
"""Return a tile with zeros where the input is NoData, otherwise one."""
269+
return _apply_column_function('rf_local_data', tile_col)
258270

259271
def _apply_column_function(name, *args):
260272
jfcn = RFContext.active().lookup(name)

0 commit comments

Comments
 (0)