Skip to content

Commit 76e2241

Browse files
vpipktmetasim
vpipkt
authored and
metasim
committed
Added tile subdivision capability to GeoTrellisRelation.
Updated tests for subdivision capability. Improved random tile generation code. Signed-off-by: Simeon H.K. Fitch <[email protected]>
1 parent 8b3ff5a commit 76e2241

File tree

19 files changed

+602
-140
lines changed

19 files changed

+602
-140
lines changed

‎core/src/main/scala/astraea/spark/rasterframes/extensions/ProjectedRasterMethods.scala‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,21 @@ package astraea.spark.rasterframes.extensions
22

33
import java.time.ZonedDateTime
44

5+
import astraea.spark.rasterframes.util._
56
import astraea.spark.rasterframes.{RasterFrame, StandardColumns}
6-
import geotrellis.raster.{ProjectedRaster, Tile, TileLayout}
7+
import geotrellis.raster.{ProjectedRaster, Tile}
78
import geotrellis.spark._
89
import geotrellis.spark.tiling._
9-
import geotrellis.spark.io._
1010
import geotrellis.util.MethodExtensions
1111
import org.apache.spark.sql.SparkSession
12-
import astraea.spark.rasterframes.util._
1312

1413
/**
1514
* Extension methods on [[ProjectedRaster]] for creating [[RasterFrame]]s.
1615
*
1716
* @since 8/10/17
1817
*/
1918
trait ProjectedRasterMethods extends MethodExtensions[ProjectedRaster[Tile]] with StandardColumns {
20-
import Implicits.WithSpatialContextRDDMethods
21-
import Implicits.WithSpatioTemporalContextRDDMethods
19+
import Implicits.{WithSpatialContextRDDMethods, WithSpatioTemporalContextRDDMethods}
2220

2321
/**
2422
* Convert the wrapped [[ProjectedRaster]] into a [[RasterFrame]] with a

‎core/src/main/scala/astraea/spark/rasterframes/extensions/RasterFrameMethods.scala‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -21,15 +21,14 @@ import java.time.ZonedDateTime
2121
import astraea.spark.rasterframes.util._
2222
import astraea.spark.rasterframes.{MetadataKeys, RasterFrame}
2323
import geotrellis.proj4.CRS
24-
import geotrellis.raster.resample.{Bilinear, ResampleMethod}
25-
import geotrellis.raster.{CellGrid, MultibandTile, ProjectedRaster, Tile, TileLayout}
24+
import geotrellis.raster.resample.{NearestNeighbor, ResampleMethod}
25+
import geotrellis.raster.{MultibandTile, ProjectedRaster, Tile, TileLayout}
2626
import geotrellis.spark._
2727
import geotrellis.spark.io._
2828
import geotrellis.spark.tiling.{LayoutDefinition, Tiler}
2929
import geotrellis.util.{LazyLogging, MethodExtensions}
3030
import geotrellis.vector.ProjectedExtent
3131
import org.apache.spark.annotation.Experimental
32-
import org.apache.spark.rdd.RDD
3332
import org.apache.spark.sql._
3433
import org.apache.spark.sql.functions._
3534
import org.apache.spark.sql.types.{Metadata, TimestampType}
@@ -302,7 +301,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame]
302301
def toRaster(tileCol: Column,
303302
rasterCols: Int,
304303
rasterRows: Int,
305-
resampler: ResampleMethod = Bilinear): ProjectedRaster[Tile] = {
304+
resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[Tile] = {
306305

307306
val clipped = clipLayerExtent
308307

@@ -338,7 +337,7 @@ trait RasterFrameMethods extends MethodExtensions[RasterFrame]
338337
tileCols: Seq[Column],
339338
rasterCols: Int,
340339
rasterRows: Int,
341-
resampler: ResampleMethod = Bilinear): ProjectedRaster[MultibandTile] = {
340+
resampler: ResampleMethod = NearestNeighbor): ProjectedRaster[MultibandTile] = {
342341

343342
val clipped = clipLayerExtent
344343

‎core/src/main/scala/astraea/spark/rasterframes/package.scala‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,14 +18,16 @@ package astraea.spark
1818

1919
import astraea.spark.rasterframes.encoders.StandardEncoders
2020
import geotrellis.raster.{Tile, TileFeature}
21-
import geotrellis.spark.{Bounds, ContextRDD, Metadata, TileLayerMetadata}
22-
import geotrellis.util.GetComponent
21+
import geotrellis.spark.{ContextRDD, Metadata, SpaceTimeKey, SpatialKey, TileLayerMetadata}
2322
import org.apache.spark.rdd.RDD
2423
import org.apache.spark.sql._
2524
import org.locationtech.geomesa.spark.jts.DataFrameFunctions
2625
import org.locationtech.geomesa.spark.jts.encoders.SpatialEncoders
2726
import shapeless.tag.@@
2827

28+
import scala.language.higherKinds
29+
import scala.reflect.runtime.universe._
30+
2931
/**
3032
* Module providing support for RasterFrames.
3133
* `import astraea.spark.rasterframes._`., and then call `rfInit(SQLContext)`.
@@ -64,14 +66,6 @@ package object rasterframes extends StandardColumns
6466
/** Tagged type for allowing compiler to help keep track of what has RasterFrame assurances applied to it. */
6567
trait RasterFrameTag
6668

67-
/**
68-
* Type lambda alias for components that have bounds with parameterized key.
69-
* @tparam K bounds key type
70-
*/
71-
type BoundsComponentOf[K] = {
72-
type get[M] = GetComponent[M, Bounds[K]]
73-
}
74-
7569
type TileFeatureLayerRDD[K, D] =
7670
RDD[(K, TileFeature[Tile, D])] with Metadata[TileLayerMetadata[K]]
7771

@@ -81,6 +75,7 @@ package object rasterframes extends StandardColumns
8175
new ContextRDD(rdd, metadata)
8276
}
8377

78+
/** Provides evidence that a given primitive has an associated CellType. */
8479
trait HasCellType[T] extends Serializable
8580
object HasCellType {
8681
implicit val intHasCellType = new HasCellType[Int] {}
@@ -89,4 +84,19 @@ package object rasterframes extends StandardColumns
8984
implicit val shortHasCellType = new HasCellType[Short] {}
9085
implicit val floatHasCellType = new HasCellType[Float] {}
9186
}
87+
88+
/** Evidence type class for communicating that only standard key types are supported with the more general GeoTrellis type parameters. */
89+
trait StandardLayerKey[T] extends Serializable {
90+
val selfType: TypeTag[T]
91+
def isType[R: TypeTag]: Boolean = typeOf[R] =:= selfType.tpe
92+
}
93+
object StandardLayerKey {
94+
def apply[T: StandardLayerKey]: StandardLayerKey[T] = implicitly
95+
implicit val spatialKeySupport = new StandardLayerKey[SpatialKey] {
96+
override val selfType: TypeTag[SpatialKey] = implicitly
97+
}
98+
implicit val spatioTemporalKeySupport = new StandardLayerKey[SpaceTimeKey] {
99+
override val selfType: TypeTag[SpaceTimeKey] = implicitly
100+
}
101+
}
92102
}

‎core/src/main/scala/astraea/spark/rasterframes/util/SubdivideSupport.scala‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2018 Astraea. Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
*
19+
*/
20+
21+
package astraea.spark.rasterframes.util
22+
23+
import geotrellis.raster.crop.Crop
24+
import geotrellis.raster.{CellGrid, TileLayout}
25+
import geotrellis.spark.{Bounds, KeyBounds, SpatialComponent, SpatialKey, TileLayerMetadata}
26+
import geotrellis.util._
27+
28+
/**
29+
*
30+
*
31+
* @since 4/5/18
32+
*/
33+
trait SubdivideSupport {
34+
implicit class TileLayoutHasSubdivide(self: TileLayout) {
35+
def subdivide(divs: Int): TileLayout = {
36+
def shrink(num: Int) = {
37+
require(num % divs == 0, s"Subdivision of '$divs' does not evenly divide into dimension '$num'")
38+
num / divs
39+
}
40+
def grow(num: Int) = num * divs
41+
42+
divs match {
43+
case 0 ⇒ self
44+
case i if i < 0 ⇒ throw new IllegalArgumentException(s"divs=$divs must be positive")
45+
case _ ⇒
46+
TileLayout(
47+
layoutCols = grow(self.layoutCols),
48+
layoutRows = grow(self.layoutRows),
49+
tileCols = shrink(self.tileCols),
50+
tileRows = shrink(self.tileRows)
51+
)
52+
}
53+
}
54+
}
55+
56+
implicit class BoundsHasSubdivide[K: SpatialComponent](self: Bounds[K]) {
57+
def subdivide(divs: Int): Bounds[K] = {
58+
self.map(kb ⇒ {
59+
val currGrid = kb.toGridBounds()
60+
// NB: As with GT regrid, we keep the spatial key origin (0, 0) at the same map coordinate
61+
val newGrid = currGrid.copy(
62+
colMin = currGrid.colMin * divs,
63+
rowMin = currGrid.rowMin * divs,
64+
colMax = currGrid.colMin * divs + (currGrid.width - 1) * divs + 1,
65+
rowMax = currGrid.rowMin * divs + (currGrid.height - 1) * divs + 1
66+
)
67+
kb.setSpatialBounds(KeyBounds(newGrid))
68+
})
69+
}
70+
}
71+
72+
/**
73+
* Note: this enrichment makes the assumption that the new keys will be used in the
74+
* layout regime defined by `TileLayoutHasSubdivide`
75+
*/
76+
implicit class SpatialKeyHasSubdivide[K: SpatialComponent](self: K) {
77+
def subdivide(divs: Int): Seq[K] = {
78+
val base = self.getComponent[SpatialKey]
79+
val shifted = SpatialKey(base.col * divs, base.row * divs)
80+
81+
for{
82+
i ← 0 until divs
83+
j ← 0 until divs
84+
} yield {
85+
val newKey = SpatialKey(shifted.col + j, shifted.row + i)
86+
self.setComponent(newKey)
87+
}
88+
}
89+
}
90+
91+
implicit class TileLayerMetadataHasSubdivide[K: SpatialComponent](tlm: TileLayerMetadata[K]) {
92+
def subdivide(divs: Int): TileLayerMetadata[K] = {
93+
val tileLayout = tlm.layout.tileLayout.subdivide(divs)
94+
val layout = tlm.layout.copy(tileLayout = tileLayout)
95+
val bounds = tlm.bounds.subdivide(divs)
96+
tlm.copy(layout = layout, bounds = bounds)
97+
}
98+
}
99+
100+
implicit class TileHasSubdivide[T <: CellGrid: WithCropMethods](self: T) {
101+
def subdivide(divs: Int): Seq[T] = {
102+
val (cols, rows) = self.dimensions
103+
val (newCols, newRows) = (cols/divs, rows/divs)
104+
for {
105+
i ← 0 until divs
106+
j ← 0 until divs
107+
} yield {
108+
val startCol = j * newCols
109+
val startRow = i * newRows
110+
val endCol = startCol + newCols - 1
111+
val endRow = startRow + newRows - 1
112+
self.crop(startCol, startRow, endCol, endRow, Crop.Options(force = true))
113+
}
114+
}
115+
}
116+
}
117+
118+
object SubdivideSupport extends SubdivideSupport

‎core/src/main/scala/astraea/spark/rasterframes/util/debug/package.scala‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2018 Astraea. Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
*
19+
*/
20+
21+
package astraea.spark.rasterframes.util
22+
23+
import java.net.URI
24+
25+
import astraea.spark.rasterframes._
26+
import geotrellis.proj4.WebMercator
27+
import geotrellis.raster.MultibandTile
28+
import geotrellis.raster.io.geotiff.MultibandGeoTiff
29+
import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD, SpatialKey}
30+
import geotrellis.spark.io.slippy.HadoopSlippyTileWriter
31+
import geotrellis.spark.tiling.ZoomedLayoutScheme
32+
import org.apache.spark.annotation.Experimental
33+
34+
/**
35+
* Additional debugging routines. No guarantees these are or will remain stable.
36+
*
37+
* @since 4/6/18
38+
*/
39+
package object debug {
40+
implicit class RasterFrameWithDebug(val self: RasterFrame) {
41+
/** Create a slippy map directory structure export of raster frame, for debugging purposes only. */
42+
@Experimental
43+
def debugTileExport(dest: URI, zoomLevel: Int = 8): Unit = {
44+
val spark = self.sparkSession
45+
implicit val sc = spark.sparkContext
46+
47+
val tgtCrs = WebMercator
48+
49+
val scheme = ZoomedLayoutScheme(tgtCrs)
50+
val mapTransform = scheme
51+
.levelForZoom(zoomLevel)
52+
.layout
53+
.mapTransform
54+
55+
val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString, "tiff")({ (key, tile) =>
56+
val extent = mapTransform(key)
57+
MultibandGeoTiff(tile, extent, tgtCrs).toByteArray
58+
})
59+
60+
/** If there's a temporal component to the key, we drop it blindly. */
61+
val tlrdd: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match {
62+
case Left(spatial) ⇒ spatial
63+
case Right(origRDD) ⇒
64+
val newMD = origRDD.metadata.map(_.spatialKey)
65+
val rdd = origRDD.map { case (k, v) ⇒ (k.spatialKey, v)}
66+
ContextRDD(rdd, newMD)
67+
}
68+
69+
writer.write(zoomLevel, tlrdd)
70+
}
71+
72+
/** Renders the whole schema with metadata as a JSON string. */
73+
def describeFullSchema: String = {
74+
self.schema.prettyJson
75+
}
76+
}
77+
}

‎core/src/main/scala/astraea/spark/rasterframes/util/package.scala‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,18 @@
1919

2020
package astraea.spark.rasterframes
2121

22+
import geotrellis.proj4.WebMercator
23+
import geotrellis.raster.crop.TileCropMethods
24+
import geotrellis.raster.io.geotiff.{MultibandGeoTiff, SinglebandGeoTiff}
25+
import geotrellis.raster.{CellGrid, MultibandTile, Tile, TileLayout}
2226
import geotrellis.raster.mapalgebra.local.LocalTileBinaryOp
23-
import geotrellis.util.LazyLogging
27+
import geotrellis.raster.mask.TileMaskMethods
28+
import geotrellis.raster.merge.TileMergeMethods
29+
import geotrellis.raster.prototype.TilePrototypeMethods
30+
import geotrellis.spark.io.slippy.HadoopSlippyTileWriter
31+
import geotrellis.spark.tiling.{TilerKeyMethods, ZoomedLayoutScheme}
32+
import geotrellis.spark.{Bounds, ContextRDD, KeyBounds, MultibandTileLayerRDD, SpaceTimeKey, SpatialComponent, SpatialKey, TileLayerMetadata, TileLayerRDD}
33+
import geotrellis.util.{GetComponent, LazyLogging}
2434
import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
2535
import org.apache.spark.sql.catalyst.expressions.{Alias, AttributeReference}
2636
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
@@ -29,14 +39,29 @@ import org.apache.spark.sql.rf._
2939
import org.apache.spark.sql.{Column, DataFrame, SQLContext}
3040
import shapeless.Lub
3141

32-
3342
/**
3443
* Internal utilities.
3544
*
3645
* @since 12/18/17
3746
*/
3847
package object util extends LazyLogging {
3948

49+
/**
50+
* Type lambda alias for components that have bounds with parameterized key.
51+
* @tparam K bounds key type
52+
*/
53+
type BoundsComponentOf[K] = {
54+
type Get[M] = GetComponent[M, Bounds[K]]
55+
}
56+
57+
// Type lambda aliases
58+
type WithMergeMethods[V] = (V ⇒ TileMergeMethods[V])
59+
type WithPrototypeMethods[V <: CellGrid] = (V ⇒ TilePrototypeMethods[V])
60+
type WithCropMethods[V <: CellGrid] = (V ⇒ TileCropMethods[V])
61+
type WithMaskMethods[V] = (V ⇒ TileMaskMethods[V])
62+
63+
type KeyMethodsProvider[K1, K2] = K1 ⇒ TilerKeyMethods[K1, K2]
64+
4065
/** Internal method for slapping the RasterFrame seal of approval on a DataFrame. */
4166
private[rasterframes] def certifyRasterframe(df: DataFrame): RasterFrame =
4267
shapeless.tag[RasterFrameTag][DataFrame](df)

0 commit comments

Comments
 (0)