Skip to content

Commit 00319fc

Browse files
metasimmetasim
committed
Added auto-generated HTML file for slippy map tile export.
Signed-off-by: Simeon H.K. Fitch <[email protected]>
1 parent 000222b commit 00319fc

File tree

4 files changed

+62
-17
lines changed

4 files changed

+62
-17
lines changed

‎docs/src/main/tut/release-notes.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@
88
* Added for for writing GeoTIFFs from RasterFrames via with DataFrameWriter.
99
* Added `spark.read.geotrellis.withNumPartitions(Int)` for setting the initial number of partitions to use when reading a layer.
1010
* Added `spark.read.geotrellis.withTileSubdivisions(Int)` for evenly subdividing tiles before they become rows in a RasterFrame.
11-
* Added `astraea.spark.rasterframes.util.debug` hosting debug feature to export RasterFrame as a slippy-tile directory structure.
12-
_Experimental and not for long-term use_.
11+
* Added `experimental` package for sandboxing new feature ideas.
12+
* Added `SlippyExport` experimental feature for exporting the contents of a RasterFrame as a [SlippyMap](https://wiki.openstreetmap.org/wiki/Slippy_Map)
13+
tile image directory structure and Leaflet-enabled HTML file.
1314
* _Change_: Default interpoation for `toRaster` and `toMultibandRaster` has been changed from `Bilinear` to `NearestNeighbor`.
1415
* _Breaking_: Renamed/moved `astraea.spark.rasterframes.functions.CellStatsAggregateFunction.Statistics` to
1516
`astraea.spark.rasterframes.stats.CellStatistics`.

‎experimental/src/main/resources/slippy.html‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<html lang="en">
2323
<head>
2424
<meta charset="UTF-8">
25-
<title>RasterFrames Debuging View</title>
25+
<title>RasterFrames</title>
2626
<meta charset="utf-8" />
2727
<meta name="viewport" content="width=device-width, initial-scale=1.0">
2828
<link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css" integrity="sha512-Rksm5RenBEKSKFjgI3a41vrjkw4EVPlJ3+OiI65vTjIdo9brlAacEuKOiQ5OFh7cOI1bkDwLqdLw3Zg0cRJAAQ==" crossorigin=""/>
@@ -45,17 +45,26 @@
4545

4646
<script>
4747

48-
var map = L.map('mapid').setView([38.174, -78.588], 6);
48+
var map = L.map('mapid')
49+
.setView([${viewLat}, ${viewLon}], ${maxZoom});
4950

50-
L.tileLayer('https://api.tiles.mapbox.com/v4/{id}/{z}/{x}/{y}.png?access_token=pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw', {
51+
L.tileLayer(
52+
'https://api.tiles.mapbox.com/v4/{id}/{z}/{x}/{y}.png?access_token=pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw', {
5153
maxZoom: 18,
5254
attribution: 'Map data &copy; <a href="http://openstreetmap.org">OpenStreetMap</a> contributors, ' +
5355
'<a href="http://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>, ' +
5456
'Imagery © <a href="http://mapbox.com">Mapbox</a>',
5557
id: 'mapbox.streets'
5658
}).addTo(map);
5759

58-
L.control.scale().addTo(map)
60+
L.tileLayer(
61+
'{id}/{z}/{x}/{y}.png', {
62+
maxZoom: ${maxZoom},
63+
id: '${id}'
64+
}
65+
).addTo(map);
66+
67+
L.control.scale().addTo(map);
5968

6069
L.Control.geocoder().addTo(map);
6170

‎experimental/src/main/scala/astraea/spark/rasterframes/experimental/SlippyExport.scala‎

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,22 +20,27 @@
2020

2121
package astraea.spark.rasterframes.experimental
2222

23-
import geotrellis.util.MethodExtensions
2423
import java.net.URI
2524

2625
import astraea.spark.rasterframes._
2726
import astraea.spark.rasterframes.util._
28-
import geotrellis.proj4.WebMercator
29-
import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
27+
import geotrellis.proj4.{LatLng, WebMercator}
28+
import geotrellis.raster._
3029
import geotrellis.raster.io.geotiff._
30+
import geotrellis.raster.io.geotiff.tags.codes.ColorSpace
31+
import geotrellis.raster.render.{ColorMap, ColorRamps}
3132
import geotrellis.raster.resample.Bilinear
32-
import geotrellis.raster._
33-
import geotrellis.raster.render.{ColorMap, ColorMaps, ColorRamps}
3433
import geotrellis.spark._
3534
import geotrellis.spark.io.slippy.HadoopSlippyTileWriter
35+
import geotrellis.spark.pyramid.Pyramid
3636
import geotrellis.spark.tiling.ZoomedLayoutScheme
37+
import geotrellis.util.MethodExtensions
38+
import org.apache.commons.lang3.text.StrSubstitutor
39+
import org.apache.hadoop.conf.Configuration
40+
import org.apache.hadoop.fs.{FileSystem, Path}
3741
import org.apache.spark.annotation.Experimental
3842

43+
import scala.io.Source
3944
import scala.util.Try
4045

4146
/**
@@ -82,12 +87,17 @@ trait SlippyExport extends MethodExtensions[RasterFrame]{
8287
/**
8388
* Export tiles as a slippy map. For debugging purposes only.
8489
* NB: Temporal components are ignored blindly.
90+
* @param dest URI for Hadoop supported storage endpoint (e.g. 'file://', 'hdfs://', etc.).
91+
* @param colorMap Optional color map to use for rendering tiles in non-RGB RasterFrames.
8592
*/
8693
@Experimental
8794
def exportSlippyMap(dest: URI, colorMap: Option[ColorMap] = None): Unit = {
8895
val spark = self.sparkSession
8996
implicit val sc = spark.sparkContext
9097

98+
val tileDirName = "rf-tiles"
99+
100+
91101
val inputRDD: MultibandTileLayerRDD[SpatialKey] = self.toMultibandTileLayerRDD(self.tileColumns: _*) match {
92102
case Left(spatial) ⇒ spatial
93103
case Right(origRDD) ⇒
@@ -99,8 +109,7 @@ trait SlippyExport extends MethodExtensions[RasterFrame]{
99109
val layoutScheme = ZoomedLayoutScheme(WebMercator, tileSize = 256)
100110

101111
val (zoom, reprojected) = inputRDD.reproject(WebMercator, layoutScheme, Bilinear)
102-
103-
val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString, "png")({ (_, tile) =>
112+
val writer = new HadoopSlippyTileWriter[MultibandTile](dest.toASCIIString + "/" + tileDirName, "png")({ (_, tile) =>
104113
val png = if(colorMap.isEmpty && tile.bands.lengthCompare(3) == 0) {
105114
// `Try` below is due to https://github.com/locationtech/geotrellis/issues/2621
106115
tile.mapBands((_, t) ⇒ Try(t.rescale(0, 255)).getOrElse(t)).renderPng()
@@ -113,11 +122,37 @@ trait SlippyExport extends MethodExtensions[RasterFrame]{
113122
png.bytes
114123
})
115124

116-
writer.write(zoom, reprojected)
125+
val center = reprojected.metadata.extent.center.reproject(WebMercator, LatLng)
126+
127+
SlippyExport.writeHtml(dest, sc.hadoopConfiguration, Map(
128+
"maxZoom" -> zoom.toString,
129+
"id" -> tileDirName,
130+
"viewLat" -> center.y.toString,
131+
"viewLon" -> center.x.toString
132+
))
133+
134+
// Pyramiding up the zoom levels, write our tiles out to the local file system.
135+
Pyramid.upLevels(reprojected, layoutScheme, zoom, Bilinear) { (rdd, z) =>
136+
writer.write(z, rdd)
137+
}
117138
}
118139
}
119140

120141
object SlippyExport {
142+
import scala.collection.JavaConverters._
121143
implicit class RasterFrameHasSlippy(val self: RasterFrame) extends SlippyExport
144+
private def writeHtml(dest: URI, conf: Configuration, subs: Map[String, String]): Unit = {
145+
val rawLines = Source.fromInputStream(getClass.getResourceAsStream("/slippy.html")).getLines()
146+
147+
val subst = new StrSubstitutor(subs.asJava)
148+
149+
val fs = FileSystem.get(dest, conf)
150+
withResource(fs.create(new Path(new Path(dest), "index.html"), true)) { out ⇒
151+
for(line ← rawLines) {
152+
out.writeBytes(subst.replace(line))
153+
out.writeChar('\n')
154+
}
155+
}
156+
}
122157
}
123158

experimental/src/test/scala/astraea/spark/rasterframes/experimental/SlippyExportTest.scala renamed to experimental/src/test/scala/astraea/spark/rasterframes/experimental/SlippyExportDriver.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ import geotrellis.raster._
2828
import geotrellis.raster.io.geotiff.SinglebandGeoTiff
2929
import org.apache.spark.sql.SparkSession
3030

31-
object SlippyExportTest {
31+
object SlippyExportDriver {
3232
def main(args: Array[String]): Unit = {
3333

3434
implicit val spark = SparkSession
@@ -51,8 +51,8 @@ object SlippyExportTest {
5151

5252
val rf = pr.toRF(64, 64)
5353

54-
rf.exportGeoTiffTiles(new File("target/slippy-tiff").toURI)
54+
//rf.exportGeoTiffTiles(new File("target/slippy-tiff").toURI)
5555

56-
rf.exportSlippyMap(new File("target/slippy-png").toURI)
56+
rf.exportSlippyMap(new File("target/slippy-png/").toURI)
5757
}
5858
}

0 commit comments

Comments
 (0)