PR feedback.

Author: metasim

core/src/test/scala/org/locationtech/rasterframes/ReprojectGeometrySpec.scala

@@ -33,15 +33,15 @@ import org.locationtech.jts.geom._
 class ReprojectGeometrySpec extends TestEnvironment {
   // Note: Test data copied from ReprojectSpec in GeoTrellis
   val fact = new GeometryFactory()
-  val latLng: Geometry = fact.createLineString(Array(
+  val llLineString: Geometry = fact.createLineString(Array(
     new Coordinate(-111.09374999999999, 34.784483415461345),
     new Coordinate(-111.09374999999999, 43.29919735147067),
     new Coordinate(-75.322265625, 43.29919735147067),
     new Coordinate(-75.322265625, 34.784483415461345),
     new Coordinate(-111.09374999999999, 34.784483415461345)
   ))
 
-  val webMercator: Geometry = fact.createLineString(Array(
+  val wmLineString: Geometry = fact.createLineString(Array(
     new Coordinate(-12366899.680315234, 4134631.734001753),
     new Coordinate(-12366899.680315234, 5357624.186564572),
     new Coordinate(-8384836.254770693, 5357624.186564572),
@@ -54,7 +54,7 @@ class ReprojectGeometrySpec extends TestEnvironment {
 
     it("should handle two literal CRSs") {
 
-      val df = Seq((latLng, webMercator)).toDF("ll", "wm")
+      val df = Seq((llLineString, wmLineString)).toDF("ll", "wm")
 
       val rp = df.select(
         st_reproject($"ll", LatLng, WebMercator) as "wm2",
@@ -65,14 +65,14 @@ class ReprojectGeometrySpec extends TestEnvironment {
 
       val (wm2, ll2, wm3) = rp.first()
 
-      wm2 should matchGeom(webMercator, 0.00001)
-      ll2 should matchGeom(latLng, 0.00001)
-      wm3 should matchGeom(webMercator, 0.00001)
+      wm2 should matchGeom(wmLineString, 0.00001)
+      ll2 should matchGeom(llLineString, 0.00001)
+      wm3 should matchGeom(wmLineString, 0.00001)
     }
 
     it("should handle one literal crs") {
       implicit val enc = Encoders.tuple(jtsGeometryEncoder, jtsGeometryEncoder, crsEncoder)
-      val df = Seq((latLng, webMercator, LatLng: CRS)).toDF("ll", "wm", "llCRS")
+      val df = Seq((llLineString, wmLineString, LatLng: CRS)).toDF("ll", "wm", "llCRS")
 
       val rp = df.select(
         st_reproject($"ll", $"llCRS", WebMercator) as "wm2",
@@ -83,9 +83,9 @@ class ReprojectGeometrySpec extends TestEnvironment {
 
       val (wm2, ll2, wm3) = rp.first()
 
-      wm2 should matchGeom(webMercator, 0.00001)
-      ll2 should matchGeom(latLng, 0.00001)
-      wm3 should matchGeom(webMercator, 0.00001)
+      wm2 should matchGeom(wmLineString, 0.00001)
+      ll2 should matchGeom(llLineString, 0.00001)
+      wm3 should matchGeom(wmLineString, 0.00001)
     }
 
     it("should accept other geometry types") {
@@ -98,7 +98,7 @@ class ReprojectGeometrySpec extends TestEnvironment {
 
     it("should work in SQL") {
       implicit val enc = Encoders.tuple(jtsGeometryEncoder, jtsGeometryEncoder, crsEncoder)
-      val df = Seq((latLng, webMercator, LatLng: CRS)).toDF("ll", "wm", "llCRS")
+      val df = Seq((llLineString, wmLineString, LatLng: CRS)).toDF("ll", "wm", "llCRS")
       df.createOrReplaceTempView("geom")
 
       val rp = spark.sql(
@@ -112,9 +112,9 @@ class ReprojectGeometrySpec extends TestEnvironment {
 
       val (wm2, ll2, wm3) = rp.first()
 
-      wm2 should matchGeom(webMercator, 0.00001)
-      ll2 should matchGeom(latLng, 0.00001)
-      wm3 should matchGeom(webMercator, 0.00001)
+      wm2 should matchGeom(wmLineString, 0.00001)
+      ll2 should matchGeom(llLineString, 0.00001)
+      wm3 should matchGeom(wmLineString, 0.00001)
 
       checkDocs("st_reproject")
     }

datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala

@@ -40,12 +40,10 @@ import org.locationtech.rasterframes.model.{LazyCRS, TileDimensions}
 import org.locationtech.rasterframes.util._
 
 /**
- * Spark SQL data source over GeoTIFF files.
- * @since 1/14/18
+  * Spark SQL data source over GeoTIFF files.
  */
-class GeoTiffDataSource extends DataSourceRegister
-  with RelationProvider with CreatableRelationProvider
-  with DataSourceOptions with LazyLogging {
+class GeoTiffDataSource
+    extends DataSourceRegister with RelationProvider with CreatableRelationProvider with DataSourceOptions with LazyLogging {
   import GeoTiffDataSource._
 
   def shortName() = GeoTiffDataSource.SHORT_NAME
@@ -56,42 +54,40 @@ class GeoTiffDataSource extends DataSourceRegister
 
     val p = parameters.path.get
 
-    if(p.getPath.contains("*")) {
+    if (p.getPath.contains("*")) {
       val bandCount = parameters.get(GeoTiffDataSource.BAND_COUNT_PARAM).map(_.toInt).getOrElse(1)
       GeoTiffCollectionRelation(sqlContext, p, bandCount)
-    }
-    else GeoTiffRelation(sqlContext, p)
+    } else GeoTiffRelation(sqlContext, p)
   }
 
   override def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], df: DataFrame): BaseRelation = {
     require(parameters.path.isDefined, "Valid URI 'path' parameter required.")
     val path = parameters.path.get
-    require(path.getScheme == "file" || path.getScheme == null,
-      "Currently only 'file://' destinations are supported")
+    require(path.getScheme == "file" || path.getScheme == null, "Currently only 'file://' destinations are supported")
     sqlContext.withRasterFrames
 
     val tileCols = df.tileColumns
 
     require(tileCols.nonEmpty, "Could not find any tile columns.")
 
-    val raster = if(df.isAlreadyLayer) {
+    val raster = if (df.isAlreadyLayer) {
       val layer = df.certify
       val tlm = layer.tileLayerMetadata.merge
 
       // If no desired image size is given, write at full size.
-      val TileDimensions(cols, rows) =  parameters.rasterDimensions
+      val TileDimensions(cols, rows) = parameters.rasterDimensions
         .getOrElse {
           val actualSize = tlm.layout.toRasterExtent().gridBoundsFor(tlm.extent)
           TileDimensions(actualSize.width, actualSize.height)
         }
 
       // Should we really play traffic cop here?
-      if(cols.toDouble * rows * 64.0 > Runtime.getRuntime.totalMemory() * 0.5)
-        logger.warn(s"You've asked for the construction of a very large image ($cols x $rows), destined for ${path}. Out of memory error likely.")
+      if (cols.toDouble * rows * 64.0 > Runtime.getRuntime.totalMemory() * 0.5)
+        logger.warn(
+          s"You've asked for the construction of a very large image ($cols x $rows), destined for ${path}. Out of memory error likely.")
 
       layer.toMultibandRaster(tileCols, cols.toInt, rows.toInt)
-    }
-    else {
+    } else {
       require(parameters.crs.nonEmpty, "A destination CRS must be provided")
       require(tileCols.nonEmpty, "need at least one tile column")
 
@@ -102,10 +98,9 @@ class GeoTiffDataSource extends DataSourceRegister
       val (extCol, crsCol, tileCol) = {
         // Favor "ProjectedRaster" columns
         val prCols = df.projRasterColumns
-        if(prCols.nonEmpty) {
+        if (prCols.nonEmpty) {
           (rf_extent(prCols.head), rf_crs(prCols.head), rf_tile(prCols.head))
-        }
-        else {
+        } else {
           // If no "ProjectedRaster" column, look for single Extent and CRS columns.
           val crsCols = df.crsColumns
           require(crsCols.size == 1, "Exactly one CRS column must be in DataFrame")
@@ -117,45 +112,48 @@ class GeoTiffDataSource extends DataSourceRegister
 
       // Scan table and constuct what the TileLayerMetadata would be in the specified destination CRS.
       val tlm: TileLayerMetadata[SpatialKey] = df
-        .select(ProjectedLayerMetadataAggregate(
-          destCRS, extCol, crsCol, rf_cell_type(tileCol), rf_dimensions(tileCol)
-        ))
+        .select(
+          ProjectedLayerMetadataAggregate(
+            destCRS,
+            extCol,
+            crsCol,
+            rf_cell_type(tileCol),
+            rf_dimensions(tileCol)
+          ))
         .first()
 
       val c = ProjectedRasterDefinition(tlm)
 
-      val config = parameters.rasterDimensions.map { dims =>
-        c.copy(totalCols = dims.cols, totalRows = dims.rows)
-      }.getOrElse(c)
+      val config = parameters.rasterDimensions
+        .map { dims =>
+          c.copy(totalCols = dims.cols, totalRows = dims.rows)
+        }
+        .getOrElse(c)
 
       val aggs = tileCols
-        .map(t => TileRasterizerAggregate(
-          config, crsCol, extCol, rf_tile(t))("tile").as(t.columnName)
-        )
+        .map(t => TileRasterizerAggregate(config, crsCol, extCol, rf_tile(t))("tile").as(t.columnName))
 
       val agg = df.select(aggs: _*)
 
       val row = agg.first()
 
-      val bands = for(i <- 0 until row.size) yield row.getAs[Tile](i)
+      val bands = for (i <- 0 until row.size) yield row.getAs[Tile](i)
 
       ProjectedRaster(MultibandTile(bands), tlm.extent, tlm.crs)
     }
 
     val tags = Tags(
       RFBuildInfo.toMap.filter(_._1.startsWith("rf")).mapValues(_.toString),
-      tileCols.map(c ⇒ Map("RF_COL" -> c.columnName)).toList
+      tileCols.map(c => Map("RF_COL" -> c.columnName)).toList
     )
 
     // We make some assumptions here.... eventually have column metadata encode this.
     val colorSpace = tileCols.size match {
-      case 3 | 4 ⇒  ColorSpace.RGB
-      case _ ⇒ ColorSpace.BlackIsZero
+      case 3 | 4 => ColorSpace.RGB
+      case _ => ColorSpace.BlackIsZero
     }
 
-    val tiffOptions = GeoTiffOptions(Tiled,
-      if (parameters.compress) DeflateCompression else NoCompression, colorSpace
-    )
+    val tiffOptions = GeoTiffOptions(Tiled, if (parameters.compress) DeflateCompression else NoCompression, colorSpace)
 
     val geotiff = new MultibandGeoTiff(raster.tile, raster.extent, raster.crs, tags, tiffOptions)
 
@@ -174,19 +172,19 @@ object GeoTiffDataSource {
   final val CRS_PARAM = "crs"
   final val BAND_COUNT_PARAM = "bandCount"
 
-  private[geotiff]
-  implicit class ParamsDictAccessors(val parameters: Map[String, String]) extends AnyVal {
+  private[geotiff] implicit class ParamsDictAccessors(val parameters: Map[String, String]) extends AnyVal {
     def path: Option[URI] = uriParam(PATH_PARAM, parameters)
     def compress: Boolean = parameters.get(COMPRESS_PARAM).exists(_.toBoolean)
     def crs: Option[CRS] = parameters.get(CRS_PARAM).map(s => LazyCRS(s))
     def rasterDimensions: Option[TileDimensions] = {
       numParam(IMAGE_WIDTH_PARAM, parameters)
         .zip(numParam(IMAGE_HEIGHT_PARAM, parameters))
-        .map { case (cols, rows) =>
-          require(cols <= Int.MaxValue && rows <= Int.MaxValue,
-            s"Can't construct a GeoTIFF of size $cols x $rows. (Too big!)")
-          TileDimensions(cols.toInt, rows.toInt)
-        }.headOption
+        .map {
+          case (cols, rows) =>
+            require(cols <= Int.MaxValue && rows <= Int.MaxValue, s"Can't construct a GeoTIFF of size $cols x $rows. (Too big!)")
+            TileDimensions(cols.toInt, rows.toInt)
+        }
+        .headOption
     }
   }
 }

docs/README.md

@@ -67,11 +67,11 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     sbt:RasterFrames> pyrasterframes/doc
     ```
     There's a command alias for this last step: `pyDocs`.
-4. To evaluate a single `.pymd` file, you pass the `-f` option and the filename relative to the `python` directory:
+4. To evaluate a single `.pymd` file, you pass the `-s` option and the filename relative to the `python` directory:
     ```
-    sbt:RasterFrames> pyrasterframes/pySetup pweave -f docs/getting-started.pymd
+    sbt:RasterFrames> pyrasterframes/pySetup pweave -s docs/getting-started.pymd
     [info] Synchronizing 44 files to '<src-root>/pyrasterframes/target/python'
-    [info] Running 'python setup.py pweave -f docs/getting-started.pymd' in '<src-root>/pyrasterframes/target/python'
+    [info] Running 'python setup.py pweave -s docs/getting-started.pymd' in '<src-root>/pyrasterframes/target/python'
     running pweave
     --------------------------------------------------
     Running getting-started
@@ -85,9 +85,9 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
 5. The _output_ Markdown files are written to `<src-root>/pyrasterframes/target/python/docs`. _Note_: don't edit any files in the `pyrasterframes/target` directory... they will get overwritten each time `sbt` runs a command.
 6. During content development it's sometimes helpful to see the output rendered as basic HTML. To do this, add the `-d html` option to the pweave command:
     ```
-    sbt:RasterFrames> pyrasterframes/pySetup pweave -d html -f docs/getting-started.pymd
+    sbt:RasterFrames> pyrasterframes/pySetup pweave -f html -s docs/getting-started.pymd
     [info] Synchronizing 54 files to '<src-roog>/pyrasterframes/target/python'
-    [info] Running 'python setup.py pweave -d html -f docs/getting-started.pymd' in '<src-root>/pyrasterframes/target/python'
+    [info] Running 'python setup.py pweave -f html -s docs/getting-started.pymd' in '<src-root>/pyrasterframes/target/python'
     running pweave
     --------------------------------------------------
     Running getting-started

docs/src/main/paradox/release-notes.md

@@ -4,6 +4,7 @@
 
 ### 0.8.0
 
+* Super-duper new Python-centric [RasterFrames Users' Manual](https://rasterframes.io/)!
 * Upgraded to the following core dependencies: Spark 2.3.3, GeoTrellis 2.3.0, GeoMesa 2.2.1, JTS 1.16.0.
 * Build `pyrasterframes` binary distribution for pip installation.
 * Added support for rendering RasterFrame types in IPython/Jupyter.

pyrasterframes/src/main/python/docs/concepts.md

@@ -1,6 +1,6 @@
 # Concepts
 
-There are a number of EO concepts that crop up in the discussion of RasterFrames features. We'll cover these briefly in the sections below. However, here are a few links providing a more extensive introduction to working with Earth observation data. 
+There are a number of Earth-observation (EO) concepts that crop up in the discussion of RasterFrames features. We'll cover these briefly in the sections below. However, here are a few links providing a more extensive introduction to working with Earth observation data. 
 
 * [_Fundamentals of Remote Sensing_](https://www.nrcan.gc.ca/maps-tools-and-publications/satellite-imagery-and-air-photos/tutorial-fundamentals-remote-sensing/9309)
 * [_Newcomers Earth Observation Guide_](https://business.esa.int/newcomers-earth-observation-guide)
@@ -51,7 +51,13 @@ An extent (or bounding box) is a rectangular region specifying the geospatial co
 
 ## Tile
 
-A tile (sometimes called a "chip") is a sub-region of  a scene, usually square, usually with dimensions that are some power of 2. It is usually encoded as a two-dimensional array, or a a one-dimensional array with width/height dimensions. Tiles may be singleband or multiband (multi-channel). The tile is the primary discretization unit used in RasterFrames; the bands of a scene are separated into different columns, and the band's raster is carved up into tiles for each row.
+A tile (sometimes called a "chip") is a rectangular subset of a @ref:[scene](concepts.md#scene). A tile can conceptually be though of as a two-dimensional array.
+
+Some EO data has many bands or channels. Tiles in this context are conceptually a three-dimensional array, with the extra dimension representing the bands.
+
+Tiles are often square and the dimensions are some power of two, for example 256 by 256. 
+
+The tile is the primary discretization unit used in RasterFrames. Each band of a scene is in a separate column. The scene's overall @ref:[extent](concepts.md#extent) is carved up into smaller extents for each tile. Each row of the DataFrame contains a two-dimensional tile per band column.
 
 ## Projected Extent