Merge branch 'develop' into feature/local_is_in

Author: metasim

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/TileRasterizerAggregate.scala

@@ -138,7 +138,7 @@ object TileRasterizerAggregate {
       }
     }
 
-    // Scan table and constuct what the TileLayerMetadata would be in the specified destination CRS.
+    // Scan table and construct what the TileLayerMetadata would be in the specified destination CRS.
     val tlm: TileLayerMetadata[SpatialKey] = df
       .select(
         ProjectedLayerMetadataAggregate(

core/src/main/scala/org/locationtech/rasterframes/util/DataFrameRenderers.scala

@@ -24,12 +24,14 @@ package org.locationtech.rasterframes.util
 import geotrellis.raster.render.ColorRamps
 import org.apache.spark.sql.Dataset
 import org.apache.spark.sql.functions.{base64, concat, concat_ws, length, lit, substring, when}
+import org.apache.spark.sql.jts.JTSTypes
 import org.apache.spark.sql.types.{StringType, StructField}
 import org.locationtech.rasterframes.expressions.DynamicExtractors
 import org.locationtech.rasterframes.{rfConfig, rf_render_png, rf_resample}
+import org.apache.spark.sql.rf.WithTypeConformity
 
 /**
-  * DataFrame extensiosn for rendering sample content in a number of ways
+  * DataFrame extension for rendering sample content in a number of ways
   */
 trait DataFrameRenderers {
   private val truncateWidth = rfConfig.getInt("max-truncate-row-element-length")
@@ -47,8 +49,9 @@ trait DataFrameRenderers {
               lit("\"></img>")
             )
           else {
+            val isGeom = WithTypeConformity(c.dataType).conformsTo(JTSTypes.GeometryTypeInstance)
             val str = resolved.cast(StringType)
-            if (truncate)
+            if (truncate || isGeom)
               when(length(str) > lit(truncateWidth),
                 concat(substring(str, 1, truncateWidth), lit("..."))
               )

core/src/test/resources/MCD43A4.A2019111.h30v06.006.2019120033434_01.mrf.aux.xml

@@ -39,7 +39,7 @@ import scala.xml.parsing.XhtmlParser
 class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSupport {
   lazy val rf = sampleTileLayerRDD.toLayer
 
-  describe("DataFrame exention methods") {
+  describe("DataFrame extension methods") {
     it("should maintain original type") {
       val df = rf.withPrefixedColumnNames("_foo_")
       "val rf2: RasterFrameLayer = df" should compile
@@ -49,7 +49,7 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
       "val Some(col) = df.spatialKeyColumn" should compile
     }
   }
-  describe("RasterFrameLayer exention methods") {
+  describe("RasterFrameLayer extension methods") {
     it("should provide spatial key column") {
       noException should be thrownBy {
         rf.spatialKeyColumn
@@ -124,6 +124,10 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
 
       val md3 = rf.toMarkdown(truncate=true, renderTiles = false)
       md3 shouldNot include("<img")
+
+      // Should truncate JTS types even when we don't ask for it.
+      val md4 = rf.withGeometry().select("geometry").toMarkdown(truncate = false)
+      md4 should include ("...")
     }
 
     it("should render HTML") {

core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala

@@ -49,6 +49,7 @@ class GeoTiffDataSource
 
   def shortName() = GeoTiffDataSource.SHORT_NAME
 
+  /** Read single geotiff as a relation. */
   def createRelation(sqlContext: SQLContext, parameters: Map[String, String]) = {
     require(parameters.path.isDefined, "Valid URI 'path' parameter required.")
     sqlContext.withRasterFrames
@@ -57,6 +58,7 @@ class GeoTiffDataSource
     GeoTiffRelation(sqlContext, p)
   }
 
+  /** Write dataframe containing bands into a single geotiff. Note: performs a driver collect, and is not "big data" friendly.  */
   override def createRelation(sqlContext: SQLContext, mode: SaveMode, parameters: Map[String, String], df: DataFrame): BaseRelation = {
     require(parameters.path.isDefined, "Valid URI 'path' parameter required.")
     val path = parameters.path.get
@@ -67,8 +69,6 @@ class GeoTiffDataSource
 
     require(tileCols.nonEmpty, "Could not find any tile columns.")
 
-
-
     val destCRS = parameters.crs.orElse(df.asLayerSafely.map(_.crs)).getOrElse(
       throw new IllegalArgumentException("A destination CRS must be provided")
     )

datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala

@@ -37,7 +37,7 @@ class RasterSourceDataSource extends DataSourceRegister with RelationProvider {
   override def shortName(): String = SHORT_NAME
   override def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = {
     val bands = parameters.bandIndexes
-    val tiling = parameters.tileDims
+    val tiling = parameters.tileDims.orElse(Some(NOMINAL_TILE_DIMS))
     val lazyTiles = parameters.lazyTiles
     val spec = parameters.pathSpec
     val catRef = spec.fold(_.registerAsTable(sqlContext), identity)

datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceDataSource.scala

@@ -192,29 +192,36 @@ class GeoTiffDataSourceSpec
     }
 
     it("should write GeoTIFF without layer") {
-
       val pr = col("proj_raster_b0")
-      val rf = spark.read.raster.withBandIndexes(0, 1, 2).load(rgbCogSamplePath.toASCIIString)
 
-      val out = Paths.get("target", "example2-geotiff.tif")
-      logger.info(s"Writing to $out")
+      val sample = rgbCogSample
+      val expectedExtent = sample.extent
+      val (expCols, expRows) = sample.tile.dimensions
 
-      withClue("explicit extent/crs") {
+      val rf = spark.read.raster.withBandIndexes(0, 1, 2).load(rgbCogSamplePath.toASCIIString)
+
+      withClue("extent/crs columns provided") {
+        val out = Paths.get("target", "example2a-geotiff.tif")
         noException shouldBe thrownBy {
           rf
             .withColumn("extent", rf_extent(pr))
             .withColumn("crs", rf_crs(pr))
-            .write.geotiff.withCRS(LatLng).save(out.toString)
+            .write.geotiff.withCRS(sample.crs).save(out.toString)
+          checkTiff(out, expCols, expRows, expectedExtent, Some(sample.cellType))
         }
       }
 
-      withClue("without explicit extent/crs") {
+      withClue("without extent/crs columns") {
+        val out = Paths.get("target", "example2b-geotiff.tif")
         noException shouldBe thrownBy {
           rf
-            .write.geotiff.withCRS(LatLng).save(out.toString)
+            .write.geotiff.withCRS(sample.crs).save(out.toString)
+          checkTiff(out, expCols, expRows, expectedExtent, Some(sample.cellType))
         }
       }
+
       withClue("with downsampling") {
+        val out = Paths.get("target", "example2c-geotiff.tif")
         noException shouldBe thrownBy {
           rf
             .write.geotiff
@@ -223,9 +230,6 @@ class GeoTiffDataSourceSpec
             .save(out.toString)
         }
       }
-
-      checkTiff(out, 128, 128,
-        Extent(-76.52586750038186, 36.85907177863949, -76.17461216980891, 37.1303690755922))
     }
 
     it("should produce the correct subregion from layer") {

datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala

@@ -4,9 +4,13 @@
 
 ### 0.8.4
 
-* _Breaking_ (potentially): removed `GeoTiffCollectionRelation` due to usage limitation and overlap with `RasterSourceDataSource` functionality.
 * Upgraded to Spark 2.4.4
- * Add `rf_mask_by_values` and `rf_local_is_in` raster functions; added optional `inverse` argument to `rf_mask` functions
+* Add `rf_mask_by_values` and `rf_local_is_in` raster functions; added optional `inverse` argument to `rf_mask` functions. ([#403](https://github.com/locationtech/rasterframes/pull/403), [#384](https://github.com/locationtech/rasterframes/issues/384))
+* Added forced truncation of WKT types in Markdown/HTML rendering. ([#408](https://github.com/locationtech/rasterframes/pull/408))
+* Add `rf_local_is_in` raster function. ([#400](https://github.com/locationtech/rasterframes/pull/400))
+* Added partitioning to catalogs before processing in RasterSourceDataSource ([#397](https://github.com/locationtech/rasterframes/pull/397))
+* Fixed bug where `rf_tile_dimensions` would cause unnecessary reading of tiles. ([#394](https://github.com/locationtech/rasterframes/pull/394))  
+* _Breaking_ (potentially): removed `GeoTiffCollectionRelation` due to usage limitation and overlap with `RasterSourceDataSource` functionality.
 
 ### 0.8.3
 

docs/src/main/paradox/release-notes.md

@@ -2,15 +2,21 @@
 
 RasterFrames® brings together Earth-observation (EO) data access, cloud computing, and DataFrame-based data science. The recent explosion of EO data from public and private satellite operators presents both a huge opportunity and a huge challenge to the data analysis community. It is _Big Data_ in the truest sense, and its footprint is rapidly getting bigger. 
 
-RasterFrames provides a DataFrame-centric view over arbitrary raster data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of Spark ML algorithms. By using DataFrames as the core cognitive and compute data model, it is able to deliver these features in a form that is both accessible to general analysts and scalable along with the rapidly growing data footprint.
+RasterFrames provides a DataFrame-centric view over arbitrary geospatial raster data, enabling spatiotemporal queries, map algebra raster operations, and interoperability with Spark ML. By using the DataFrame as the core cognitive and compute data model, RasterFrames is able to deliver an extensive set of functionality in a form that is both horizontally scalable as well as familiar to general analysts and data scientists. It provides APIs for Python, SQL, and Scala.
 
-To learn more, please see the @ref:[Getting Started](getting-started.md) section of this manual.
+![RasterFrames](static/rasterframes-pipeline-nologo.png)
 
-The source code can be found on GitHub at [locationtech/rasterframes](https://github.com/locationtech/rasterframes).
+Through its custom [Spark DataSource](https://rasterframes.io/raster-read.html), RasterFrames can read various raster formats -- including GeoTIFF, JP2000, MRF, and HDF -- and from an [array of services](https://rasterframes.io/raster-read.html#uri-formats), such as HTTP, FTP, HDFS, S3 and WASB. It also supports reading the vector formats GeoJSON and WKT/WKB. RasterFrame contents can be filtered, transformed, summarized, resampled, and rasterized through [200+ raster and vector functions](https://rasterframes.io/reference.html).
+
+As part of the LocationTech family of projects, RasterFrames builds upon the strong foundations provided by GeoMesa (spatial operations) , GeoTrellis (raster operations), JTS (geometry modeling) and SFCurve (spatiotemporal indexing), integrating various aspects of these projects into a unified, DataFrame-centric analytics package.
+
+![](static/rasterframes-locationtech-stack.png)
 
-RasterFrames is released under the [Apache 2.0 License](https://github.com/locationtech/rasterframes/blob/develop/LICENSE).
+RasterFrames is released under the commercial-friendly [Apache 2.0](https://github.com/locationtech/rasterframes/blob/develop/LICENSE) open source license.
 
-![RasterFrames](static/rasterframes-pipeline.png)
+To learn more, please see the @ref:[Getting Started](getting-started.md) section of this manual.
+
+The source code can be found on GitHub at [locationtech/rasterframes](https://github.com/locationtech/rasterframes).
 
 <hr/>