Merge remote-tracking branch 'lt/develop' into feature/unit-test-163

Author: vpipkt

README.md

@@ -1,10 +1,10 @@
-<img src="docs/src/main/paradox/_template/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&trade;</sup>
+<img src="docs/src/main/paradox/_template/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&reg;</sup>
 
  [![Join the chat at https://gitter.im/s22s/raster-frames](https://badges.gitter.im/s22s/raster-frames.svg)](https://gitter.im/s22s/raster-frames?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 _RasterFrames™_ brings the power of Spark DataFrames to geospatial raster data, empowered by the map algebra and tile layer operations of [GeoTrellis](https://geotrellis.io/).
 
-<img src="docs/src/main/paradox/RasterFramePipeline.svg" width="600px"/>
+<img src="docs/src/main/paradox/RasterFramePipeline.png" width="600px"/>
 
 Please see the [Getting Started](http://rasterframes.io/getting-started.html) section of the Users' Manual to start using RasterFrames.
 
@@ -54,6 +54,6 @@ The `pyrasterframes` build instructions are located at [pyrasterframes/src/main/
 
 ## Copyright and License
 
-RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2018.
+RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2019.
 
 

core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala

@@ -63,8 +63,8 @@ class RasterSourceIT extends TestEnvironment with TestData {
     }
   }
 
-  if (RasterSource.IsGDAL.hasGDAL) {
-    println("GDAL version: " + GDALWarp.get_version_info("--version"))
+  if (GDALRasterSource.hasGDAL) {
+    println("GDAL version: " + GDALRasterSource.gdalVersion())
 
     describe("GDAL support") {
 

core/src/main/scala/org/locationtech/rasterframes/expressions/aggregates/ProjectedLayerMetadataAggregate.scala

@@ -28,7 +28,7 @@ import org.locationtech.rasterframes.model.TileDimensions
 import geotrellis.proj4.{CRS, Transform}
 import geotrellis.raster._
 import geotrellis.raster.reproject.{Reproject, ReprojectRasterExtent}
-import geotrellis.spark.tiling.{FloatingLayoutScheme, LayoutLevel}
+import geotrellis.spark.tiling.LayoutDefinition
 import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata}
 import geotrellis.vector.Extent
 import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
@@ -76,7 +76,10 @@ class ProjectedLayerMetadataAggregate(destCRS: CRS, destDims: TileDimensions) ex
   override def evaluate(buffer: Row): Any = {
     import org.locationtech.rasterframes.encoders.CatalystSerializer._
     val buf = buffer.to[BufferRecord]
-    val LayoutLevel(_, layout) = FloatingLayoutScheme(destDims.cols, destDims.rows).levelFor(buf.extent, buf.cellSize)
+
+    val re = RasterExtent(buf.extent, buf.cellSize)
+    val layout = LayoutDefinition(re, destDims.cols, destDims.rows)
+
     val kb = KeyBounds(layout.mapTransform(buf.extent))
     TileLayerMetadata(buf.cellType, layout, buf.extent, destCRS, kb).toRow
   }

core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala

@@ -23,12 +23,14 @@ package org.locationtech.rasterframes.ref
 
 import java.net.URI
 
+import com.azavea.gdal.GDALWarp
+import com.typesafe.scalalogging.LazyLogging
+import geotrellis.contrib.vlm.gdal.{GDALRasterSource => VLMRasterSource}
 import geotrellis.proj4.CRS
-import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster}
 import geotrellis.raster.io.geotiff.Tags
+import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster}
 import geotrellis.vector.Extent
 import org.locationtech.rasterframes.ref.RasterSource.URIRasterSource
-import geotrellis.contrib.vlm.gdal.{GDALRasterSource => VLMRasterSource}
 
 case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSource {
 
@@ -67,3 +69,17 @@ case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSour
   override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
     gdal.readBounds(bounds, bands)
 }
+
+object GDALRasterSource extends LazyLogging {
+  def gdalVersion(): String = if (hasGDAL) GDALWarp.get_version_info("--version").trim else "not available"
+
+  @transient
+  lazy val hasGDAL: Boolean = try {
+    val _ = new GDALWarp()
+    true
+  } catch {
+    case _: UnsatisfiedLinkError =>
+      logger.warn("GDAL native bindings are not available. Falling back to JVM-based reader for GeoTIFF format.")
+      false
+  }
+}

core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala

@@ -23,7 +23,6 @@ package org.locationtech.rasterframes.ref
 
 import java.net.URI
 
-import com.azavea.gdal.GDALWarp
 import com.github.blemale.scaffeine.Scaffeine
 import com.typesafe.scalalogging.LazyLogging
 import geotrellis.proj4.CRS
@@ -120,27 +119,18 @@ object RasterSource extends LazyLogging {
 
     /** Determine if we should prefer GDAL for all types. */
     private val preferGdal: Boolean = org.locationtech.rasterframes.rfConfig.getBoolean("prefer-gdal")
-    @transient
-    lazy val hasGDAL: Boolean = try {
-      val _ = new GDALWarp()
-      true
-    } catch {
-      case _: UnsatisfiedLinkError =>
-        logger.warn("GDAL native bindings are not available. Falling back to JVM-based reader.")
-        false
-    }
 
-    val gdalOnlyExtensions = Seq(".jp2", ".mrf", ".hdf")
+    val gdalOnlyExtensions = Seq(".jp2", ".mrf", ".hdf", ".vrt")
 
     def gdalOnly(source: URI): Boolean =
       if (gdalOnlyExtensions.exists(source.getPath.toLowerCase.endsWith)) {
-        require(hasGDAL, s"Can only read $source if GDAL is available")
+        require(GDALRasterSource.hasGDAL, s"Can only read $source if GDAL is available")
         true
       } else false
 
     /** Extractor for determining if a scheme indicates GDAL preference.  */
     def unapply(source: URI): Boolean =
-      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && hasGDAL)
+      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && GDALRasterSource.hasGDAL)
   }
 
   object IsDefaultGeoTiff {

core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala

@@ -25,7 +25,7 @@ import geotrellis.proj4.LatLng
 import geotrellis.raster.{ByteCellType, GridBounds, TileLayout}
 import geotrellis.spark.tiling.{CRSWorldExtent, LayoutDefinition}
 import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata}
-import org.apache.spark.sql.{Encoder, Encoders}
+import org.apache.spark.sql.Encoders
 import org.locationtech.rasterframes.util.SubdivideSupport
 
 /**

core/src/test/scala/org/locationtech/rasterframes/TestData.scala

@@ -146,6 +146,7 @@ trait TestData {
 
   lazy val localSentinel: URI = getClass.getResource("/B01.jp2").toURI
   lazy val cogPath: URI = getClass.getResource("/LC08_RGB_Norfolk_COG.tiff").toURI
+  lazy val singlebandCogPath: URI = getClass.getResource("/LC08_B7_Memphis_COG.tiff").toURI
   lazy val nonCogPath: URI = getClass.getResource("/L8-B8-Robinson-IL.tiff").toURI
 
   lazy val l8B1SamplePath: URI = l8SamplePath(1)

core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala

@@ -29,11 +29,7 @@ import org.apache.spark.sql.rf.RasterSourceUDT
 import org.locationtech.rasterframes.TestEnvironment
 import org.locationtech.rasterframes.model.TileDimensions
 
-/**
- *
- *
- * @since 8/22/18
- */
+
 class RasterSourceSpec extends TestEnvironment with TestData {
   def sub(e: Extent) = {
     val c = e.center
@@ -112,7 +108,7 @@ class RasterSourceSpec extends TestEnvironment with TestData {
     }
   }
 
-  if(RasterSource.IsGDAL.hasGDAL) {
+  if(GDALRasterSource.hasGDAL) {
     describe("GDAL Rastersource") {
       val gdal = GDALRasterSource(cogPath)
       val jvm = JVMGeoTiffRasterSource(cogPath)

datasource/src/main/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSource.scala

@@ -121,6 +121,7 @@ class GeoTiffDataSource
             rf_dimensions(tileCol)
           ))
         .first()
+      logger.debug(s"Contructed TileLayerMetadata: ${tlm.toString}")
 
       val c = ProjectedRasterDefinition(tlm)
 

datasource/src/test/scala/org/locationtech/rasterframes/datasource/geotiff/GeoTiffDataSourceSpec.scala

@@ -23,6 +23,9 @@ package org.locationtech.rasterframes.datasource.geotiff
 import java.nio.file.Paths
 
 import geotrellis.proj4._
+import geotrellis.raster.ProjectedRaster
+import geotrellis.raster.io.geotiff.{MultibandGeoTiff, SinglebandGeoTiff}
+import geotrellis.vector.Extent
 import org.locationtech.rasterframes._
 import org.apache.spark.sql.functions._
 import org.locationtech.rasterframes.TestEnvironment
@@ -41,7 +44,7 @@ class GeoTiffDataSourceSpec
       assert(rf.count() > 10)
     }
 
-    it("should lay out tiles correctly"){
+    it("should lay out tiles correctly") {
 
       val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer
 
@@ -83,6 +86,9 @@ class GeoTiffDataSourceSpec
 
       assert(result === expected)
     }
+  }
+
+  describe("GeoTiff writing") {
 
     it("should write GeoTIFF RF to parquet") {
       val rf = spark.read.format("geotiff").load(cogPath.toASCIIString).asLayer
@@ -101,6 +107,81 @@ class GeoTiffDataSourceSpec
       }
     }
 
+    it("should write unstructured raster") {
+      import spark.implicits._
+      val df = spark.read.format("raster")
+        .option("tileDimensions", "32,32")  // oddball
+        .load(nonCogPath.toASCIIString) // core L8-B8-Robinson-IL.tiff
+
+      df.count() should be > 0L
+
+      val crs = df.select(rf_crs($"proj_raster")).first()
+
+      noException shouldBe thrownBy {
+        df.write.geotiff.withCRS(crs).save("unstructured.tif")
+      }
+
+      val (inCols, inRows) = {
+        val id = sampleGeoTiff.imageData // inshallah same as nonCogPath
+        (id.cols, id.rows)
+      }
+      inCols should be (774)
+      inRows should be (500) //from gdalinfo
+
+      val outputTif = SinglebandGeoTiff("unstructured.tif")
+      outputTif.imageData.cols should be (inCols)
+      outputTif.imageData.rows should be (inRows)
+
+      // TODO check datatype, extent.
+    }
+
+    it("should round trip unstructured raster from COG"){
+      import spark.implicits._
+      import org.locationtech.rasterframes.datasource.raster._
+
+      val df = spark.read.raster.withTileDimensions(64, 64).load(singlebandCogPath.toASCIIString)
+
+      val resourceCols = 963 // from gdalinfo
+      val resourceRows = 754
+      val resourceExtent = Extent(752325.0, 3872685.0, 781215.0, 3895305.0)
+
+      df.count() should be > 0L
+
+      val crs = df.select(rf_crs(col("proj_raster"))).first()
+
+      val totalExtentRow = df.select(rf_extent($"proj_raster").alias("ext"))
+        .agg(
+          min($"ext.xmin").alias("xmin"),
+          min($"ext.ymin").alias("ymin"),
+          max($"ext.xmax").alias("xmax"),
+          max($"ext.ymax").alias("ymax")
+        ).first()
+      val dfExtent = Extent(totalExtentRow.getDouble(0), totalExtentRow.getDouble(1), totalExtentRow.getDouble(2), totalExtentRow.getDouble(3))
+      logger.info(s"Dataframe extent: ${dfExtent.toString()}")
+
+      dfExtent shouldBe (resourceExtent)
+
+      noException shouldBe thrownBy {
+        df.write.geotiff.withCRS(crs).save("target/unstructured_cog.tif")
+      }
+
+      val (inCols, inRows, inExtent, inCellType) = {
+        val tif = readSingleband("LC08_B7_Memphis_COG.tiff")
+        val id = tif.imageData
+        (id.cols, id.rows, tif.extent, tif.cellType)
+      }
+      inCols should be (963)
+      inRows should be (754) //from gdalinfo
+      inExtent should be (resourceExtent)
+
+      val outputTif = SinglebandGeoTiff("target/unstructured_cog.tif")
+      outputTif.imageData.cols should be (inCols)
+      outputTif.imageData.rows should be (inRows)
+      outputTif.extent should be (resourceExtent)
+      outputTif.cellType should be (inCellType)
+
+    }
+
     it("should write GeoTIFF without layer") {
       import org.locationtech.rasterframes.datasource.raster._
       val pr = col("proj_raster_b0")
@@ -152,6 +233,8 @@ ${s(1)},${s(4)},${s(3)}
         .withDimensions(256, 256)
         .save("geotiff-overview.tif")
 
+      val outTif = MultibandGeoTiff("geotiff-overview.tif")
+      outTif.bandCount should be (3)
     }
   }
 }