Merge pull request #219 from s22s/feature/raster-read-uris

Added VSI driver handling via gdal:// scheme.

Author: metasim

core/src/main/resources/reference.conf

@@ -12,7 +12,7 @@ vlm.gdal {
       //CPL_DEBUG = "OFF"
       AWS_REQUEST_PAYER = "requester"
       GDAL_DISABLE_READDIR_ON_OPEN = "YES"
-      CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif,.tiff,.jp2,.mrf,.idx,.lrc,.mrf.aux.xml"
+      CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif,.tiff,.jp2,.mrf,.idx,.lrc,.mrf.aux.xml,.vrt"
     }
     // set this to `false` if CPL_DEBUG is `ON`
     useExceptions = true

core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala

@@ -34,7 +34,9 @@ case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSour
 
   @transient
   private lazy val gdal: VLMRasterSource = {
-    val cleaned = source.toASCIIString.replace("gdal+", "")
+    val cleaned = source.toASCIIString
+      .replace("gdal+", "")
+      .replace("gdal:/", "")
     // VSIPath doesn't like single slash "file:/path..."
     val tweaked =
       if (cleaned.matches("^file:/[^/].*"))

core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala

@@ -140,7 +140,7 @@ object RasterSource extends LazyLogging {
 
     /** Extractor for determining if a scheme indicates GDAL preference.  */
     def unapply(source: URI): Boolean =
-      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal+")) && hasGDAL)
+      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && hasGDAL)
   }
 
   object IsDefaultGeoTiff {

core/src/test/resources/L8-archive.zip

@@ -25,7 +25,7 @@ import geotrellis.proj4.LatLng
 import geotrellis.raster.{ByteCellType, GridBounds, TileLayout}
 import geotrellis.spark.tiling.{CRSWorldExtent, LayoutDefinition}
 import geotrellis.spark.{KeyBounds, SpatialKey, TileLayerMetadata}
-import org.apache.spark.sql.Encoders
+import org.apache.spark.sql.{Encoder, Encoders}
 import org.locationtech.rasterframes.util.SubdivideSupport
 
 /**
@@ -57,23 +57,27 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
   }
   describe("Miscellaneous extensions") {
     import spark.implicits._
+
     it("should find multiple extent columns") {
       val df = Seq((extent, "fred", extent, 34.0)).toDF("e1", "s", "e2", "n")
       df.extentColumns.size should be(2)
     }
+
     it("should find multiple crs columns") {
       // Not sure why implicit resolution isn't handling this properly.
       implicit val enc = Encoders.tuple(crsEncoder, Encoders.STRING, crsEncoder, Encoders.scalaDouble)
       val df = Seq((pe.crs, "fred", pe.crs, 34.0)).toDF("c1", "s", "c2", "n")
       df.crsColumns.size should be (2)
     }
+
     it("should split TileLayout") {
       val tl1 = TileLayout(2, 3, 10, 10)
       assert(tl1.subdivide(0) === tl1)
       assert(tl1.subdivide(1) === tl1)
       assert(tl1.subdivide(2) === TileLayout(4, 6, 5, 5))
       assertThrows[IllegalArgumentException](tl1.subdivide(-1))
     }
+
     it("should split KeyBounds[SpatialKey]") {
       val grid = GridBounds(0, 0, 9, 9)
       val kb = KeyBounds(grid)

core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala

@@ -21,6 +21,8 @@
 
 package org.locationtech.rasterframes.ref
 
+import java.net.URI
+
 import org.locationtech.rasterframes.TestData
 import geotrellis.vector.Extent
 import org.apache.spark.sql.rf.RasterSourceUDT
@@ -125,6 +127,15 @@ class RasterSourceSpec extends TestEnvironment with TestData {
         gdal.layoutBounds(dims) should contain allElementsOf jvm.layoutBounds(dims)
         gdal.layoutExtents(dims) should contain allElementsOf jvm.layoutExtents(dims)
       }
+
+
+      it("should support vsi file paths") {
+        val archivePath = geotiffDir.resolve("L8-archive.zip")
+        val archiveURI = URI.create("gdal://vsizip/" + archivePath.toString + "/L8-RGB-VA.tiff")
+        val gdal = GDALRasterSource(archiveURI)
+
+        gdal.bandCount should be (3)
+      }
     }
   }
 

core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala

@@ -51,19 +51,24 @@ display(tile)
 
 ## URI Formats
 
-/[ TODO Populate this with valid URI's schemes we can use with an example? including how to use gdal /]
-
-file://  (is this only valid for spark local? or if file is )
-https://
-http:// ?
-s3a://
-s3n:// ?
-hdfs:// ?
-ftp:// ?
-azure ?
-google cloud ?
-
-gdal vsi? https://gdal.org/user/virtual_file_systems.html 
+RasterFrames relies on three different IO drivers, selected based on a combination of scheme, file extentions, and library availability. GDAL is used first if a compatible version of GDAL (>= 2.4) is installed. It is the only driver that can read non-GeoTIFF files. If GDAL is not available, either the _Java I/O` or _Hadoop_ driver will be selected, depending on scheme.  
+
+
+| Prefix              | GDAL        | Java I/O | Hadoop |
+| ------------------- | ----------- | -------- | ------ |
+| `gdal://<vsidrv>//` |  ✔︎          | -︎        | -      |
+| `file://`           |  ✔︎          | ✔︎       | -      |
+| `http://`           |  ✔︎          | ✔︎       | -      |
+| `https://`          |  ✔︎          | ✔︎       | -      |
+| `ftp://`            | `/vsicurl/`  | ✔       | -      |
+| `hdfs://`           |  `/vsihdfs/` ︎| -       | ✔︎      |
+| `s3://`             |  `/vsis3/` ︎  | ✔︎       | -      |
+| `s3n://`            |  - ︎          | -       | ✔︎      |
+| `s3a://`            |  - ︎          | -       | ✔︎      |
+| `wasb://`           |  `/vsiaz/` ︎  | -       | ✔︎      |
+| `wasbs://`          |  - ︎          | -       | ✔︎      |
+
+Specific [GDAL Virtual File System drivers](https://gdal.org/user/virtual_file_systems.html) can be selected using the `gdal://<vsidrv>//` syntax. For example If you have a `archive.zip` file containing a GeoTiff named `my-file-inside.tif`, you can address it with `gdal://vsizip//path/to/archive.zip/my-file-inside.tif`. See the GDAL documentation for the format of the URIs after the `gdal:/` prefix (which is stripped off before passing the rest of the path to GDAL).
 
 
 ## Raster Catalogs

pyrasterframes/src/main/python/docs/raster-read.pymd

@@ -134,6 +134,7 @@ def doctype(self):
         'pytest==3.4.2',
         'pypandoc',
         'numpy>=1.7',
+        'shapley',
         'pandas',
         'rasterio'
     ],