Merge pull request #256 from s22s/fix/169

Use implicit file:// for schemeless paths with leading /

Author: vpipkt

core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala

@@ -129,13 +129,18 @@ object RasterSource extends LazyLogging {
       } else false
 
     /** Extractor for determining if a scheme indicates GDAL preference.  */
-    def unapply(source: URI): Boolean =
-      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && GDALRasterSource.hasGDAL)
+    def unapply(source: URI): Boolean = {
+      lazy val schemeIsGdal = Option(source.getScheme())
+        .exists(_.startsWith("gdal"))
+
+      gdalOnly(source) || ((preferGdal || schemeIsGdal) && GDALRasterSource.hasGDAL)
+    }
   }
 
   object IsDefaultGeoTiff {
     def unapply(source: URI): Boolean = source.getScheme match {
-      case "file" | "http" | "https" | "s3" | "" => true
+      case "file" | "http" | "https" | "s3" => true
+      case null | ""                        ⇒ true
       case _                                => false
     }
   }

core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala

@@ -106,6 +106,13 @@ class RasterSourceSpec extends TestEnvironment with TestData {
       val src = RasterSource(localSrc)
       assert(!src.extent.isEmpty)
     }
+    it("should interpret no scheme as file://"){
+      val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toString()
+      val schemelessUri = new URI(localSrc)
+      schemelessUri.getScheme should be (null)
+      val src = RasterSource(schemelessUri)
+      assert(!src.extent.isEmpty)
+    }
   }
 
   if(GDALRasterSource.hasGDAL) {
@@ -132,6 +139,15 @@ class RasterSourceSpec extends TestEnvironment with TestData {
 
         gdal.bandCount should be (3)
       }
+
+      it("should interpret no scheme as file://") {
+        val localSrc = geotiffDir.resolve("LC08_B7_Memphis_COG.tiff").toString()
+        val schemelessUri = new URI(localSrc)
+        val gdal = GDALRasterSource(schemelessUri)
+        val jvm = JVMGeoTiffRasterSource(schemelessUri)
+        gdal.extent should be (jvm.extent)
+        gdal.cellSize should be(jvm.cellSize)
+      }
     }
   }
 

pyrasterframes/src/main/python/docs/raster-write.pymd

@@ -66,7 +66,7 @@ Fortunately, we can use the cluster computing capability to downsample the data
 
 ```python write_geotiff
 outfile = os.path.join('/tmp', 'geotiff-overview.tif')
-spark_df.write.geotiff('file://' + outfile, crs='EPSG:4326', raster_dimensions=(256, 256))
+spark_df.write.geotiff(outfile, crs='EPSG:4326', raster_dimensions=(256, 256))
 ```
 
 View it with `rasterio` to check the results:
@@ -83,6 +83,10 @@ with rasterio.open(outfile) as src:
 
 If there are many tile or projected raster columns in the DataFrame, the GeoTIFF writer will write each one as a separate band in the file. Each band in the output will be tagged the input column names for reference.
 
+```python, echo=False
+os.remove(outfile)
+```
+
 ## GeoTrellis Layers
 
 [GeoTrellis][GeoTrellis] is one of the key libraries that RasterFrames builds upon. It provides a Scala language API to working with large raster data with Apache Spark. Ingesting raster data into a Layer is one of the key concepts for creating a dataset for processing on Spark. RasterFrames write data from an appropriate DataFrame into a [GeoTrellis Layer](https://geotrellis.readthedocs.io/en/latest/guide/tile-backends.html). RasterFrames provides a `geotrellis` DataSource that supports both reading and writing of GeoTrellis layers.

pyrasterframes/src/main/python/tests/GeoTiffWriterTests.py

@@ -33,11 +33,14 @@ def _tmpfile():
 
     def test_identity_write(self):
         rf = self.spark.read.geotiff(self.img_uri)
+        rf_count = rf.count()
+        self.assertTrue(rf_count > 0)
 
         dest = self._tmpfile()
         rf.write.geotiff(dest)
 
-        rf2 = self.spark.read.geotiff('file://' + dest)
+        rf2 = self.spark.read.geotiff(dest)
+
         self.assertEqual(rf2.count(), rf.count())
 
         os.remove(dest)
@@ -47,7 +50,7 @@ def test_unstructured_write(self):
         dest_file = self._tmpfile()
         rf.write.geotiff(dest_file, crs='EPSG:32616')
 
-        rf2 = self.spark.read.raster('file://' + dest_file)
+        rf2 = self.spark.read.raster(dest_file)
         self.assertEqual(rf2.count(), rf.count())
 
         with rasterio.open(self.img_uri) as source:
@@ -58,6 +61,22 @@ def test_unstructured_write(self):
 
         os.remove(dest_file)
 
+    def test_unstructured_write_schemeless(self):
+        # should be able to write a projected raster tile column to path like '/data/foo/file.tif'
+        from pyrasterframes.rasterfunctions import rf_agg_stats, rf_crs
+        rf = self.spark.read.raster(self.img_uri)
+        max = rf.agg(rf_agg_stats('proj_raster').max.alias('max')).first()['max']
+        crs = rf.select(rf_crs('proj_raster').crsProj4.alias('c')).first()['c']
+
+        dest_file = self._tmpfile()
+        self.assertTrue(not dest_file.startswith('file://'))
+        rf.write.geotiff(dest_file, crs=crs)
+
+        with rasterio.open(dest_file) as src:
+            self.assertEqual(src.read().max(), max)
+
+        os.remove(dest_file)
+
     def test_downsampled_write(self):
         rf = self.spark.read.raster(self.img_uri)
         dest = self._tmpfile()

pyrasterframes/src/main/python/tests/PyRasterFramesTests.py

@@ -470,6 +470,13 @@ def l8path(b):
         print(path_count.toPandas())
         self.assertTrue(path_count.count() == 3)
 
+    def test_raster_source_reader_schemeless(self):
+        import os.path
+        path = os.path.join(self.resource_dir, "L8-B8-Robinson-IL.tiff")
+        self.assertTrue(not path.startswith('file://'))
+        df = self.spark.read.raster(path)
+        self.assertTrue(df.count() > 0)
+
     def test_raster_source_catalog_reader(self):
         import pandas as pd