Merge pull request #235 from s22s/docs/gdal-install

vpipkt · web-flow · commit da0d0f47d3ed · 2019-08-06T14:38:35.000-04:00
GDAL related documentation
diff --git a/README.md b/README.md
@@ -1,10 +1,10 @@
-<img src="docs/src/main/paradox/_template/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&trade;</sup>
+<img src="docs/src/main/paradox/_template/images/RasterFramesLogo.png" width="300px"/><sup style="vertical-align: top;">&reg;</sup>
 
  [![Join the chat at https://gitter.im/s22s/raster-frames](https://badges.gitter.im/s22s/raster-frames.svg)](https://gitter.im/s22s/raster-frames?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 _RasterFrames™_ brings the power of Spark DataFrames to geospatial raster data, empowered by the map algebra and tile layer operations of [GeoTrellis](https://geotrellis.io/).
 
-<img src="docs/src/main/paradox/RasterFramePipeline.svg" width="600px"/>
+<img src="docs/src/main/paradox/RasterFramePipeline.png" width="600px"/>
 
 Please see the [Getting Started](http://rasterframes.io/getting-started.html) section of the Users' Manual to start using RasterFrames.
 
@@ -54,6 +54,6 @@ The `pyrasterframes` build instructions are located at [pyrasterframes/src/main/
 
 ## Copyright and License
 
-RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2018.
+RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2019.
 
 
diff --git a/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala b/core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala
@@ -63,8 +63,8 @@ class RasterSourceIT extends TestEnvironment with TestData {
     }
   }
 
-  if (RasterSource.IsGDAL.hasGDAL) {
-    println("GDAL version: " + GDALWarp.get_version_info("--version"))
+  if (GDALRasterSource.hasGDAL) {
+    println("GDAL version: " + GDALRasterSource.gdalVersion())
 
     describe("GDAL support") {
 
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala
@@ -23,12 +23,14 @@ package org.locationtech.rasterframes.ref
 
 import java.net.URI
 
+import com.azavea.gdal.GDALWarp
+import com.typesafe.scalalogging.LazyLogging
+import geotrellis.contrib.vlm.gdal.{GDALRasterSource => VLMRasterSource}
 import geotrellis.proj4.CRS
-import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster}
 import geotrellis.raster.io.geotiff.Tags
+import geotrellis.raster.{CellType, GridBounds, MultibandTile, Raster}
 import geotrellis.vector.Extent
 import org.locationtech.rasterframes.ref.RasterSource.URIRasterSource
-import geotrellis.contrib.vlm.gdal.{GDALRasterSource => VLMRasterSource}
 
 case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSource {
 
@@ -67,3 +69,17 @@ case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSour
   override protected def readBounds(bounds: Traversable[GridBounds], bands: Seq[Int]): Iterator[Raster[MultibandTile]] =
     gdal.readBounds(bounds, bands)
 }
+
+object GDALRasterSource extends LazyLogging {
+  def gdalVersion(): String = if (hasGDAL) GDALWarp.get_version_info("--version").trim else "not available"
+
+  @transient
+  lazy val hasGDAL: Boolean = try {
+    val _ = new GDALWarp()
+    true
+  } catch {
+    case _: UnsatisfiedLinkError =>
+      logger.warn("GDAL native bindings are not available. Falling back to JVM-based reader for GeoTIFF format.")
+      false
+  }
+}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala
@@ -23,7 +23,6 @@ package org.locationtech.rasterframes.ref
 
 import java.net.URI
 
-import com.azavea.gdal.GDALWarp
 import com.github.blemale.scaffeine.Scaffeine
 import com.typesafe.scalalogging.LazyLogging
 import geotrellis.proj4.CRS
@@ -120,27 +119,18 @@ object RasterSource extends LazyLogging {
 
     /** Determine if we should prefer GDAL for all types. */
     private val preferGdal: Boolean = org.locationtech.rasterframes.rfConfig.getBoolean("prefer-gdal")
-    @transient
-    lazy val hasGDAL: Boolean = try {
-      val _ = new GDALWarp()
-      true
-    } catch {
-      case _: UnsatisfiedLinkError =>
-        logger.warn("GDAL native bindings are not available. Falling back to JVM-based reader.")
-        false
-    }
 
-    val gdalOnlyExtensions = Seq(".jp2", ".mrf", ".hdf")
+    val gdalOnlyExtensions = Seq(".jp2", ".mrf", ".hdf", ".vrt")
 
     def gdalOnly(source: URI): Boolean =
       if (gdalOnlyExtensions.exists(source.getPath.toLowerCase.endsWith)) {
-        require(hasGDAL, s"Can only read $source if GDAL is available")
+        require(GDALRasterSource.hasGDAL, s"Can only read $source if GDAL is available")
         true
       } else false
 
     /** Extractor for determining if a scheme indicates GDAL preference.  */
     def unapply(source: URI): Boolean =
-      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && hasGDAL)
+      gdalOnly(source) || ((preferGdal || source.getScheme.startsWith("gdal")) && GDALRasterSource.hasGDAL)
   }
 
   object IsDefaultGeoTiff {
diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterSourceSpec.scala
@@ -29,11 +29,7 @@ import org.apache.spark.sql.rf.RasterSourceUDT
 import org.locationtech.rasterframes.TestEnvironment
 import org.locationtech.rasterframes.model.TileDimensions
 
-/**
- *
- *
- * @since 8/22/18
- */
+
 class RasterSourceSpec extends TestEnvironment with TestData {
   def sub(e: Extent) = {
     val c = e.center
@@ -112,7 +108,7 @@ class RasterSourceSpec extends TestEnvironment with TestData {
     }
   }
 
-  if(RasterSource.IsGDAL.hasGDAL) {
+  if(GDALRasterSource.hasGDAL) {
     describe("GDAL Rastersource") {
       val gdal = GDALRasterSource(cogPath)
       val jvm = JVMGeoTiffRasterSource(cogPath)
diff --git a/docs/build.sbt b/docs/build.sbt
@@ -0,0 +1,81 @@
+// task to create documentation PDF
+lazy val makePDF = taskKey[File]("Build PDF version of documentation")
+lazy val pdfFileName = settingKey[String]("Name of the PDF file generated")
+pdfFileName := s"RasterFrames-Users-Manual-${version.value}.pdf"
+
+makePDF := {
+  import scala.sys.process._
+
+  // Get the python source directory configured in the root project.
+  val base = (Compile / paradox / sourceDirectories).value.find(_.toString.contains("python")).head
+  println(base)
+
+  // Hard coded lacking any simple  way of determining order.
+  val files = Seq(
+    "index.md",
+    "description.md",
+    "concepts.md",
+    "getting-started.md",
+    "raster-io.md",
+    "raster-catalogs.md",
+    "raster-read.md",
+    "raster-write.md",
+    "vector-data.md",
+    "raster-processing.md",
+    "local-algebra.md",
+    "nodata-handling.md",
+    "aggregation.md",
+    "time-series.md",
+    "machine-learning.md",
+    "unsupervised-learning.md",
+    "supervised-learning.md",
+    "numpy-pandas.md",
+    "languages.md",
+    "reference.md"
+  ).map(base ** _).flatMap(_.get)
+
+  val log = streams.value.log
+  log.info("Section ordering:")
+  files.foreach(f => log.info(" - " + f.getName))
+
+  val work = target.value / "makePDF"
+  work.mkdirs()
+
+  val prepro = files.zipWithIndex.map { case (f, i) ⇒
+    val dest = work / f"$i%02d.md"
+    // Filter cross links and add a newline
+    (Seq("sed", "-e", """s/@ref:\[\([^]]*\)\](.*)/_\1_/g;s/@@.*//g""", f.toString) #> dest).!
+    // Add newline at the end of the file so as to make pandoc happy
+    ("echo" #>> dest).!
+    ("echo \\pagebreak" #>> dest).!
+    dest
+  }
+
+  val output = target.value / pdfFileName.value
+
+  val header = (Compile / sourceDirectory).value / "latex" / "header.latex"
+
+  val args = "pandoc" ::
+    "--from=markdown" ::
+    "--to=pdf" ::
+    "-t" :: "latex" ::
+    "-s" ::
+    "--toc" ::
+    "-V" :: "title:RasterFrames Users' Manual" ::
+    "-V" :: "author:Astraea, Inc." ::
+    "-V" :: "geometry:margin=0.75in" ::
+    "-V" :: "papersize:letter" ::
+    "-V" :: "links-as-notes" ::
+    "--include-in-header" :: header.toString ::
+    "-o" :: output.toString ::
+    prepro.map(_.toString).toList
+
+  log.info(s"Running: ${args.mkString(" ")}")
+  Process(args, base).!
+
+  log.info("Wrote: " + output)
+
+  output
+}
+
+makePDF := makePDF.dependsOn(Compile / paradox).value
diff --git a/docs/src/main/latex/header.latex b/docs/src/main/latex/header.latex
@@ -0,0 +1,2 @@
+\DeclareUnicodeCharacter{2218}{$\circ$}
+\DeclareUnicodeCharacter{2714}{$\checkmark}
diff --git a/docs/src/main/paradox/RasterFramePipeline.png b/docs/src/main/paradox/RasterFramePipeline.png
diff --git a/docs/src/main/paradox/release-notes.md b/docs/src/main/paradox/release-notes.md
@@ -11,17 +11,18 @@
 * Added new tile functions `rf_round`, `rf_abs`, `rf_log`, `rf_log10`, `rf_log2`, `rf_log1p`, `rf_exp`, `rf_exp10`, `rf_exp2`, `rf_expm1`, `rf_resample`.
 * Support Python-side Tile User-Defined Type backed by [numpy](https://www.numpy.org/) `ndarray` or `ma.MaskedArray`.
 * Support Python-side [Shapely](https://pypi.org/project/Shapely/) geometry User-Defined Type.
-* SQL API support for: `rf_assemble_tile`, `rf_array_to_tile`.
+* SQL API support for `rf_assemble_tile` and `rf_array_to_tile`.
 * Introduced at the source level the concept of a `RasterSource` and `RasterRef`, enabling lazy/delayed read of sub-scene tiles.
 * Added `withKryoSerialization` extension methods on `SparkSession.Builder` and `SparkConf`.
 * Added `rf_render_matrix` debugging function.
-* Added `RasterFrame.withExtent` extension method.
+* Added `RasterFrameLayer.withExtent` extension method.
 * Added `SinglebandGeoTiff.toDF` extension method.
 * Added `DataFrame.rasterJoin` extension method for merging two dataframes with tiles in disparate CRSs.
 * Added `rf_crs` for `ProjectedRasterTile` columns.
 * Added `st_extent` (for `Geometry` types) and `rf_extent` (for `ProjectedRasterTile` and `RasterSource` columns).
 * Added `st_geometry` (for `Extent` types) and `rf_geometry` (for `ProjectedRasterTile` and `RasterSource` columns).
-* _Breaking_: The type `RasterFrame` renamed `RasterFrameLayer` to be reflect its purpose.
+* Reworked build scripts for RasterFrames Jupyter Notebook. 
+* _Breaking_: The type `RasterFrame` renamed `RasterFrameLayer` to be reflect its intended purpose.
 * _Breaking_: All `asRF` methods renamed to `asLayer`.
 * _Breaking_: Root package changed from `org.locationtech.rasterframes` to `org.locationtech.rasterframes`.
 * _Breaking_: Removed `envelope`, in lieu of `st_extent`, `rf_extent` or `st_envelope` 
@@ -37,8 +38,7 @@
 * _Breaking_: `CellHistogram` no longer carries along approximate statistics, due to confusing behavior. Use `rf_agg_stats` instead.
 * Introduced `LocalCellStatistics` class to wrap together results from `LocalStatsAggregate`.
 * _Breaking_: `TileDimensions` moved from `astraea.spark.rasterframes` to `org.locationtech.rasterframes.model`.
-* _Breaking_: Renamed `RasterFrame.withBounds` to `RasterFrame.withGeometry` for consistency with DataSource schemas.
-* Reworked build scripts for RasterFrames Jupyter Notebook. 
+* _Breaking_: Renamed `RasterFrame.withBounds` to `RasterFrameLayer.withGeometry` for consistency with DataSource schemas.
    
 ## 0.7.x
 
diff --git a/pyrasterframes/src/main/python/docs/getting-started.pymd b/pyrasterframes/src/main/python/docs/getting-started.pymd
@@ -78,8 +78,41 @@ spark.stop()
 
 ## Installing GDAL 
 
-GDAL provides a wide variety of drivers to read data from many different raster formats. If GDAL is installed in the environment, RasterFrames will be able to @ref:[read](raster-read.md) those formats.  If you are using the @ref:[Jupyter Notebook image](getting-started.md#jupyter-notebook), GDAL is already installed for you. Otherwise follow the instructions below.
+GDAL provides a wide variety of drivers to read data from many different raster formats. If GDAL is installed in the environment, RasterFrames will be able to @ref:[read](raster-read.md) those formats.  If you are using the @ref:[Jupyter Notebook image](getting-started.md#jupyter-notebook), GDAL is already installed for you. Otherwise follow the instructions below. Version 2.4.1 or greater is required.
 
-__TODO__ Verify notebook env has GDAL? 
+### Installing on MacOS
 
-__TODO__ How to install it.
+Using [homebrew](https://brew.sh/):
+
+```bash
+brew install gdal
+```
+
+### Installing on Linux
+
+Using [`apt-get`](https://wiki.debian.org/Apt):
+
+```bash
+sudo apt-get update 
+sudo apt-get install gdal-bin
+```
+
+### Testing For GDAL
+
+```bash
+gdalinfo --formats
+```
+
+To support GeoTIFF and JPEG2000 formats, you should look for the following drivers from the output above:
+
+* `GTiff -raster- (rw+vs): GeoTIFF`
+* `JPEG2000 -raster,vector- (rwv): JPEG-2000 part 1 (ISO/IEC 15444-1), based on Jasper library`
+
+Do the following to see if RasterFrames was able to find GDAL:
+
+```python
+from pyrasterframes.utils import gdal_version
+print(gdal_version())
+```
+
+This will print out something like "GDAL x.y.z, released 20yy/mm/dd". If it reports `not available`, then GDAL isn't installed in a place where the RasterFrames runtime was able to find it. Please [file an issue](https://github.com/locationtech/rasterframes/issues) to get help resolving it.
diff --git a/pyrasterframes/src/main/python/docs/index.md b/pyrasterframes/src/main/python/docs/index.md
@@ -8,7 +8,7 @@ To learn more, please see the @ref:[Getting Started](getting-started.md) section
 
 The source code can be found on GitHub at [locationtech/rasterframes](https://github.com/locationtech/rasterframes).
 
-<img src="RasterFramePipeline.svg" width="600px"/>
+<img src="RasterFramePipeline.png" width="600px"/>
 
 <hr/>
 
@@ -24,7 +24,7 @@ The source code can be found on GitHub at [locationtech/rasterframes](https://gi
 
 ## Related Links
 
-* [Gitter Channel](https://gitter.im/s22s/raster-frames)&nbsp;&nbsp;[![Join the chat at https://gitter.im/s22s/raster-frames](https://badges.gitter.im/s22s/raster-frames.svg)](https://gitter.im/s22s/raster-frames?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+* [Gitter Channel](https://gitter.im/s22s/raster-frames)
 * [Scala API Documentation](latest/api/index.html)
 * [GitHub Repository](https://github.com/locationtech/rasterframes)
 * [Astraea, Inc.](http://www.astraea.earth/), the company behind RasterFrames
diff --git a/pyrasterframes/src/main/python/docs/raster-read.pymd b/pyrasterframes/src/main/python/docs/raster-read.pymd
@@ -54,19 +54,19 @@ display(tile)
 RasterFrames relies on three different IO drivers, selected based on a combination of scheme, file extentions, and library availability. GDAL is used first if a compatible version of GDAL (>= 2.4) is installed. It is the only driver that can read non-GeoTIFF files. If GDAL is not available, either the _Java I/O` or _Hadoop_ driver will be selected, depending on scheme.  
 
 
-| Prefix              | GDAL        | Java I/O | Hadoop |
-| ------------------- | ----------- | -------- | ------ |
-| `gdal://<vsidrv>//` |  ✔︎          | -︎        | -      |
-| `file://`           |  ✔︎          | ✔︎       | -      |
-| `http://`           |  ✔︎          | ✔︎       | -      |
-| `https://`          |  ✔︎          | ✔︎       | -      |
-| `ftp://`            | `/vsicurl/`  | ✔       | -      |
-| `hdfs://`           |  `/vsihdfs/` ︎| -       | ✔︎      |
-| `s3://`             |  `/vsis3/` ︎  | ✔︎       | -      |
-| `s3n://`            |  - ︎          | -       | ✔︎      |
-| `s3a://`            |  - ︎          | -       | ✔︎      |
-| `wasb://`           |  `/vsiaz/` ︎  | -       | ✔︎      |
-| `wasbs://`          |  - ︎          | -       | ✔︎      |
+| Prefix              | GDAL        | Java I/O | Hadoop   |
+| ------------------- | ----------- | -------- | -------- |
+| `gdal://<vsidrv>//` | +    | -        | -        |
+| `file://`           | +    | + | -        |
+| `http://`           | +    | + | -        |
+| `https://`          | +    | + | -        |
+| `ftp://`            | `/vsicurl/` | + | -        |
+| `hdfs://`           | `/vsihdfs/` |-         | + |
+| `s3://`             |  `/vsis3/`  | + | -        |
+| `s3n://`            |  -          | -        | + |
+| `s3a://`            |  -          | -        | + |
+| `wasb://`           |  `/vsiaz/`  | -        | + |
+| `wasbs://`          |  -          | -        | + |
 
 Specific [GDAL Virtual File System drivers](https://gdal.org/user/virtual_file_systems.html) can be selected using the `gdal://<vsidrv>//` syntax. For example If you have a `archive.zip` file containing a GeoTiff named `my-file-inside.tif`, you can address it with `gdal://vsizip//path/to/archive.zip/my-file-inside.tif`. See the GDAL documentation for the format of the URIs after the `gdal:/` prefix (which is stripped off before passing the rest of the path to GDAL).
 
diff --git a/pyrasterframes/src/main/python/docs/supervised-learning.pymd b/pyrasterframes/src/main/python/docs/supervised-learning.pymd
@@ -5,7 +5,8 @@ In this example we will demonstrate how to fit and score a supervised learning m
 ```python, setup, echo=False
 from IPython.core.display import display
 from pyrasterframes.utils import create_rf_spark_session
-from pyrasterframes.rasterfunctions import * 
+from pyrasterframes.rasterfunctions import *
+from pyspark.sql.functions import lit 
 import pandas as pd
 import numpy as np
 import matplotlib.pyplot as plt
@@ -71,7 +72,7 @@ print('Found ', len(crses), 'distinct CRS.')
 crs = crses[0][0]
 
 label_df = spark.read.geojson(os.path.join(resource_dir_uri(), 'luray-labels.geojson')) \
-					 .select('id', st_reproject('geometry', 'EPSG:4326', crs).alias('geometry')) \
+					 .select('id', st_reproject('geometry', lit('EPSG:4326'), lit(crs)).alias('geometry')) \
 					 .hint('broadcast')
 
 df_joined = df.join(label_df, st_intersects(st_geometry('extent'), 'geometry')) 
diff --git a/pyrasterframes/src/main/python/pyrasterframes/utils.py b/pyrasterframes/src/main/python/pyrasterframes/utils.py
@@ -22,8 +22,9 @@
 from pyspark.sql import SparkSession
 import os
 import sys
+from . import RFContext
 
-__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly"]
+__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version"]
 
 
 def find_pyrasterframes_jar_dir():
@@ -92,3 +93,8 @@ def create_rf_spark_session():
     except TypeError as te:
         print("Error setting up SparkSession; cannot find the pyrasterframes assembly jar\n", te)
         return None
+
+
+def gdal_version():
+    fcn = RFContext.active().lookup("buildInfo")
+    return fcn()["GDAL"]
diff --git a/pyrasterframes/src/main/python/tests/RasterFunctionsTests.py b/pyrasterframes/src/main/python/tests/RasterFunctionsTests.py
diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala
diff --git a/rf-notebook/build.sbt b/rf-notebook/build.sbt
diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile

Original file line number	Diff line number	Diff line change
`@@ -63,8 +63,8 @@ class RasterSourceIT extends TestEnvironment with TestData {`
`63`	`63`	`}`
`64`	`64`	`}`
`65`	`65`
`66`		`- if (RasterSource.IsGDAL.hasGDAL) {`
`67`		`- println("GDAL version: " + GDALWarp.get_version_info("--version"))`
	`66`	`+ if (GDALRasterSource.hasGDAL) {`
	`67`	`+ println("GDAL version: " + GDALRasterSource.gdalVersion())`
`68`	`68`
`69`	`69`	`describe("GDAL support") {`
`70`	`70`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+\DeclareUnicodeCharacter{2218}{$\circ$}`
	`2`	`+\DeclareUnicodeCharacter{2714}{$\checkmark}`