Merge remote-tracking branch 'locationtech/develop' into feature/xz2

vpipkt · vpipkt · commit e799a03b04b6 · 2019-10-04T09:50:05.000-04:00
Signed-off-by: Jason T. Brown &lt;jason@astraea.earth&gt;
diff --git a/build/circleci/Dockerfile b/build/circleci/Dockerfile
@@ -1,6 +1,6 @@
 FROM circleci/openjdk:8-jdk
 
-ENV OPENJPEG_VERSION 2.3.0
+ENV OPENJPEG_VERSION 2.3.1
 ENV GDAL_VERSION 2.4.1
 ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
 
@@ -11,26 +11,37 @@ RUN sudo apt-get update && \
       python python-minimal python2.7 python2.7-minimal \
       libpython-stdlib libpython2.7 libpython2.7-minimal libpython2.7-stdlib \
     && sudo apt-get install -y \
-      python3 \
-      python3-pip \
       pandoc \
       wget \
       gcc g++ build-essential \
+      libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
             libcurl4-gnutls-dev \
             libproj-dev \
             libgeos-dev \
             libhdf4-alt-dev \
-            libhdf5-serial-dev \
             bash-completion \
             cmake \
             imagemagick \
             libpng-dev \
-            swig \
-            ant \
+            libffi-dev \
     && sudo apt autoremove \
-    && sudo apt-get clean all \
-    && pip3 install setuptools ipython==6.2.1 \
-    && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+    && sudo apt-get clean all
+#    && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
+# todo s
+
+RUN cd /tmp && \
+  wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
+  tar xzf Python-3.7.4.tgz && \
+  cd Python-3.7.4 && \
+  ./configure  --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
+  make && \
+  sudo make altinstall && \
+  rm -rf Python-3.7.4*
+
+RUN sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
+  sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+  sudo python get-pip.py && \
+  sudo pip3 install setuptools ipython==6.2.1
 
 # install OpenJPEG
 RUN cd /tmp && \
@@ -52,11 +63,9 @@ RUN cd /tmp && \
     ./configure \
         --with-curl \
         --with-hdf4 \
-        --with-hdf5 \
         --with-geos \
         --with-geotiff=internal \
         --with-hide-internal-symbols \
-        --with-java=$JAVA_HOME \
         --with-libtiff=internal \
         --with-libz=internal \
         --with-mrf \
@@ -68,7 +77,5 @@ RUN cd /tmp && \
     && \
     make -j 8 && \
     sudo make install && \
-    cd swig/java && \
-    sudo make install && \
     sudo ldconfig && \
     cd /tmp && sudo rm -Rf gdal*
diff --git a/build/circleci/Makefile b/build/circleci/Makefile
@@ -0,0 +1,2 @@
+all:
+	docker build -t "s22s/rasterframes-circleci:latest" .
diff --git a/build/circleci/README.md b/build/circleci/README.md
@@ -1,6 +1,6 @@
 # CircleCI Dockerfile Build file
 
 ```bash
-docker build -t s22s/rasterframes-circleci:latest .
+make
 docker push s22s/rasterframes-circleci:latest
 ```
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala b/core/src/main/scala/org/apache/spark/sql/rf/TileUDT.scala
@@ -26,6 +26,7 @@ import org.apache.spark.sql.types.{DataType, _}
 import org.locationtech.rasterframes.encoders.CatalystSerializer
 import org.locationtech.rasterframes.encoders.CatalystSerializer._
 import org.locationtech.rasterframes.model.{Cells, TileDataContext}
+import org.locationtech.rasterframes.ref.RasterRef.RasterRefTile
 import org.locationtech.rasterframes.tiles.InternalRowTile
 
 
@@ -75,12 +76,15 @@ case object TileUDT  {
   implicit def tileSerializer: CatalystSerializer[Tile] = new CatalystSerializer[Tile] {
 
     override val schema: StructType = StructType(Seq(
-      StructField("cell_context", schemaOf[TileDataContext], false),
+      StructField("cell_context", schemaOf[TileDataContext], true),
       StructField("cell_data", schemaOf[Cells], false)
     ))
 
     override def to[R](t: Tile, io: CatalystIO[R]): R = io.create(
-      io.to(TileDataContext(t)),
+      t match {
+        case _: RasterRefTile => null
+        case o => io.to(TileDataContext(o))
+      },
       io.to(Cells(t))
     )
 
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/DelegatingRasterSource.scala
@@ -43,7 +43,7 @@ abstract class DelegatingRasterSource(source: URI, delegateBuilder: () => GTRast
       f(_delRef)
     }
     catch {
-      // On this exeception we attempt to recreate the delegate and read again.
+      // On this Exeception we attempt to recreate the delegate and read again.
       case _: java.nio.BufferUnderflowException =>
         _delRef = null
         val newDel = delegateBuilder()
@@ -62,7 +62,7 @@ abstract class DelegatingRasterSource(source: URI, delegateBuilder: () => GTRast
   override def hashCode(): Int = source.hashCode()
 
   // This helps reduce header reads between serializations
-  def info: SimpleRasterInfo = SimpleRasterInfo.cache.get(source.toASCIIString, _ =>
+  def info: SimpleRasterInfo = SimpleRasterInfo(source.toASCIIString, _ =>
     retryableRead(rs => SimpleRasterInfo(rs))
   )
 
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/GDALRasterSource.scala
@@ -48,7 +48,7 @@ case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSour
     VLMRasterSource(tweaked)
   }
 
-  protected def tiffInfo = SimpleRasterInfo.cache.get(source.toASCIIString, _ => SimpleRasterInfo(gdal))
+  protected def tiffInfo = SimpleRasterInfo(source.toASCIIString, _ => SimpleRasterInfo(gdal))
 
   override def crs: CRS = tiffInfo.crs
 
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterRef.scala
@@ -46,7 +46,7 @@ case class RasterRef(source: RasterSource, bandIndex: Int, subextent: Option[Ext
   def cols: Int = grid.width
   def rows: Int = grid.height
   def cellType: CellType = source.cellType
-  def tile: ProjectedRasterTile = ProjectedRasterTile(RasterRefTile(this), extent, crs)
+  def tile: ProjectedRasterTile = RasterRefTile(this)
 
   protected lazy val grid: GridBounds =
     subgrid.getOrElse(source.rasterExtent.gridBoundsFor(extent, true))
@@ -61,12 +61,12 @@ object RasterRef extends LazyLogging {
   private val log = logger
 
   case class RasterRefTile(rr: RasterRef) extends ProjectedRasterTile {
-    val extent: Extent = rr.extent
-    val crs: CRS = rr.crs
+    def extent: Extent = rr.extent
+    def crs: CRS = rr.crs
     override def cellType = rr.cellType
 
-    override val cols: Int = rr.cols
-    override val rows: Int = rr.rows
+    override def cols: Int = rr.cols
+    override def rows: Int = rr.rows
 
     protected def delegate: Tile = rr.realizedTile
     // NB: This saves us from stack overflow exception
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/RasterSource.scala
@@ -92,7 +92,7 @@ object RasterSource extends LazyLogging {
 
   val cacheTimeout: Duration = Duration.fromNanos(rfConfig.getDuration("raster-source-cache-timeout").toNanos)
 
-  private val rsCache = Scaffeine()
+  private[ref] val rsCache = Scaffeine()
     .recordStats()
     .expireAfterAccess(RasterSource.cacheTimeout)
     .build[String, RasterSource]
diff --git a/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala b/core/src/main/scala/org/locationtech/rasterframes/ref/SimpleRasterInfo.scala
@@ -44,6 +44,11 @@ case class SimpleRasterInfo(
 )
 
 object SimpleRasterInfo {
+  // Not a fan of this.... need a better abstraction that doesn't put the
+  // type-specific logic in here.
+  def apply(key: String, builder: String => SimpleRasterInfo): SimpleRasterInfo =
+    cache.get(key, builder)
+
   def apply(info: GeoTiffReader.GeoTiffInfo): SimpleRasterInfo =
     SimpleRasterInfo(
       info.segmentLayout.totalCols,
@@ -76,10 +81,9 @@ object SimpleRasterInfo {
     )
   }
 
-  private[rasterframes]
-  lazy val cache = Scaffeine()
+  private lazy val cache = Scaffeine()
     .recordStats()
     .build[String, SimpleRasterInfo]
 
   def cacheStats = cache.stats()
-}
+}
diff --git a/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala b/core/src/main/scala/org/locationtech/rasterframes/tiles/ProjectedRasterTile.scala
@@ -72,12 +72,15 @@ object ProjectedRasterTile {
   }
   implicit val serializer: CatalystSerializer[ProjectedRasterTile] = new CatalystSerializer[ProjectedRasterTile] {
     override val schema: StructType = StructType(Seq(
-      StructField("tile_context", schemaOf[TileContext], false),
+      StructField("tile_context", schemaOf[TileContext], true),
       StructField("tile", TileType, false))
     )
 
     override protected def to[R](t: ProjectedRasterTile, io: CatalystIO[R]): R = io.create(
-      io.to(TileContext(t.extent, t.crs)),
+      t match {
+        case _: RasterRefTile => null
+        case o => io.to(TileContext(o.extent, o.crs))
+      },
       io.to[Tile](t)(TileUDT.tileSerializer)
     )
 
diff --git a/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/RasterFrameSpec.scala
@@ -232,23 +232,6 @@ class RasterFrameSpec extends TestEnvironment with MetadataKeys
       assert(bounds._2 === SpaceTimeKey(3, 1, now))
     }
 
-//    it("should clip TileLayerMetadata extent") {
-//      val tiled = sampleTileLayerRDD
-//
-//      val rf = tiled.reproject(LatLng, tiled.metadata.layout)._2.toLayer
-//
-//      val worldish = Extent(-179, -89, 179, 89)
-//      val areaish = Extent(-90, 30, -81, 40)
-//
-//      val orig = rf.tileLayerMetadata.widen.extent
-//      assert(worldish.contains(orig))
-//      assert(areaish.contains(orig))
-//
-//      val clipped = rf.clipLayerExtent.tileLayerMetadata.widen.extent
-//      assert(!clipped.contains(worldish))
-//      assert(clipped.contains(areaish))
-//    }
-
     def basicallySame(expected: Extent, computed: Extent): Unit = {
       val components = Seq(
         (expected.xmin, computed.xmin),
diff --git a/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala b/core/src/test/scala/org/locationtech/rasterframes/ref/RasterRefSpec.scala
@@ -252,5 +252,16 @@ class RasterRefSpec extends TestEnvironment with TestData {
         assertEqual(t.toArrayTile(), result)
       }
     }
+
+    it("should construct a RasterRefTile without I/O") {
+      new Fixture {
+        // SimpleRasterInfo is a proxy for header data requests.
+        val start = SimpleRasterInfo.cacheStats.hitCount()
+        val t: ProjectedRasterTile = RasterRefTile(subRaster)
+        val result = Seq(t, subRaster.tile).toDF("tile").first()
+        val end = SimpleRasterInfo.cacheStats.hitCount()
+        end should be(start)
+      }
+    }
   }
 }
diff --git a/docs/src/main/paradox/release-notes.md b/docs/src/main/paradox/release-notes.md
@@ -6,6 +6,7 @@
 
 * Updated `rf_crs` to accept string columns containing CRS specifications. ([#366](https://github.com/locationtech/rasterframes/issues/366))
 * Added `rf_spatial_index` function. ([#368](https://github.com/locationtech/rasterframes/issues/368))
+* _Breaking_ (potentially): removed `pyrasterframes.create_spark_session` in lieu of `pyrasterframes.utils.create_rf_spark_session`
 
 ### 0.8.2
 
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -15,5 +15,6 @@ addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.1")
 addSbtPlugin("net.vonbuchholtz" % "sbt-dependency-check" % "0.2.10")
 addSbtPlugin("com.github.gseitz" %% "sbt-release" % "1.0.9")
 addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.19")
+addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.6.0")
 
 
diff --git a/pyrasterframes/src/main/python/docs/description.pymd b/pyrasterframes/src/main/python/docs/description.pymd
@@ -2,10 +2,11 @@
 
 ```python, setup, echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 import pyrasterframes.rf_ipython
 from pyrasterframes.rasterfunctions import rf_crs, rf_extent, rf_tile, rf_data_cells
 from pyspark.sql.functions import col, lit
-spark = pyrasterframes.get_spark_session()
+spark = create_rf_spark_session()
 
 # Note that this is the same URI as in the getting started page...
 df = spark.read.raster('https://modis-pds.s3.amazonaws.com/MCD43A4.006/11/08/2019059/MCD43A4.A2019059.h11v08.006.2019072203257_B02.TIF')
diff --git a/pyrasterframes/src/main/python/docs/getting-started.pymd b/pyrasterframes/src/main/python/docs/getting-started.pymd
@@ -29,7 +29,8 @@ from pyrasterframes import rf_ipython
 
 ```python, version
 import pyrasterframes
-spark = pyrasterframes.get_spark_session()
+from pyrasterframes.utils import create_rf_spark_session
+spark = create_rf_spark_session()
 ```
 
 Then, you can read a raster and work with it in a Spark DataFrame.
diff --git a/pyrasterframes/src/main/python/docs/nodata-handling.pymd b/pyrasterframes/src/main/python/docs/nodata-handling.pymd
@@ -12,14 +12,15 @@ To understand how NoData is handled in RasterFrames, we first need to understand
 
 ```python setup, echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 import pyrasterframes.rf_ipython
 from IPython.display import display
 import pandas as pd
 import numpy as np
 from pyrasterframes.rf_types import Tile
 
-spark = pyrasterframes.get_spark_session()
+spark = create_rf_spark_session()
 ```
 
 The `CellType` class from the `rf_types` submodule allows us to create a representation of any valid cell type. There are convenience methods to create instances for a variety of basic types.
diff --git a/pyrasterframes/src/main/python/docs/numpy-pandas.pymd b/pyrasterframes/src/main/python/docs/numpy-pandas.pymd
@@ -34,9 +34,10 @@ As discussed in the @ref:[raster writing chapter](raster-write.md#dataframe-samp
 
 ```python, spark_session,  echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from IPython.display import display
-spark = pyrasterframes.get_spark_session()
+spark = create_rf_spark_session()
 
 ```
 
diff --git a/pyrasterframes/src/main/python/docs/raster-write.pymd b/pyrasterframes/src/main/python/docs/raster-write.pymd
@@ -6,11 +6,12 @@ However, there are times in any analysis where writing a representative sample o
 
 ```python imports, echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from IPython.display import display
 import os.path
 
-spark = pyrasterframes.get_spark_session()
+spark = create_rf_spark_session()
 ```
 
 ## Tile Samples
diff --git a/pyrasterframes/src/main/python/docs/reference.pymd b/pyrasterframes/src/main/python/docs/reference.pymd
@@ -12,11 +12,12 @@ For the Scala documentation on these functions, see @scaladoc[`RasterFunctions`]
 
 ```python imports, echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from IPython.display import display
 import os.path
 
-spark = pyrasterframes.get_spark_session()
+spark = create_rf_spark_session()
 ```
 
 ## List of Available SQL and Python Functions
diff --git a/pyrasterframes/src/main/python/docs/time-series.pymd b/pyrasterframes/src/main/python/docs/time-series.pymd
@@ -6,14 +6,15 @@
 from IPython.display import display
 
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 import pyrasterframes.rf_ipython
 
 from pyspark.sql.functions import udf, lit
 from geomesa_pyspark.types import MultiPolygonUDT
 
 # This job is more memory bound, so reduce the concurrent tasks.
-spark = pyrasterframes.get_spark_session("local[4]")
+spark = create_rf_spark_session("local[4]")
 ```
 
 In this example, we will show how the flexibility of the DataFrame concept for raster data allows a simple and intuitive way to extract a time series from Earth observation data. We will continue our example from the @ref:[Zonal Map Algebra page](zonal-algebra.md).
diff --git a/pyrasterframes/src/main/python/docs/vector-data.pymd b/pyrasterframes/src/main/python/docs/vector-data.pymd
@@ -10,10 +10,11 @@ RasterFrames provides a variety of ways to work with spatial vector data (points
 
 ```python, setup, echo=False
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 import pyrasterframes.rf_ipython
 import geopandas
 import folium
-spark = pyrasterframes.get_spark_session('local[2]')
+spark = create_rf_spark_session('local[2]')
 ```
 
 ## GeoJSON DataSource
diff --git a/pyrasterframes/src/main/python/docs/zonal-algebra.pymd b/pyrasterframes/src/main/python/docs/zonal-algebra.pymd
@@ -4,14 +4,15 @@
 from IPython.display import display
 
 import pyrasterframes
+from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 import pyrasterframes.rf_ipython
 
 from pyspark.sql.functions import udf, lit
 from geomesa_pyspark.types import MultiPolygonUDT
 
 # This seemed to work for time-series!
-spark = pyrasterframes.get_spark_session("local[4]")
+spark = create_rf_spark_session("local[4]")
 ```
 
 ## Definition
diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_context.py b/pyrasterframes/src/main/python/pyrasterframes/rf_context.py
diff --git a/pyrasterframes/src/main/python/pyrasterframes/rf_types.py b/pyrasterframes/src/main/python/pyrasterframes/rf_types.py
diff --git a/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala b/pyrasterframes/src/main/scala/org/locationtech/rasterframes/py/PyRFContext.scala

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+all:`
	`2`	`+ docker build -t "s22s/rasterframes-circleci:latest" .`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ case class GDALRasterSource(source: URI) extends RasterSource with URIRasterSour`
`48`	`48`	`VLMRasterSource(tweaked)`
`49`	`49`	`}`
`50`	`50`
`51`		`- protected def tiffInfo = SimpleRasterInfo.cache.get(source.toASCIIString, _ => SimpleRasterInfo(gdal))`
	`51`	`+ protected def tiffInfo = SimpleRasterInfo(source.toASCIIString, _ => SimpleRasterInfo(gdal))`
`52`	`52`
`53`	`53`	`override def crs: CRS = tiffInfo.crs`
`54`	`54`
Original file line number	Diff line number	Diff line change
`@@ -72,12 +72,15 @@ object ProjectedRasterTile {`
`72`	`72`	`}`
`73`	`73`	`implicit val serializer: CatalystSerializer[ProjectedRasterTile] = new CatalystSerializer[ProjectedRasterTile] {`
`74`	`74`	`override val schema: StructType = StructType(Seq(`
`75`		`- StructField("tile_context", schemaOf[TileContext], false),`
	`75`	`+ StructField("tile_context", schemaOf[TileContext], true),`
`76`	`76`	`StructField("tile", TileType, false))`
`77`	`77`	`)`
`78`	`78`
`79`	`79`	`override protected def to[R](t: ProjectedRasterTile, io: CatalystIO[R]): R = io.create(`
`80`		`- io.to(TileContext(t.extent, t.crs)),`
	`80`	`+ t match {`
	`81`	`+ case _: RasterRefTile => null`
	`82`	`+ case o => io.to(TileContext(o.extent, o.crs))`
	`83`	`+ },`
`81`	`84`	`io.to[Tile](t)(TileUDT.tileSerializer)`
`82`	`85`	`)`
`83`	`86`