Release tweaks.

metasim · metasim · commit f6ed4675e2b1 · 2021-04-01T14:10:58.000-04:00
diff --git a/.java-version b/.java-version
@@ -0,0 +1 @@
+1.8
diff --git a/RELEASE.md b/RELEASE.md
@@ -8,11 +8,10 @@
     a. `clean`  
     b. `test it:test`  
     c. `makeSite`  
-    d. `rf-notebook/publishLocal`    
-    e. `publishSigned` (LocationTech credentials required)  
-    f. `sonatypeReleaseAll`. It can take a while, but should eventually show up [here](https://search.maven.org/search?q=g:org.locationtech.rasterframes).  
-    g. `docs/ghpagesPushSite`  
-    h. `rf-notebook/publish`
+    d. `publishSigned` (LocationTech credentials required)  
+    e. `sonatypeReleaseAll`. It can take a while, but should eventually show up [here](https://search.maven.org/search?q=g:org.locationtech.rasterframes).  
+    f. `docs/ghpagesPushSite`  
+    g. `rf-notebook/publish`
 6. `cd pyrasterframes/target/python/dist`
 7. `python3 -m twine upload pyrasterframes-x.y.z-py2.py3-none-any.whl`
 8. Commit any changes that were necessary.
diff --git a/docs/src/main/paradox/release-notes.md b/docs/src/main/paradox/release-notes.md
@@ -13,6 +13,7 @@
 * Upgraded many of the pyrasterframes dependencies, including: 
   `descartes`, `fiona`, `folium`, `geopandas`, `matplotlib`, `numpy`, `pandas`, `rasterio`, `shapely`
 * Changed `rasterframes.prefer-gdal` configuration parameter to default to `False`, as JVM GeoTIFF performs just as well for COGs as the GDAL one.
+* Fixed [#545](https://github.com/locationtech/rasterframes/issues/545).
 
 ### 0.9.0
 
diff --git a/project/PythonBuildPlugin.scala b/project/PythonBuildPlugin.scala
@@ -101,7 +101,7 @@ object PythonBuildPlugin extends AutoPlugin {
     val retcode = pySetup.toTask(" build bdist_wheel").value
     if(retcode != 0) throw new MessageOnlyException(s"'python setup.py' returned $retcode")
     val whls = (buildDir / "dist" ** "pyrasterframes*.whl").get()
-    require(whls.length == 1, "Running setup.py should have produced a single .whl file. Try running `clean` first.")
+    require(whls.length == 1, s"Running setup.py should have produced a single .whl file. Found $whls")
     log.info(s"Python .whl file written to '${whls.head}'")
     whls.head
   }.dependsOn(pyWhlJar)
diff --git a/project/RFProjectPlugin.scala b/project/RFProjectPlugin.scala
@@ -9,7 +9,7 @@ import xerial.sbt.Sonatype.autoImport._
  */
 object RFProjectPlugin extends AutoPlugin {
   override def trigger: PluginTrigger = allRequirements
-  override def requires = GitPlugin
+  override def requires = GitPlugin && RFDependenciesPlugin
 
   override def projectSettings = Seq(
     organization := "org.locationtech.rasterframes",
@@ -22,6 +22,7 @@ object RFProjectPlugin extends AutoPlugin {
     licenses += ("Apache-2.0", url("https://www.apache.org/licenses/LICENSE-2.0.html")),
     scalaVersion := "2.11.12",
     scalacOptions ++= Seq(
+      "-target:jvm-1.8",
       "-feature",
       "-deprecation",
       "-Ywarn-dead-code",
@@ -30,34 +31,42 @@ object RFProjectPlugin extends AutoPlugin {
     scalacOptions in (Compile, doc) ++= Seq("-no-link-warnings"),
     Compile / console / scalacOptions := Seq("-feature"),
     javacOptions ++= Seq("-source", "1.8", "-target", "1.8"),
+    initialize := {
+      val _ = initialize.value // run the previous initialization
+      val sparkVer = VersionNumber(RFDependenciesPlugin.autoImport.rfSparkVersion.value)
+      if (sparkVer.matchesSemVer(SemanticSelector("<3.0"))) {
+        val curr = VersionNumber(sys.props("java.specification.version"))
+        val req = SemanticSelector("=1.8")
+        assert(curr.matchesSemVer(req), s"Java $req required for $sparkVer. Found $curr.")
+      }
+    },
     cancelable in Global := true,
     publishTo in ThisBuild := sonatypePublishTo.value,
     publishMavenStyle := true,
     publishArtifact in (Compile, packageDoc) := true,
     publishArtifact in Test := false,
     fork in Test := true,
-    javaOptions in Test := Seq("-Xmx1500m", "-XX:+HeapDumpOnOutOfMemoryError",
-      "-XX:HeapDumpPath=/tmp"),
+    javaOptions in Test := Seq("-Xmx1500m", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp"),
     parallelExecution in Test := false,
     testOptions in Test += Tests.Argument("-oDF"),
     developers := List(
       Developer(
         id = "metasim",
         name = "Simeon H.K. Fitch",
         email = "fitch@astraea.earth",
-        url = url("http://www.astraea.earth")
+        url = url("https://github.com/metasim")
       ),
       Developer(
         id = "vpipkt",
         name = "Jason Brown",
         email = "jbrown@astraea.earth",
-        url = url("http://www.astraea.earth")
+        url = url("https://github.com/vpipkt")
       ),
       Developer(
-        id = "mteldridge",
-        name = "Matt Eldridge",
-        email = "meldridge@astraea.earth",
-        url = url("http://www.astraea.earth")
+        id = "echeipesh",
+        name = "Eugene Cheipesh",
+        email = "echeipesh@gmail.com",
+        url = url("https://github.com/echeipesh")
       ),
       Developer(
         id = "bguseman",
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.3.8
+sbt.version=1.4.9
diff --git a/pyrasterframes/src/main/python/docs/aggregation.pymd b/pyrasterframes/src/main/python/docs/aggregation.pymd
@@ -2,7 +2,6 @@
 
 ```python, setup, echo=False
 from pyrasterframes import rf_ipython
-from docs import *
 from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from pyspark.sql import *
diff --git a/pyrasterframes/src/main/python/docs/raster-join.pymd b/pyrasterframes/src/main/python/docs/raster-join.pymd
@@ -7,7 +7,10 @@ import pandas as pd
 from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from pyspark.sql.functions import *
-spark = create_rf_spark_session()
+spark = create_rf_spark_session(**{
+    'spark.driver.memory': '4G',
+    'spark.ui.enabled': 'false'
+})
 
 ```
 
diff --git a/pyrasterframes/src/main/python/docs/raster-read.pymd b/pyrasterframes/src/main/python/docs/raster-read.pymd
@@ -7,7 +7,10 @@ import pandas as pd
 from pyrasterframes.utils import create_rf_spark_session
 from pyrasterframes.rasterfunctions import *
 from pyspark.sql.functions import *
-spark = create_rf_spark_session()
+spark = create_rf_spark_session(**{
+    'spark.driver.memory': '4G',
+    'spark.ui.enabled': 'false'
+})
 ```
 
 RasterFrames registers a DataSource named `raster` that enables reading of GeoTIFFs (and other formats when @ref:[GDAL is installed](getting-started.md#installing-gdal)) from arbitrary URIs. The `raster` DataSource operates on either a single raster file location or another DataFrame, called a _catalog_, containing pointers to many raster file locations.
diff --git a/pyrasterframes/src/main/python/docs/raster-write.pymd b/pyrasterframes/src/main/python/docs/raster-write.pymd
@@ -69,7 +69,7 @@ If there are many _tile_ or projected raster columns in the DataFrame, the GeoTI
 
 * `path`: the path local to the driver where the file will be written
 * `crs`: the PROJ4 string of the CRS the GeoTIFF is to be written in
-* `raster_dimensions`: optional, a tuple of two ints giving the size of the resulting file. If specified, RasterFrames will downsample the data in distributed fashion using bilinear resampling. If not specified, the default is to write the dataframe at full resolution, which can result in an `OutOfMemoryError`.
+* `raster_dimensions`: optional, a tuple of two ints giving the size of the resulting file. If specified, RasterFrames will downsample the data in distributed fashion using bilinear resampling. If not specified, the default is to write the dataframe at full resolution, which can result in an out of memory error.
 
 ### Example 
 
diff --git a/pyrasterframes/src/main/python/docs/unsupervised-learning.pymd b/pyrasterframes/src/main/python/docs/unsupervised-learning.pymd
@@ -130,12 +130,13 @@ retiled = clustered.groupBy('extent', 'crs') \
 Next we will @ref:[write the output to a GeoTiff file](raster-write.md#geotiffs). Doing so in this case works quickly and well for a few specific reasons that may not hold in all cases. We can write the data at full resolution, by omitting the `raster_dimensions` argument, because we know the input raster dimensions are small. Also, the data is all in a single CRS, as we demonstrated above. Because the `catalog_df` is only a single row, we know the output GeoTIFF value at a given location corresponds to a single input. Finally, the `retiled` `DataFrame` only has a single `Tile` column, so the band interpretation is trivial.
 
 ```python, viz
+import rasterio
 output_tif = 'unsupervised.tif'
 
 retiled.write.geotiff(output_tif, crs=crs)
 
 with rasterio.open(output_tif) as src:
     for b in range(1, src.count + 1):
         print("Tags on band", b, src.tags(b))
-    show(src)
+    display(src)
 ```
diff --git a/pyrasterframes/src/main/python/docs/zonal-algebra.pymd b/pyrasterframes/src/main/python/docs/zonal-algebra.pymd
@@ -121,7 +121,7 @@ rf_ndvi = rf_park_tile \
 
 zonal_mean = rf_ndvi \
         .groupby('OBJECTID', 'UNIT_NAME') \
-        .agg(rf_agg_mean('ndvi'))
+        .agg(rf_agg_mean('ndvi_masked'))
 
 zonal_mean
 ```
diff --git a/pyrasterframes/src/main/python/setup.py b/pyrasterframes/src/main/python/setup.py
@@ -37,6 +37,7 @@
     sys.exit(-1)
 
 VERSION = __version__
+print(f"setup.py sees the version as {VERSION}")
 
 here = path.abspath(path.dirname(__file__))
 
@@ -98,8 +99,11 @@ def run(self):
                 'class': PegdownMarkdownFormatter,
                 'description': 'Pegdown compatible markdown'
             }
+        if self.format == 'notebook':
+            # Just convert to an unevaluated notebook.
+            pweave.rcParams["chunk"]["defaultoptions"].update({'evaluate': False})
 
-        for file in sorted(self.files, reverse=True):
+        for file in sorted(self.files, reverse=False):
             name = path.splitext(path.basename(file))[0]
             dest = self.dest_file(file)
 
diff --git a/rf-notebook/build.sbt b/rf-notebook/build.sbt
@@ -30,7 +30,7 @@ Docker / mappings := Def.sequential(
 
     val py = (LocalProject("pyrasterframes") / pyWhl).value
 
-    Def.taskDyn {
+    val _ = Def.taskDyn {
       val withNB = includeNotebooks.value
       if (withNB)
         (LocalProject("pyrasterframes") / pySetup).toTask(" notebooks")
diff --git a/rf-notebook/src/main/docker/Dockerfile b/rf-notebook/src/main/docker/Dockerfile
@@ -22,8 +22,11 @@ ARG APACHE_SPARK_REMOTE_PATH="spark-${APACHE_SPARK_VERSION}/${APACHE_SPARK_FILEN
 RUN \
     cd /tmp && \
     wget --quiet https://archive.apache.org/dist/spark/${APACHE_SPARK_REMOTE_PATH} && \
-    echo "${APACHE_SPARK_CHECKSUM} *${APACHE_SPARK_FILENAME}" | sha512sum -c - && \
-    tar xzf ${APACHE_SPARK_FILENAME} -C /opt --owner root --group root --no-same-owner && \
+    echo "${APACHE_SPARK_CHECKSUM} *${APACHE_SPARK_FILENAME}" | sha512sum -c -
+
+RUN \
+    cd /tmp && \
+    tar xzf ${APACHE_SPARK_FILENAME} -C /usr/local --owner root --group root --no-same-owner && \
     rm ${APACHE_SPARK_FILENAME}
 
 RUN cd /usr/local && ln -s spark-${APACHE_SPARK_VERSION}-bin-hadoop${HADOOP_VERSION} spark
