Misc version fiddling.

Author: metasim

datasource/src/main/scala/org/locationtech/rasterframes/datasource/tiles/TilesDataSource.scala

@@ -87,7 +87,6 @@ class TilesDataSource extends DataSourceRegister with CreatableRelationProvider
     val fName = "catalog.csv"
     val hPath = new Path(new Path(pathURI), "_" + fName)
     pipeline
-      .coalesce(1)
       .write
       .option("header", "true")
       .csv(hPath.toString)

project/PythonBuildPlugin.scala

@@ -83,7 +83,7 @@ object PythonBuildPlugin extends AutoPlugin {
     val log = streams.value.log
     val buildDir = (Python / target).value
     val asmbl = (Compile / assembly).value
-    val dest = buildDir / "deps" / "jars" / asmbl.getName
+    val dest = buildDir / "pyrasterframes" / "jars" / asmbl.getName
     IO.copyFile(asmbl, dest)
     log.info(s"PyRasterFrames assembly written to '$dest'")
     dest
@@ -93,7 +93,7 @@ object PythonBuildPlugin extends AutoPlugin {
     val log = streams.value.log
     val buildDir = (Python / target).value
 
-    val jars = (buildDir / "deps" / "jars" ** "*.jar").get()
+    val jars = (buildDir / "pyrasterframes" / "jars" ** "*.jar").get()
     if (jars.size > 1) {
       throw new MessageOnlyException("Two assemblies found in the package. Run 'clean'.\n" + jars.mkString(", "))
     }

project/RFDependenciesPlugin.scala

@@ -69,10 +69,10 @@ object RFDependenciesPlugin extends AutoPlugin {
       "oss-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
       "jitpack" at "https://jitpack.io"
     ),
-    // dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.f746890",
     // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py
     rfSparkVersion := "3.1.2",
-    rfGeoTrellisVersion := "3.6.1-SNAPSHOT",
-    rfGeoMesaVersion := "3.2.0"
+    rfGeoTrellisVersion := "3.6.1",
+    rfGeoMesaVersion := "3.2.0",
+    excludeDependencies += "log4j" % "log4j"
   )
 }

pyrasterframes/src/main/python/MANIFEST.in

@@ -1,3 +1,3 @@
 
 global-exclude *.py[cod] __pycache__ .DS_Store
-recursive-include deps/jars *.jar
+recursive-include pyrasterframes/jars *.jar

pyrasterframes/src/main/python/deps/jars/README.md

@@ -23,9 +23,9 @@
 from pyspark import SparkConf
 import os
 from . import RFContext
-from typing import Union, Dict
+from typing import Union, Dict, Optional
 
-__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version", 'is_notebook', 'gdal_version', 'build_info', 'quiet_logs']
+__all__ = ["create_rf_spark_session", "find_pyrasterframes_jar_dir", "find_pyrasterframes_assembly", "gdal_version", 'gdal_version', 'build_info', 'quiet_logs']
 
 
 def find_pyrasterframes_jar_dir() -> str:
@@ -39,20 +39,10 @@ def find_pyrasterframes_jar_dir() -> str:
     try:
         module_home = find_spec("pyrasterframes").origin
         jar_dir = os.path.join(os.path.dirname(module_home), 'jars')
-    except ImportError:
-        pass
-
-    # Case for when we're running from source build
-    if jar_dir is None or not os.path.exists(jar_dir):
-        def pdir(curr):
-            return os.path.dirname(curr)
-
-        here = pdir(os.path.realpath(__file__))
-        target_dir = pdir(pdir(here))
-        # See if we're running outside of sbt build and adjust
-        if os.path.basename(target_dir) != "target":
-            target_dir = os.path.join(pdir(pdir(target_dir)), 'target')
-        jar_dir = os.path.join(target_dir, 'scala-2.12')
+    except ImportError as e:
+        import logging
+        logging.critical("Error finding runtime JAR directory", exc_info=e)
+        raise e
 
     return os.path.realpath(jar_dir)
 
@@ -62,9 +52,9 @@ def find_pyrasterframes_assembly() -> Union[bytes, str]:
     jarpath = glob.glob(os.path.join(jar_dir, 'pyrasterframes-assembly*.jar'))
 
     if not len(jarpath) == 1:
-        raise RuntimeError("""
-Expected to find exactly one assembly. Found '{}' instead. 
-Try running 'sbt pyrasterframes/clean pyrasterframes/package' first. """.format(jarpath))
+        raise RuntimeError(f""" 
+Expected to find exactly one assembly in '{jar_dir}'. 
+Found '{jarpath}' instead.""")
     return jarpath[0]
 
 
@@ -74,7 +64,7 @@ def quiet_logs(sc):
     logger.LogManager.getLogger("akka").setLevel(logger.Level.ERROR)
 
 
-def create_rf_spark_session(master="local[*]", **kwargs: str) -> SparkSession:
+def create_rf_spark_session(master="local[*]", **kwargs: str) -> Optional[SparkSession]:
     """ Create a SparkSession with pyrasterframes enabled and configured. """
     jar_path = find_pyrasterframes_assembly()
 

pyrasterframes/src/main/python/pyrasterframes/pyproject.toml

@@ -1,4 +1,4 @@
 # These packages should be installed from conda-forge, given their complex binary components.
-gdal==3.1.2
+gdal
 rasterio[s3]
 rtree

pyrasterframes/src/main/python/pyrasterframes/utils.py

@@ -147,7 +147,7 @@ def dest_file(self, src_file):
 matplotlib = 'matplotlib'
 fiona = 'fiona'
 folium = 'folium'
-gdal = 'gdal==3.1.2'
+gdal = 'gdal'
 geopandas = 'geopandas'
 ipykernel = 'ipykernel'
 ipython = 'ipython'
@@ -232,9 +232,6 @@ def dest_file(self, src_file):
         'geomesa_pyspark',
         'pyrasterframes.jars',
     ],
-    package_dir={
-        'pyrasterframes.jars': 'deps/jars'
-    },
     package_data={
         'pyrasterframes.jars': ['*.jar']
     },

pyrasterframes/src/main/python/requirements-condaforge.txt

@@ -1 +1 @@
-ThisBuild / version := "0.10.1-SNAPSHOT"
+ThisBuild / version := "0.10.1"