Merge pull request #3 from bguseman/develop

Cleanup and build changes for pyrasterframes

Author: metasim

pyrasterframes/build.sbt

@@ -1,7 +1,10 @@
 import scala.sys.process.Process
+import sbt.Keys.`package`
 
 enablePlugins(SparkPackagePlugin, AssemblyPlugin)
 
+lazy val pyZip = taskKey[File]("Create a minimal pyrasterframes zip distribution.")
+
 val pysparkCmd = taskKey[Unit]("Builds pyspark package and emits command string for running pyspark with package")
 
 lazy val pyTest = taskKey[Unit]("Run pyrasterframes tests.")
@@ -61,6 +64,18 @@ spPublishLocal := {
   spPublishLocal.value
 }
 
+pyZip := {
+  val jar = (`package` in Compile).value
+  val license = baseDirectory.value / "python" / "LICENSE.md"
+  val pyDir = baseDirectory.value / "python" / "pyrasterframes"
+  val files = (IO.listFiles(pyDir, GlobFilter("*.py") | GlobFilter("*.rst")) ++ Seq(jar, license))
+    .map(f => (f, "pyrasterframes/" + f.getName))
+  val zipFile = target.value / "python-dist" / "pyrasterframes.zip"
+  IO.zip(files, zipFile)
+
+  zipFile
+}
+
 pysparkCmd := {
   val _ = spPublishLocal.value
   val id = (projectID in spPublishLocal).value

pyrasterframes/python/LICENSE pyrasterframes/python/LICENSE.mdpyrasterframes/python/LICENSE renamed to pyrasterframes/python/LICENSE.md

@@ -0,0 +1,15 @@
+PyRasterFrames
+--------------
+
+PyRasterFrames provides a Python API for RasterFrames!
+
+To initialize PyRasterFrames:
+
+    >>> from pyrasterframes import *
+    >>> spark = SparkSession.builder \
+    ...     .master("local[*]") \
+    ...     .appName("Using RasterFrames") \
+    ...     .config("spark.some.config.option", "some-value") \
+    ...     .getOrCreate() \
+    ...     .withRasterFrames()
+

pyrasterframes/python/pyrasterframes/README.rst

@@ -24,6 +24,7 @@ def _rf_init(spark_session):
     if not hasattr(spark_session, "rasterframes"):
         spark_session.rasterframes = RFContext(spark_session)
         spark_session.sparkContext._rf_context = spark_session.rasterframes
+
     return spark_session
 
 
@@ -45,7 +46,9 @@ def _convertDF(df, sp_key = None, metadata = None):
 
 
 _prevFJ = UserDefinedType.fromJson
-def _fromJson(json_val):
+# this classmethod annotation is needed for 2.7
+@classmethod
+def _fromJson(cls, json_val):
     if str(json_val['class']).startswith('org.apache.spark.sql.jts'):
         json_val['pyClass'] = 'pyrasterframes.GeometryUDT'
 
@@ -66,3 +69,5 @@ def _fromJson(json_val):
 # If you don't have Python support, you will get it anyway
 UserDefinedType.fromJson = _fromJson
 
+
+

pyrasterframes/python/pyrasterframes/__init__.py

@@ -0,0 +1,13 @@
+"""
+This module contains access to the jvm SparkContext with RasterFrame support.
+"""
+
+from pyspark import SparkContext
+
+def _checked_context():
+    """ Get the active SparkContext and throw an error if it is not enabled for RasterFrames."""
+    sc = SparkContext._active_spark_context
+    if not hasattr(sc, '_rf_context'):
+        raise AttributeError(
+            "RasterFrames have not been enabled for the active session. Call 'SparkSession.withRasterFrames()'.")
+    return sc._rf_context._jrfctx

pyrasterframes/python/pyrasterframes/context.py

@@ -6,19 +6,9 @@
 
 
 from __future__ import absolute_import
-from pyspark import SparkContext
 from pyspark.sql.types import *
 from pyspark.sql.column import Column, _to_java_column
-
-
-def _checked_context():
-    """ Get the active SparkContext and throw an error if it is not enabled for RasterFrames."""
-    sc = SparkContext._active_spark_context
-    if not hasattr(sc, '_rf_context'):
-        print(vars(sc))
-        raise AttributeError(
-            "RasterFrames have not been enabled for the active session. Call 'SparkSession.withRasterFrames()'.")
-    return sc._rf_context._jrfctx
+from pyrasterframes.context import _checked_context
 
 
 def _celltype(cellTypeStr):

pyrasterframes/python/pyrasterframes/rasterfunctions.py

@@ -11,7 +11,7 @@ class here provides the PyRasterFrames entry point.
 from pyspark.sql.types import *
 from pyspark.ml.wrapper import JavaTransformer
 from pyspark.ml.util import JavaMLReadable, JavaMLWritable
-from pyrasterframes.rasterfunctions import _checked_context
+from pyrasterframes.context import _checked_context
 
 __all__ = ['RFContext', 'RasterFrame', 'TileUDT', 'GeometryUDT', 'TileExploder']
 

pyrasterframes/python/pyrasterframes/types.py

@@ -1,7 +1,11 @@
+"""
+The operations in this file are designed for development and testing only.
+"""
+
+
 from setuptools import setup, find_packages
 import distutils.log
 import importlib
-from pathlib import Path
 
 
 def _extract_module(mod):
@@ -24,19 +28,21 @@ class ExampleCommand(distutils.cmd.Command):
     ]
 
     def initialize_options(self):
+        from pathlib import Path
         """Set default values for options."""
         # Each user option must be listed here with their default value.
         self.examples = filter(lambda x: not x.name.startswith('_'),
                                list(Path('./examples').resolve().glob('*.py')))
 
     def _check_ex_path(self, ex):
+        from pathlib import Path
         file = Path(ex)
         if not file.suffix:
             file = file.with_suffix('.py')
         file = (Path('./examples') / file).resolve()
 
         assert file.is_file(), ('Invalid example %s' % file)
-        return file.with_suffix('')
+        return file
 
     def finalize_options(self):
         """Post-process options."""
@@ -59,6 +65,40 @@ def run(self):
                 print(traceback.format_exc())
 
 
+class ZipCommand(distutils.cmd.Command):
+    """A custom command to create a minimal zip distribution."""
+
+    description = 'create a minimal pyrasterframes source zip file'
+    user_options = []
+
+    def initialize_options(self):
+        pass
+
+    def finalize_options(self):
+        pass
+
+    def run(self):
+        """Create the zip."""
+        import zipfile
+        from pathlib import Path
+        import os
+        zfile = 'pyrasterframes.zip'
+
+        if os.path.isfile(zfile):
+            os.remove(zfile)
+        with zipfile.ZipFile(zfile, 'w') as przip:
+            przip.write('pyrasterframes')
+            # Bring in source files and readme
+            patterns = ['*.py', '*.rst', '*.jar']
+            root = Path('.').resolve()
+            for pattern in patterns:
+                for file in list(root.glob('pyrasterframes/' + pattern)):
+                    przip.write(str(file.relative_to(root)))
+            # Put a copy of the license in the zip
+            przip.write('LICENSE.md', 'pyrasterframes/LICENSE.md')
+
+
+
 
 
 with open('README.rst') as f:
@@ -75,31 +115,33 @@ def run(self):
     author='Simeon H.K. Fitch',
     author_email='[email protected]',
     license='Apache 2',
-    setup_requires=['pytest-runner', pyspark_ver],
+    setup_requires=['pytest-runner', pyspark_ver, 'pathlib'],
     install_requires=[
-        pyspark_ver,
+        # pyspark_ver,
+        # 'pathlib'
     ],
     tests_require=[
         pyspark_ver,
         'pytest==3.4.2'
     ],
     test_suite="pytest-runner",
-    packages=['.'] + find_packages(exclude=['tests']),
+    packages=find_packages(exclude=['tests', 'examples']),
     include_package_data=True,
-    package_data={'.':['LICENSE', 'static/*']},
-    exclude_package_data={'.':['setup.cfg']},
+    package_data={'.':['LICENSE.md'], 'pyrasterframes':['*.jar']},
+    exclude_package_data={'.':['setup.*', 'README.*']},
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Environment :: Other Environment',
         'License :: OSI Approved :: Apache Software License',
         'Natural Language :: English',
         'Operating System :: Unix',
-        'Programming Language :: Python :: 3 :: Only',
+        'Programming Language :: Python',
         'Topic :: Software Development :: Libraries'
     ],
     zip_safe=False,
     cmdclass={
-        'examples': ExampleCommand
+        'examples': ExampleCommand,
+        'minzip': ZipCommand
     }
     # entry_points={
     #     "console_scripts": ['pyrasterframes=pyrasterframes:console']

pyrasterframes/python/setup.py

@@ -62,7 +62,7 @@ class PyRFContext(implicit sparkSession: SparkSession) extends RasterFunctions {
   /** DESERIALIZATION **/
 
   def generateTile(cellType: String, cols: Int, rows: Int, bytes: Array[Byte]): ArrayTile = {
-    ArrayTile.fromBytes(bytes, PyRFContext.this.cellType(cellType), cols, rows)
+    ArrayTile.fromBytes(bytes, this.cellType(cellType), cols, rows)
   }
 
   def generateGeometry(obj: Array[Byte]): Geometry = {