Added DataFrame.display function for passing parameters to rendering normally called by IPython display.

Upgraded suite of Python dependencies.

Author: metasim

.circleci/config.yml

@@ -42,7 +42,8 @@ orbs:
         steps:
           - run:
               name: Install requirements
-              command: python -m pip install --progress-bar=off --user -r pyrasterframes/src/main/python/requirements.txt
+              command: conda install -c conda-forge --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
 
   rasterframes:
     commands:

docs/src/main/paradox/release-notes.md

@@ -5,9 +5,12 @@
 ### 0.9.1
 
 * Upgraded to Spark 2.4.7
+* Added `pyspark.sql.DataFrame.display(num_rows, truncate)` extension method when `rf_ipython` is imported.
 * Added `method_name` parameter to the `rf_resample` method.
    * __BREAKING__: In SQL, the function `rf_resample` now takes 3 arguments. You can use `rf_resample_nearest` with two arguments or refactor to `rf_resample(t, v, "nearest")`.
 * Added resample method parameter to SQL and Python APIs. @ref:[See updated docs](raster-join.md).
+* Upgraded many of the pyrasterframes dependencies, including: 
+  `descartes`, `fiona`, `folium`, `geopandas`, `matplotlib`, `numpy`, `pandas`, `rasterio`, `shapely`
 
 
 ### 0.9.0

pyrasterframes/src/main/python/README.md

@@ -38,6 +38,15 @@ Issue tracking is through [github](https://github.com/locationtech/rasterframes/
 
 Community contributions are always welcome. To get started, please review our [contribution guidelines](https://github.com/locationtech/rasterframes/blob/develop/CONTRIBUTING.md), [code of conduct](https://github.com/locationtech/rasterframes/blob/develop/CODE_OF_CONDUCT.md), and [developer's guide](../../../README.md).  Reach out to us on [gitter][gitter] so the community can help you get started!
 
+## Development environment setup
 
+For best results, we suggest using `conda` and the `conda-forge` channel to install the compiled dependencies before installing the packages in `setup.py`. Assuming you're in the same directory as this file:
+
+    conda create -n rasterframes python==3.7
+    conda install --file ./requirements-condaforge.txt
+    
+Then you can install the source dependencies:
+
+    pip install -e . 
 
 [gitter]: https://gitter.im/locationtech/rasterframes

pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py

@@ -226,7 +226,7 @@ def _folium_map_formatter(map) -> str:
 
 try:
     from IPython import get_ipython
-    from IPython.display import display_png, display_markdown, display
+    from IPython.display import display_png, display_markdown, display_html, display
     # modifications to currently running ipython session, if we are in one; these enable nicer visualization for Pandas
     if get_ipython() is not None:
         import pandas
@@ -259,9 +259,12 @@ def _folium_map_formatter(map) -> str:
         pyspark.sql.DataFrame.showHTML = spark_df_to_html
         Tile.show = plot_tile
 
-        # This is a trick we may have to introduce above.
-        # from IPython.display import display_html
-        # display_html(rf.showHTML(truncate=True), raw=True)
+        def _display(df: pyspark.sql.DataFrame, num_rows: int = 5, truncate: bool = False) -> ():
+            # noinspection PyTypeChecker
+            display_html(spark_df_to_html(df, num_rows, truncate), raw=True)
+
+        pyspark.sql.DataFrame.display = _display
+
 
 except ImportError as e:
     pass

pyrasterframes/src/main/python/requirements-condaforge.txt

@@ -0,0 +1,4 @@
+# These packages should be installed from conda-forge, given their complex binary components.
+gdal==2.4.4
+rasterio[s3]
+rtree

pyrasterframes/src/main/python/requirements.txt

@@ -136,28 +136,28 @@ def dest_file(self, src_file):
 boto3 = 'boto3'
 deprecation = 'deprecation'
 descartes = 'descartes'
-fiona = 'fiona==1.8.6'
+fiona = 'fiona'
 folium = 'folium'
 gdal = 'gdal==2.4.4'
-geopandas = 'geopandas>=0.7'
-ipykernel = 'ipykernel==4.8.0'
-ipython = 'ipython==6.2.1'
+geopandas = 'geopandas'
+ipykernel = 'ipykernel'
+ipython = 'ipython'
 jupyter_client = 'jupyter-client<6.0'  # v6 breaks pweave
-matplotlib = 'matplotlib'
 nbclient = 'nbclient==0.1.0'  # compatible with our pweave => jupyter_client restrictions
-nbconvert = 'nbconvert==5.5.0'
-numpy = 'numpy>=1.17.3,<2.0'
-pandas = 'pandas>=0.25.3,<1.0'
+matplotlib = 'matplotlib'
+nbconvert = 'nbconvert'
+numpy = 'numpy'
+pandas = 'pandas'
 pweave = 'pweave==0.30.3'
 pypandoc = 'pypandoc'
 pyspark = 'pyspark==2.4.7'
-pytest = 'pytest>=4.0.0,<5.0.0'
+pytest = 'pytest'
 pytest_runner = 'pytest-runner'
 pytz = 'pytz'
-rasterio = 'rasterio>=1.0.0'
+rasterio = 'rasterio'
 requests = 'requests'
-setuptools = 'setuptools>=45.2.0'
-shapely = 'Shapely>=1.6.0'
+setuptools = 'setuptools'
+shapely = 'Shapely'
 tabulate = 'tabulate'
 tqdm = 'tqdm'
 utm = 'utm'

pyrasterframes/src/main/python/setup.py

@@ -22,22 +22,25 @@
 
 
 import pyrasterframes
-import pyrasterframes.rf_ipython
-from pyrasterframes.rasterfunctions import *
 from pyrasterframes.rf_types import *
 
-from IPython.display import display_markdown
-from IPython.display import display_html
-
 import numpy as np
 
 from py4j.protocol import Py4JJavaError
+from IPython.testing import globalipapp
 from . import TestEnvironment
 
 class IpythonTests(TestEnvironment):
 
-    def setUp(self):
-        self.create_layer()
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        globalipapp.start_ipython()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        globalipapp.get_ipython().atexit_operations()
+
 
     @skip("Pending fix for issue #458")
     def test_all_nodata_tile(self):
@@ -60,3 +63,23 @@ def test_all_nodata_tile(self):
             self.fail("test_all_nodata_tile failed with Py4JJavaError")
         except:
             self.fail("um")
+
+    def test_display_extension(self):
+        # noinspection PyUnresolvedReferences
+        import pyrasterframes.rf_ipython
+
+        self.create_layer()
+        ip = globalipapp.get_ipython()
+
+        num_rows = 2
+        row_count = 0
+
+        def counter(data, _):
+            nonlocal row_count
+            row_count = data.count('<tr>')
+        ip.mime_renderers['text/html'] = counter
+
+        self.df.display(num_rows=num_rows)
+
+        # Plus one for the header row.
+        self.assertIs(row_count, num_rows+1)