Updated languages.pymd with three variants of same analysis.

Added `doctype` option to pweave setup subcommand.

Author: metasim

core/src/test/scala/Scratch.sc

@@ -105,7 +105,8 @@ class ReprojectGeometrySpec extends TestEnvironment {
         """
           | SELECT st_reproject(ll, llCRS, 'EPSG:3857') as wm2,
           |        st_reproject(wm, 'EPSG:3857', llCRS) as ll2,
-          |        st_reproject(st_reproject(ll, llCRS, '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs'), '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs', 'EPSG:3857') as wm3
+          |        st_reproject(st_reproject(ll, llCRS, '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs'),
+          |        '+proj=sinu +lon_0=0 +x_0=0 +y_0=0 +a=6371007.181 +b=6371007.181 +units=m +no_defs', 'EPSG:3857') as wm3
           | FROM geom
         """.stripMargin).as[(Geometry, Geometry, Geometry)]
 

core/src/test/scala/org/locationtech/rasterframes/ReprojectGeometrySpec.scala

@@ -67,7 +67,7 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     sbt:RasterFrames> pyrasterframes/doc
     ```
     There's a command alias for this last step: `pyDocs`.
-4. To evaluate a single `.pymd` file, you pass the `-f` option and the  filename relative to the `python` direoctry:
+4. To evaluate a single `.pymd` file, you pass the `-f` option and the filename relative to the `python` directory:
     ```
     sbt:RasterFrames> pyrasterframes/pySetup pweave -f docs/getting-started.pymd
     [info] Synchronizing 44 files to '<src-root>/pyrasterframes/target/python'
@@ -81,11 +81,22 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     Processing chunk 1 named None from line 14
     ...
     Weaved docs/getting-started.pymd to docs/getting-started.md
-    [success] Total time: 21 s, completed Jul 5, 2019 12:31:09 PM
-    sbt:RasterFrames>
     ```
 5. The _output_ Markdown files are written to `<src-root>/pyrasterframes/target/python/docs`. _Note_: don't edit any files in the `pyrasterframes/target` directory... they will get overwritten each time `sbt` runs a command.
-6. To build all the documentation and convert to a static html site, run:
+6. During content development it's sometimes helpful to see the output rendered as basic HTML. To do this, add the `-d html` option to the pweave command:
+    ```
+    sbt:RasterFrames> pyrasterframes/pySetup pweave -d html -f docs/getting-started.pymd
+    [info] Synchronizing 54 files to '<src-roog>/pyrasterframes/target/python'
+    [info] Running 'python setup.py pweave -d html -f docs/getting-started.pymd' in '<src-root>/pyrasterframes/target/python'
+    running pweave
+    --------------------------------------------------
+    Running getting-started
+    --------------------------------------------------
+    ...
+    Weaved docs/getting-started.pymd to docs/getting-started.html
+    ```
+    Note: This feature requires `pandoc` to be installed.
+7. To build all the documentation and convert to a static html site, run:
     ```bash
     sbt makeSite
     ``` 

docs/README.md

@@ -0,0 +1,35 @@
+import geotrellis.proj4.LatLng
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.datasource.raster._
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+
+
+implicit val spark = SparkSession.builder().
+  master("local[*]").appName("RasterFrames").getOrCreate().withRasterFrames
+spark.sparkContext.setLogLevel("ERROR")
+
+import spark.implicits._
+
+val modis = spark.read.format("aws-pds-modis-catalog").load()
+
+val red_nir_monthly_2017 = modis
+  .select($"granule_id", month($"acquisition_date") as "month", $"B01" as "red", $"B02" as "nir")
+  .where(year($"acquisition_date") === 2017 && (dayofmonth($"acquisition_date") === 15) && $"granule_id" === "h21v09")
+
+val red_nir_tiles_monthly_2017 = spark.read.raster
+  .fromCatalog(red_nir_monthly_2017, "red", "nir")
+  .load()
+
+val result = red_nir_tiles_monthly_2017
+  .where(st_intersects(
+    st_reproject(rf_geometry($"red"), rf_crs($"red"), LatLng),
+    st_makePoint(34.870605, -4.729727)
+  ))
+  .groupBy("month")
+  .agg(rf_agg_stats(rf_normalized_difference($"nir", $"red")) as "ndvi_stats")
+  .orderBy("month")
+  .select("month", "ndvi_stats.*")
+
+
+result.show()

experimental/src/test/scala/ndvi.sc

@@ -59,7 +59,7 @@ object RFDependenciesPlugin extends AutoPlugin {
     ),
 
     // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py
-    rfSparkVersion := "2.3.2",
+    rfSparkVersion := "2.3.3",
     rfGeoTrellisVersion := "2.2.0",
     rfGeoMesaVersion := "2.2.1",
     dependencyOverrides += "com.azavea.gdal" % "gdal-warp-bindings" % "33.58d4965"

project/RFDependenciesPlugin.scala

@@ -3,43 +3,54 @@
 One of the great powers of RasterFrames, afforded by Spark SQL, is the ability to express computation in multiple programming languages. This manual is centered around Python because that's the most common language used in data science and GIS analytics. However, Scala (the implementation language of RasterFrames) and SQL are also fully supported. Examples in Python can be mechanically translated into the other two languages without much difficulty once the naming conventions are understood. In the sections below we will show the same example program (computing average NDVI per month for a single tile in Tanzania).
 
 ```python, imports, echo=False
-from pyspark.sql.functions import month, dayofmonth, year
+from pyspark.sql.functions import *
 from pyrasterframes.utils import create_rf_spark_session
+
 from pyrasterframes.rasterfunctions import *
 import pyrasterframes.rf_ipython
 import pandas as pd
 import os
+spark = create_rf_spark_session()
 ```
 
 ## Python
 
-Step 1: Load the catalog
+### Step 1: Load the catalog
 
 ```python, step_1_python
 modis = spark.read.format('aws-pds-modis-catalog').load()
 ```
-Step 2: Down-select data by month
+### Step 2: Down-select data by month
 
 ```python, step_2_python
 red_nir_monthly_2017 = modis \
-    .select('granule_id', month('acquisition_date').alias('month'), col('B01').alias('red'), col('B02').alias('nir')) \
+    .select(
+        col('granule_id'), 
+        month('acquisition_date').alias('month'), 
+        col('B01').alias('red'), 
+        col('B02').alias('nir')
+    ) \
     .where((year('acquisition_date') == 2017) & (dayofmonth('acquisition_date') == 15) & (col('granule_id') == 'h21v09'))
 ```
 
-Step 3: Read tiles
+### Step 3: Read tiles
 
 ```python, step_3_python
 red_nir_tiles_monthly_2017 = spark.read.raster(catalog=red_nir_monthly_2017, catalog_col_names=['red', 'nir'])
 ```
 
-Step 4: Compute aggregates
+### Step 4: Compute aggregates
 
 ```python, step_4_python
 result = red_nir_tiles_monthly_2017 \
+    .where(st_intersects(
+        st_reproject(rf_geometry(col('red')), rf_crs(col('red')).crsProj4, rf_mk_crs('EPSG:4326')), 
+        st_makePoint(lit(34.870605), lit(-4.729727)))
+    ) \
     .groupBy('month') \
-    .agg(first('month'), rf_agg_stats(rf_normalized_difference(col('nir'), col('red')).alias('ndvi_stats'))) \
-    .where(st_intersects(st_reproject(rf_geometry(col('red')), rf_crs(col('red')), rf_mk_crs('EPSG:4326')), st_makePoint(34.870605, -4.729727))) \
-    .orderBy(col('month'))
+    .agg(rf_agg_stats(rf_normalized_difference(col('nir'), col('red'))).alias('ndvi_stats')) \
+    .orderBy(col('month')) \
+    .select('month', 'ndvi_stats.*')
 result.show()
 ```
 
@@ -48,18 +59,17 @@ result.show()
 For convenience we're going to evaluate SQL from the Python environment. The SQL fragments should work in the `spark-sql` shell just the same.
 
 ```python, sql_setup
-spark = create_rf_spark_session()
 def sql(stmt):
     return spark.sql(stmt)
 ```
 
-Step 1: Load the catalog
+### Step 1: Load the catalog
 
 ```python, step_1_sql
 sql("CREATE OR REPLACE TEMPORARY VIEW modis USING `aws-pds-modis-catalog`")
 ```
 
-Step 2: Down-select data by month
+### Step 2: Down-select data by month
 
 ```python, step_2_sql
 sql("""
@@ -71,7 +81,7 @@ WHERE year(acquisition_date) = 2017 AND day(acquisition_date) = 15 AND granule_i
 sql('DESCRIBE red_nir_monthly_2017').show() 
 ```
 
-Step 3: Read tiles
+### Step 3: Read tiles
 
 ```python, step_3_sql
 sql("""
@@ -81,7 +91,7 @@ OPTIONS (catalogTable='red_nir_monthly_2017', catalogColumns='red,nir')
 """)
 ```
 
-Step 4: Compute aggregates
+### Step 4: Compute aggregates
 
 ```python, step_4_sql
 sql("""
@@ -97,3 +107,54 @@ SELECT month, ndvi_stats.* FROM (
 
 ## Scala
 
+### Step 1: Load the catalog
+
+```scala
+import geotrellis.proj4.LatLng
+import org.locationtech.rasterframes._
+import org.locationtech.rasterframes.datasource.raster._
+import org.apache.spark.sql._
+import org.apache.spark.sql.functions._
+
+
+implicit val spark = SparkSession.builder()
+  .master("local[*]")
+  .appName("RasterFrames")
+  .withKryoSerialization
+  .getOrCreate()
+  .withRasterFrames
+
+import spark.implicits._
+
+val modis = spark.read.format("aws-pds-modis-catalog").load()
+```
+
+### Step 2: Down-select data by month
+
+```scala
+val red_nir_monthly_2017 = modis
+  .select($"granule_id", month($"acquisition_date") as "month", $"B01" as "red", $"B02" as "nir")
+  .where(year($"acquisition_date") === 2017 && (dayofmonth($"acquisition_date") === 15) && $"granule_id" === "h21v09")
+```
+
+### Step 3: Read tiles
+
+```scala
+val red_nir_tiles_monthly_2017 = spark.read.raster
+  .fromCatalog(red_nir_monthly_2017, "red", "nir")
+  .load()
+```
+
+### Step 4: Compute aggregates
+
+```scala
+val result = red_nir_tiles_monthly_2017
+  .where(st_intersects(
+    st_reproject(rf_geometry($"red"), rf_crs($"red"), LatLng),
+    st_makePoint(34.870605, -4.729727)
+  ))
+  .groupBy("month")
+  .agg(rf_agg_stats(rf_normalized_difference($"nir", $"red")) as "ndvi_stats")
+  .orderBy("month")
+  .select("month", "ndvi_stats.*")
+``` 

pyrasterframes/src/main/python/docs/languages.pymd

@@ -87,10 +87,22 @@ Fetches the extent (bounding box or envelope) of a `ProjectedRasterTile` or `Ras
 
 ### rf_crs
 
-    Struct[String] rf_crs(ProjectedRasterTile proj_raster)
-    Struct[String] rf_crs(RasterSource proj_raster)
+    Struct rf_crs(ProjectedRasterTile proj_raster)
+    Struct rf_crs(RasterSource proj_raster)
 
-Fetch the [proj4](https://proj4.org/) string representation of the coordinate reference system of a `ProjectedRasterTile` or `RasterSource` type tile columns.
+Fetch CRS structure representing the coordinate reference system of a `ProjectedRasterTile` or `RasterSource` type tile columns.
+
+### rf_mk_crs
+
+Construct a CRS structure from one of its string representations. Three froms are supported:
+
+    Struct rf_mk_crs(String crsText)   
+
+* [EPSG code](https://www.epsg-registry.org/): `EPSG:<integer>`
+* [Proj4 string](https://proj.org/): `+proj <proj4 parameters>`
+* [WKT String](http://www.geoapi.org/3.0/javadoc/org/opengis/referencing/doc-files/WKT.html) with embedded EPSG code: `GEOGCS["<name>", <datum>, <prime meridian>, <angular unit> {,<twin axes>} {,<authority>}]` 
+
+Example: `SELECT rf_mk_crs('EPSG:4326')`
 
 ### rf_convert_cell_type
 
@@ -117,7 +129,6 @@ Functions to create a new Tile column, either from scratch or from existing data
 Tile rf_make_zeros_tile(Int tile_columns, Int tile_rows, String cell_type_name)
 ```
 
-
 Create a `tile` of shape `tile_columns` by `tile_rows` full of zeros, with the specified cell type. See function @ref:[`rf_cell_types`](reference.md#rf-cell-types) for valid values. All arguments are literal values and not column expressions.
 
 ### rf_make_ones_tile

pyrasterframes/src/main/python/docs/reference.pymd

@@ -53,6 +53,7 @@ class PweaveDocs(distutils.cmd.Command):
     user_options = [
         # The format is (long option, short option, description).
         ('files=', 'f', 'Specific files to pweave. Defaults to all in `docs` directory.'),
+        ('doctype=', 'd', 'Output format type. Defaults to `markdown`')
     ]
 
     def initialize_options(self):
@@ -62,15 +63,15 @@ def initialize_options(self):
             lambda x: not path.basename(x)[:1] == '_',
             glob(path.join(here, 'docs', '*.pymd'))
         )
+        self.doctype = 'markdown'
 
     def finalize_options(self):
         """Post-process options."""
         import re
         if isinstance(self.files, str):
             self.files = filter(lambda s: len(s) > 0, re.split(',', self.files))
-
-    def doctype(self):
-        return "markdown"
+            if self.doctype is "html":
+                self.doctype = "pandoc2html"
 
     def run(self):
         """Run pweave."""
@@ -83,7 +84,7 @@ def run(self):
             try:
                 pweave.weave(
                     file=str(file),
-                    doctype=self.doctype()
+                    doctype=self.doctype
                 )
             except Exception:
                 print(_divided('%s Failed:' % file))
@@ -92,8 +93,9 @@ def run(self):
 
 
 class PweaveNotebooks(PweaveDocs):
-    def doctype(self):
-        return "notebook"
+    def initialize_options(self):
+        super().initialize_options()
+        self.doctype = 'notebook'
 
 
 setup(

pyrasterframes/src/main/python/setup.py

@@ -7,11 +7,12 @@ services:
       # jupyter notebook port
       - "8888:8888"
       # spark UI ports
-      - "4040:4040"
-      - "4041:4041"
-      - "4042:4042"
-      - "4043:4043"
-      - "4044:4044"
+      - "44040:4040"
+      - "44041:4041"
+      - "44042:4042"
+      - "44043:4043"
+      - "44044:4044"
 # To save locally at './work' from the container:
     volumes:
-      - ../../../src/main/notebooks:/home/jovyan/work
+      - ../../../src/main/notebooks:/home/jovyan/work
+      - ~/.rf_cache:/home/jovyan/.rf_cache