Merge branch 'develop' into feature/tile-show

* develop: (26 commits)
  Set next development version.
  Tweaks to get .whl file under the default size limit on PyPi.
  Set version to 0.8.0.
  Removed tile display contrast fiddling.
  Quiet down pip; cache .local.
  Additional CI + Python dependency attempts.
  Final documentation tweaks.
  Move PegdownMarkdownFormatter to docs class
  Edits: languages.pymd & reference.pymd
  Fix setup.py dependency, fix rf_rasterize signature regression in unit test
  Edits: aggregation.pymd, supervised-learning.pymd, unsupervised-learning.pymd, time-series.pymd
  Edits: local-algebra.pymd, nodata-handling.pymd. Rendering tweaks.
  Edits: raster-write.pymd & vector-data.pymd.
  Python SetupTools + TravisCI == Painful Combo.
  Edits: raster-io.md, raster-catalogs.pymd, raster-read.pymd.
  Fixes to PDF generation code.
  Edits: concepts.md & getting-started.md.
  Edits: index.md & description.md
  Tweaks to time-series.pymd to lower memory pressure and remove SQL.
  Tweaked pweave markdown generator.
  ...

# Conflicts:
#	pyrasterframes/src/main/python/pyrasterframes/rf_ipython.py

Author: metasim

.circleci/config.yml

@@ -28,9 +28,7 @@ _save_cache: &save_cache
     - ~/.cache/coursier
     - ~/.ivy2/cache
     - ~/.sbt
-
-before_script:
-  - ulimit -c unlimited -S
+    - ~/.local
 
 jobs:
   test:
@@ -68,6 +66,7 @@ jobs:
           <<: *restore_cache
 
       - run: ulimit -c unlimited -S
+      - run: pip3 install --quiet --user -r pyrasterframes/src/main/python/requirements.txt
       - run:
           command: cat /dev/null | sbt makeSite
           no_output_timeout: 30m
@@ -191,6 +190,7 @@ workflows:
             branches:
               only:
                 - /feature\/.*docs.*/
+                - /docs\/.*/
 
   nightly:
     triggers:

.travis.yml

@@ -10,6 +10,7 @@ cache:
     - $HOME/.ivy2/cache
     - $HOME/.sbt/boot
     - $HOME/.rf_cache
+    - $HOME/.cache/coursier
 
 scala:
   - 2.11.11
@@ -24,7 +25,7 @@ addons:
       - pandoc
 
 install:
-  - pip install rasterio shapely pandas numpy
+  - pip install rasterio shapely pandas numpy pweave
   - wget -O - https://piccolo.link/sbt-1.2.8.tgz | tar xzf -
 
 script:

build.sbt

@@ -21,6 +21,7 @@
 
 addCommandAlias("makeSite", "docs/makeSite")
 addCommandAlias("previewSite", "docs/previewSite")
+addCommandAlias("ghpagesPushSite", "docs/ghpagesPushSite")
 addCommandAlias("console", "datasource/console")
 
 // Prefer our own IntegrationTest config definition, which inherits from Test.
@@ -36,6 +37,7 @@ lazy val root = project
 lazy val `rf-notebook` = project
   .dependsOn(pyrasterframes)
   .enablePlugins(RFAssemblyPlugin, DockerPlugin)
+  .settings(publish / skip := true)
 
 lazy val core = project
   .enablePlugins(BuildInfoPlugin)

core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala

@@ -280,6 +280,11 @@ trait RasterFunctions {
       udf(F.rasterize(_: Geometry, _: Geometry, _: Int, cols, rows)).apply(geometry, bounds, value)
     )
 
+  def rf_rasterize(geometry: Column, bounds: Column, value: Column, cols: Column, rows: Column): TypedColumn[Any, Tile] =
+    withTypedAlias("rf_rasterize", geometry)(
+      udf(F.rasterize).apply(geometry, bounds, value, cols, rows)
+    )
+
   /** Reproject a column of geometry from one CRS to another.
     * @param sourceGeom Geometry column to reproject
     * @param srcCRS Native CRS of `sourceGeom` as a literal

core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/RealizeTile.scala

@@ -42,7 +42,7 @@ import org.locationtech.rasterframes.model.TileContext
 case class RealizeTile(child: Expression) extends UnaryRasterOp with CodegenFallback {
   override def dataType: DataType = TileType
 
-  override def nodeName: String = "rf_realize_tile"
+  override def nodeName: String = "rf_tile"
   implicit val tileSer = TileUDT.tileSerializer
 
   override protected def eval(tile: Tile, ctx: Option[TileContext]): Any =

core/src/main/scala/org/locationtech/rasterframes/util/package.scala

@@ -36,6 +36,7 @@ import org.apache.spark.sql.catalyst.analysis.UnresolvedAttribute
 import org.apache.spark.sql.catalyst.expressions.{Alias, Expression, NamedExpression}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.catalyst.rules.Rule
+import org.apache.spark.sql.functions._
 import org.apache.spark.sql.rf._
 import org.apache.spark.sql.types.StringType
 import org.apache.spark.sql._
@@ -183,6 +184,27 @@ package object util {
     }
   }
 
+  implicit class DFWithPrettyPrint(val df: Dataset[_]) extends AnyVal {
+    def toMarkdown(numRows: Int = 5, truncate: Boolean = false): String = {
+      import df.sqlContext.implicits._
+      val cols = df.columns
+      val header = cols.mkString("| ", " | ", " |") + "\n" + ("|---" * cols.length) + "|\n"
+      val stringifiers = cols
+        .map(c => s"`$c`")
+        .map(c => df.col(c).cast(StringType))
+        .map(c => if (truncate) substring(c, 1, 40) else c)
+      val cat = concat_ws(" | ", stringifiers: _*)
+      val body = df
+        .select(cat).limit(numRows)
+        .as[String]
+        .collect()
+        .map(_.replaceAll("\\[", "\\\\["))
+        .map(_.replace('\n', '↩'))
+        .mkString("| ", " |\n| ", " |")
+      header + body
+    }
+  }
+
   object Shims {
     // GT 1.2.1 to 2.0.0
     def toArrayTile[T <: CellGrid](tile: T): T =

core/src/test/scala/org/locationtech/rasterframes/ExtensionMethodSpec.scala

@@ -67,7 +67,7 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
       // Not sure why implicit resolution isn't handling this properly.
       implicit val enc = Encoders.tuple(crsEncoder, Encoders.STRING, crsEncoder, Encoders.scalaDouble)
       val df = Seq((pe.crs, "fred", pe.crs, 34.0)).toDF("c1", "s", "c2", "n")
-      df.crsColumns.size should be (2)
+      df.crsColumns.size should be(2)
     }
 
     it("should split TileLayout") {
@@ -92,10 +92,10 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
 
     it("should split key") {
       val s1 = SpatialKey(0, 0).subdivide(2)
-      assert(s1 === Seq(SpatialKey(0,0), SpatialKey(1,0), SpatialKey(0,1), SpatialKey(1,1)))
+      assert(s1 === Seq(SpatialKey(0, 0), SpatialKey(1, 0), SpatialKey(0, 1), SpatialKey(1, 1)))
 
       val s2 = SpatialKey(2, 3).subdivide(3)
-      assert(s2 === Seq(SpatialKey(6,9), SpatialKey(7,9), SpatialKey(8,9), SpatialKey(6,10), SpatialKey(7,10), SpatialKey(8,10), SpatialKey(6,11), SpatialKey(7,11), SpatialKey(8,11)))
+      assert(s2 === Seq(SpatialKey(6, 9), SpatialKey(7, 9), SpatialKey(8, 9), SpatialKey(6, 10), SpatialKey(7, 10), SpatialKey(8, 10), SpatialKey(6, 11), SpatialKey(7, 11), SpatialKey(8, 11)))
     }
 
     it("should split TileLayerMetadata[SpatialKey]") {
@@ -107,7 +107,12 @@ class ExtensionMethodSpec extends TestEnvironment with TestData with SubdivideSu
 
       val divided = tlm.subdivide(2)
 
-      assert(divided.tileLayout.tileDimensions === (tileSize/2, tileSize/2))
+      assert(divided.tileLayout.tileDimensions === (tileSize / 2, tileSize / 2))
+    }
+
+    it("should render Markdown") {
+      import org.locationtech.rasterframes.util._
+      rf.toMarkdown().count(_ == '|') shouldBe >=(3 * 5)
     }
   }
 }

docs/README.md

@@ -25,6 +25,21 @@ The build constructs in `<src-root>/docs` are (due to legacy reasons) the top-le
 }
 ```
 
+## Building the docs
+
+To build the static site locally:
+
+    sbt makeSite
+    
+The site will be at `<src-root>/docs/target/site/index.html`.
+    
+    
+To start an interactive server running the docs:
+
+    sbt previewSite
+    
+The sbt server log a message with an address to view the site.
+
 ## Content Development Process
 
 Start with one of the existing files in `<src-root>/pyrasterframes/src/main/python/docs` as a template. [`local-algebra.pymd`](../pyrasterframes/src/main/python/docs/local-algebra.pymd) is a good example. If the content will have code blocks you want evaluated an results injected into the output, use the file extension `.pymd`. If the content doesn't use evaluatable code blocks, use `.md`.
@@ -44,9 +59,9 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     [info] Set current project to RasterFrames (in build file:<src-root>/)
     sbt:RasterFrames>    
     ```  
-2. The first time you check out the code, or whenever RasterFrames code is updated, you need to build the project artifacts so they are available for Pweave.
+2. The first time you check out the code, or whenever RasterFrames code is updated, you need to build the project artifacts so they are available for Pweave. Some docs also refer to test resources, so the easiest way to do it is to run the unit tests.
     ```
-    sbt:RasterFrames> pyrasterframes/package
+    sbt:RasterFrames> pyrasterframes/test
     [info] Compiling 4 Scala sources to <src-root>/core/target/scala-2.11/classes ...
    ... lots of noise ...
    [info] PyRasterFrames assembly written to '<src-root>/pyrasterframes/target/python/deps/jars/pyrasterframes-assembly-0.8.0-SNAPSHOT.jar'
@@ -66,7 +81,7 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     sbt:RasterFrames> pyrasterframes/doc
     ```
     There's a command alias for this last step: `pyDocs`.
-4. To evaluate a single `.pymd` file, you pass the `-s` option and the filename relative to the `python` directory:
+4. To evaluate a single `.pymd` file, you pass the `-s` option and the filename relative to the `pyraterframes/src/main/python` directory. You can also specify the output [format](http://mpastell.com/pweave/formats.html) with the `-f` argument.
     ```
     sbt:RasterFrames> pyrasterframes/pySetup pweave -s docs/getting-started.pymd
     [info] Synchronizing 44 files to '<src-root>/pyrasterframes/target/python'
@@ -82,7 +97,7 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
     Weaved docs/getting-started.pymd to docs/getting-started.md
     ```
 5. The _output_ Markdown files are written to `<src-root>/pyrasterframes/target/python/docs`. _Note_: don't edit any files in the `pyrasterframes/target` directory... they will get overwritten each time `sbt` runs a command.
-6. During content development it's sometimes helpful to see the output rendered as basic HTML. To do this, add the `-d html` option to the pweave command:
+6. During content development it's sometimes helpful to see the output rendered as basic HTML. To do this, add the `-f html` option to the pweave command:
     ```
     sbt:RasterFrames> pyrasterframes/pySetup pweave -f html -s docs/getting-started.pymd
     [info] Synchronizing 54 files to '<src-roog>/pyrasterframes/target/python'
@@ -103,8 +118,11 @@ To set up an environment whereby you can easily test/evaluate your code blocks d
 
 ## Notebooks
 
-The `rf-notebooks` sub-project creates a Docker image with Jupyter Notebooks pre-configured with RasterFrames. Any `.pymd` file under `.../python/docs/` is converted to an evaluated Jupyter Notebook and included as a part of the build (an additional bonus of the Pweave tool). 
+The `rf-notebooks` sub-project creates a Docker image with Jupyter Notebooks pre-configured with RasterFrames. Any `.pymd` file under `.../python/docs/` is converted to an evaluated Jupyter Notebook and included as a part of the build.
 
 ## Submission Process
 
-Submit new and updated documentation as a PR against locationtech/rasterframes. Make sure you've signed the Eclipse Foundation ECA and you ["Signed-off-by:"](https://stackoverflow.com/questions/1962094/what-is-the-sign-off-feature-in-git-for) each commit in the PR. The "Signed-off-by" address needs to be the exact same one as registered with the [Eclipse Foundation](https://wiki.eclipse.org/Development_Resources/Contributing_via_Git).
+Submit new and updated documentation as a PR against locationtech/rasterframes. Make sure you've signed the Eclipse Foundation ECA and you ["Signed-off-by:"](https://stackoverflow.com/questions/1962094/what-is-the-sign-off-feature-in-git-for) each commit in the PR. The "Signed-off-by" email address needs to be the exact same one as registered with the [Eclipse Foundation](https://wiki.eclipse.org/Development_Resources/Contributing_via_Git).
+
+If you are using circle CI, the circle configuration is set up to build the docs with `sbt makeSite` for branch names matching `feature/.*docs.*` or `docs/.*`
+

docs/build.sbt

@@ -8,7 +8,6 @@ makePDF := {
 
   // Get the python source directory configured in the root project.
   val base = (Compile / paradox / sourceDirectories).value.find(_.toString.contains("python")).head
-  println(base)
 
   // Hard coded lacking any simple  way of determining order.
   val files = Seq(
@@ -42,9 +41,9 @@ makePDF := {
   work.mkdirs()
 
   val prepro = files.zipWithIndex.map { case (f, i) ⇒
-    val dest = work / f"$i%02d.md"
+    val dest = work / f"$i%02d-${f.getName}%s"
     // Filter cross links and add a newline
-    (Seq("sed", "-e", """s/@ref:\[\([^]]*\)\](.*)/_\1_/g;s/@@.*//g""", f.toString) #> dest).!
+    (Seq("sed", "-e", """s/@ref://g;s/@@.*//g""", f.toString) #> dest).!
     // Add newline at the end of the file so as to make pandoc happy
     ("echo" #>> dest).!
     ("echo \\pagebreak" #>> dest).!
@@ -56,7 +55,7 @@ makePDF := {
   val header = (Compile / sourceDirectory).value / "latex" / "header.latex"
 
   val args = "pandoc" ::
-    "--from=markdown" ::
+    "--from=markdown+pipe_tables" ::
     "--to=pdf" ::
     "-t" :: "latex" ::
     "-s" ::
@@ -65,7 +64,6 @@ makePDF := {
     "-V" :: "author:Astraea, Inc." ::
     "-V" :: "geometry:margin=0.75in" ::
     "-V" :: "papersize:letter" ::
-    "-V" :: "links-as-notes" ::
     "--include-in-header" :: header.toString ::
     "-o" :: output.toString ::
     prepro.map(_.toString).toList

docs/src/main/latex/header.latex

@@ -1,2 +1,9 @@
 \DeclareUnicodeCharacter{2218}{$\circ$}
-\DeclareUnicodeCharacter{2714}{$\checkmark}
+\DeclareUnicodeCharacter{2714}{$\checkmark$}
+\DeclareUnicodeCharacter{21A9}{$\newline$}
+\hypersetup{
+    colorlinks=true,
+    linkcolor=blue,
+    allbordercolors={0 0 0},
+    pdfborderstyle={/S/U/W 1}
+}