Merge pull request #514 from s22s/feature/ipy-display

DataFrame.display injection

Author: metasim

.circleci/config.yml

@@ -42,7 +42,8 @@ orbs:
         steps:
           - run:
               name: Install requirements
-              command: python -m pip install --progress-bar=off --user -r pyrasterframes/src/main/python/requirements.txt
+              command: /opt/conda/bin/conda install -c conda-forge --yes --file pyrasterframes/src/main/python/requirements-condaforge.txt
+
 
   rasterframes:
     commands:
@@ -117,6 +118,7 @@ jobs:
       - checkout
       - sbt/setup
       - python/setup
+      - python/requirements
       - rasterframes/setup
       - rasterframes/restore-cache
       - sbt/compile

.gitignore

@@ -27,3 +27,4 @@ tour/*.tiff
 scoverage-report*
 
 zz-*
+rf-notebook/src/main/notebooks/.ipython

README.md

@@ -62,6 +62,6 @@ Additional, Python sepcific build instruction may be found at [pyrasterframes/sr
 
 ## Copyright and License
 
-RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2019.
+RasterFrames is released under the Apache 2.0 License, copyright Astraea, Inc. 2017-2020.
 
 

core/src/main/resources/reference.conf

@@ -1,6 +1,6 @@
 rasterframes {
   nominal-tile-size = 256
-  prefer-gdal = true
+  prefer-gdal = false
   showable-tiles = false
   showable-max-cells = 20
   max-truncate-row-element-length = 40

core/src/main/scala/org/locationtech/rasterframes/util/debug/package.scala

@@ -21,10 +21,46 @@
 
 package org.locationtech.rasterframes.util
 
+import java.lang.reflect.{AccessibleObject, Modifier}
+
+import org.apache.spark.Partition
+import org.apache.spark.rdd.RDD
+
+import scala.util.Try
+
 /**
  * Additional debugging routines. No guarantees these are or will remain stable.
  *
  * @since 4/6/18
  */
 package object debug {
+
+  implicit class DescribeablePartition(val p: Partition) extends AnyVal {
+    def describe: String = Try {
+      def acc[A <: AccessibleObject](a: A): A = {
+        a.setAccessible(true); a
+      }
+
+      val getters = p.getClass.getDeclaredMethods
+        .filter(_.getParameterCount == 0)
+        .filter(m ⇒ (m.getModifiers & Modifier.PUBLIC) > 0)
+        .filterNot(_.getName == "hashCode")
+        .map(acc)
+        .map(m ⇒ m.getName + "=" + String.valueOf(m.invoke(p)))
+
+      val fields = p.getClass.getDeclaredFields
+        .filter(f ⇒ (f.getModifiers & Modifier.PUBLIC) > 0)
+        .map(acc)
+        .map(m ⇒ m.getName + "=" + String.valueOf(m.get(p)))
+
+      p.getClass.getSimpleName + "(" + (fields ++ getters).mkString(", ") + ")"
+
+    }.getOrElse(p.toString)
+  }
+
+  implicit class RDDWithPartitionDescribe(val r: RDD[_]) extends AnyVal {
+    def describePartitions: String = r.partitions.map(p ⇒ ("Partition " + p.index) -> p.describe).mkString("\n")
+  }
+
 }
+

datasource/src/it/scala/org/locationtech/rasterframes/datasource/raster/RaterSourceDataSourceIT.scala

@@ -31,7 +31,7 @@ class RaterSourceDataSourceIT extends TestEnvironment with TestData {
       // A regression test.
       val rf = spark.read.raster
         .withSpatialIndex()
-        .load("https://s22s-test-geotiffs.s3.amazonaws.com/water_class/seasonality_90W_50N.tif")
+        .load("https://rasterframes.s3.amazonaws.com/samples/water_class/seasonality_90W_50N.tif")
 
       val target_rf =
         rf.select(rf_extent($"proj_raster").alias("extent"), rf_crs($"proj_raster").alias("crs"), rf_tile($"proj_raster").alias("target"))

datasource/src/main/scala/org/locationtech/rasterframes/datasource/raster/RasterSourceRelation.scala

@@ -153,6 +153,6 @@ case class RasterSourceRelation(
         .repartitionByRange(numParts,$"spatial_index")
       indexed.rdd
     }
-    else df.rdd
+    else df.repartition(numParts).rdd
   }
 }

datasource/src/test/resources/log4j.properties

@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Set everything to be logged to the console
+log4j.rootCategory=INFO, console
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
+
+# Set the default spark-shell log level to WARN. When running the spark-shell, the
+# log level for this class is used to overwrite the root logger's log level, so that
+# the user can have different defaults for the shell and regular Spark apps.
+log4j.logger.org.apache.spark.repl.Main=WARN
+
+
+log4j.logger.org.apache=ERROR
+log4j.logger.com.amazonaws=WARN
+log4j.logger.geotrellis=WARN
+
+# Settings to quiet third party logs that are too verbose
+log4j.logger.org.spark_project.jetty=WARN
+log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
+log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
+log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
+log4j.logger.org.locationtech.rasterframes=DEBUG
+log4j.logger.org.locationtech.rasterframes.ref=DEBUG
+log4j.logger.org.apache.parquet.hadoop.ParquetRecordReader=OFF
+
+# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
+log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
+log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
+
+log4j.logger.org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator=ERROR
+log4j.logger.org.apache.spark.sql.execution.WholeStageCodegenExec=ERROR
+log4j.logger.geotrellis.raster.gdal=ERROR

docs/src/main/paradox/index.md

@@ -29,18 +29,20 @@ The source code can be found on GitHub at [locationtech/rasterframes](https://gi
 
 ## Detailed Contents
 
-@@ toc { depth=4 }
+@@ toc { depth=3 }
 
 @@@ index
-* [Overview](description.md)
-* [Getting Started](getting-started.md)
-* [Concepts](concepts.md)
-* [Raster Data I/O](raster-io.md)
-* [Vector Data](vector-data.md)
-* [Raster Processing](raster-processing.md)
-* [Numpy and Pandas](numpy-pandas.md) 
-* [Scala and SQL](languages.md)
-* [Function Reference](reference.md)
-* [Release Notes](release-notes.md)
+* @ref:[Overview](description.md)
+* @ref:[Getting Started](getting-started.md)
+* @ref:[Concepts](concepts.md)
+* @ref:[Raster Data I/O](raster-io.md)
+* @ref:[Vector Data](vector-data.md)
+* @ref:[Raster Processing](raster-processing.md)
+* @ref:[Machine Learning](machine-learning.md)
+* @ref:[Numpy and Pandas](numpy-pandas.md) 
+* @ref:[IPython Extensions](ipython.md) 
+* @ref:[Scala and SQL](languages.md)
+* @ref:[Function Reference](reference.md)
+* @ref:[Release Notes](release-notes.md)
 @@@
 

docs/src/main/paradox/raster-processing.md

@@ -9,7 +9,6 @@
 * @ref:[Aggregation](aggregation.md)
 * @ref:[Time Series](time-series.md)
 * @ref:[Raster Join](raster-join.md)
-* @ref:[Machine Learning](machine-learning.md)
 
 @@@