Skip to content

Commit 0cb3ea4

Browse files
vpipktvpipkt
committed
Merge remote-tracking branch 'locationtech/develop' into feature/local_is_in
Signed-off-by: Jason T. Brown <[email protected]>
2 parents eb899de + 549c308 commit 0cb3ea4

File tree

29 files changed

+450
-1007
lines changed

29 files changed

+450
-1007
lines changed

‎.travis.yml‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
sudo: false
21
dist: xenial
32
language: python
43

@@ -28,11 +27,10 @@ install:
2827
- pip install rasterio shapely pandas numpy pweave
2928
- wget -O - https://piccolo.link/sbt-1.2.8.tgz | tar xzf -
3029

31-
script:
32-
- sbt/bin/sbt -java-home $JAVA_HOME -batch test
33-
- sbt/bin/sbt -java-home $JAVA_HOME -batch it:test
34-
# - sbt -Dfile.encoding=UTF8 clean coverage test coverageReport
35-
# Tricks to avoid unnecessary cache updates
36-
- find $HOME/.sbt -name "*.lock" | xargs rm
37-
- find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
3830

31+
jobs:
32+
include:
33+
- stage: "Unit Tests"
34+
script: sbt/bin/sbt -java-home $JAVA_HOME -batch test
35+
- stage: "Integration Tests"
36+
script: sbt/bin/sbt -java-home $JAVA_HOME -batch it:test

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ RasterFrames® brings together Earth-observation (EO) data access, cloud computi
66

77
RasterFrames provides a DataFrame-centric view over arbitrary raster data, enabling spatiotemporal queries, map algebra raster operations, and compatibility with the ecosystem of Spark ML algorithms. By using DataFrames as the core cognitive and compute data model, it is able to deliver these features in a form that is both accessible to general analysts and scalable along with the rapidly growing data footprint.
88

9-
<img src="docs/src/main/paradox/RasterFramePipeline.png" width="600px"/>
9+
<img src="pyrasterframes/src/main/python/docs/static/rasterframes-pipeline-nologo.png" width="600px"/>
1010

1111
Please see the [Getting Started](http://rasterframes.io/getting-started.html) section of the Users' Manual to start using RasterFrames.
1212

‎build/circleci/Dockerfile‎

Lines changed: 29 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -6,47 +6,40 @@ ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/
66

77
# most of these libraries required for
88
# python-pip pandoc && pip install setuptools => required for pyrasterframes testing
9-
RUN sudo apt-get update && \
9+
RUN \
10+
sudo apt-get update && \
1011
sudo apt remove \
1112
python python-minimal python2.7 python2.7-minimal \
1213
libpython-stdlib libpython2.7 libpython2.7-minimal libpython2.7-stdlib \
13-
&& sudo apt-get install -y \
14-
pandoc \
15-
wget \
16-
gcc g++ build-essential \
17-
libreadline-gplv2-dev libncursesw5-dev \
18-
libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
19-
liblzma-dev \
20-
libcurl4-gnutls-dev \
21-
libproj-dev \
22-
libgeos-dev \
23-
libhdf4-alt-dev \
24-
bash-completion \
25-
cmake \
26-
imagemagick \
27-
libpng-dev \
28-
libffi-dev \
29-
&& sudo apt autoremove \
30-
&& sudo apt-get clean all
31-
# && sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 1
32-
# todo s
14+
&& \
15+
sudo apt-get install -y \
16+
pandoc wget \
17+
gcc g++ build-essential bash-completion cmake imagemagick \
18+
libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev \
19+
liblzma-dev libcurl4-gnutls-dev libproj-dev libgeos-dev libhdf4-alt-dev libpng-dev libffi-dev \
20+
&& \
21+
sudo apt autoremove && \
22+
sudo apt-get clean all
3323

34-
RUN cd /tmp && \
35-
wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
36-
tar xzf Python-3.7.4.tgz && \
37-
cd Python-3.7.4 && \
38-
./configure --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
39-
make && \
40-
sudo make altinstall && \
41-
rm -rf Python-3.7.4*
24+
RUN \
25+
cd /tmp && \
26+
wget https://www.python.org/ftp/python/3.7.4/Python-3.7.4.tgz && \
27+
tar xzf Python-3.7.4.tgz && \
28+
cd Python-3.7.4 && \
29+
./configure --with-ensurepip=install --prefix=/usr/local --enable-optimization && \
30+
make && \
31+
sudo make altinstall && \
32+
rm -rf Python-3.7.4*
4233

43-
RUN sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
44-
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
45-
sudo python get-pip.py && \
46-
sudo pip3 install setuptools ipython==6.2.1
34+
RUN \
35+
sudo ln -s /usr/local/bin/python3.7 /usr/local/bin/python && \
36+
sudo curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
37+
sudo python get-pip.py && \
38+
sudo pip3 install setuptools ipython==6.2.1
4739

4840
# install OpenJPEG
49-
RUN cd /tmp && \
41+
RUN \
42+
cd /tmp && \
5043
wget https://github.com/uclouvain/openjpeg/archive/v${OPENJPEG_VERSION}.tar.gz && \
5144
tar -xf v${OPENJPEG_VERSION}.tar.gz && \
5245
cd openjpeg-${OPENJPEG_VERSION}/ && \
@@ -58,7 +51,8 @@ RUN cd /tmp && \
5851
cd /tmp && rm -Rf v${OPENJPEG_VERSION}.tar.gz openjpeg*
5952

6053
# Compile and install GDAL with Java bindings
61-
RUN cd /tmp && \
54+
RUN \
55+
cd /tmp && \
6256
wget http://download.osgeo.org/gdal/${GDAL_VERSION}/gdal-${GDAL_VERSION}.tar.gz && \
6357
tar -xf gdal-${GDAL_VERSION}.tar.gz && \
6458
cd gdal-${GDAL_VERSION} && \

‎core/src/main/scala/org/locationtech/rasterframes/RasterFunctions.scala‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,22 @@ trait RasterFunctions {
5959
/** Extracts the bounding box from a RasterSource or ProjectedRasterTile */
6060
def rf_extent(col: Column): TypedColumn[Any, Extent] = GetExtent(col)
6161

62+
/** Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS
63+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
64+
def rf_spatial_index(targetExtent: Column, targetCRS: Column, indexResolution: Short) = XZ2Indexer(targetExtent, targetCRS, indexResolution)
65+
66+
/** Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource and its CRS
67+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
68+
def rf_spatial_index(targetExtent: Column, targetCRS: Column) = XZ2Indexer(targetExtent, targetCRS, 18: Short)
69+
70+
/** Constructs a XZ2 index with level 18 resolution in WGS84 from either a ProjectedRasterTile or RasterSource
71+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
72+
def rf_spatial_index(targetExtent: Column, indexResolution: Short) = XZ2Indexer(targetExtent, indexResolution)
73+
74+
/** Constructs a XZ2 index with level 18 resolution in WGS84 from either a ProjectedRasterTile or RasterSource
75+
* For details: https://www.geomesa.org/documentation/user/datastores/index_overview.html */
76+
def rf_spatial_index(targetExtent: Column) = XZ2Indexer(targetExtent, 18: Short)
77+
6278
/** Extracts the CRS from a RasterSource or ProjectedRasterTile */
6379
def rf_crs(col: Column): TypedColumn[Any, CRS] = GetCRS(col)
6480

‎core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,14 @@ package org.locationtech.rasterframes.expressions
2323

2424
import geotrellis.proj4.CRS
2525
import geotrellis.raster.{CellGrid, Tile}
26+
import geotrellis.vector.Extent
2627
import org.apache.spark.sql.Row
2728
import org.apache.spark.sql.catalyst.InternalRow
29+
import org.apache.spark.sql.jts.JTSTypes
2830
import org.apache.spark.sql.rf.{RasterSourceUDT, TileUDT}
2931
import org.apache.spark.sql.types._
3032
import org.apache.spark.unsafe.types.UTF8String
33+
import org.locationtech.jts.geom.Envelope
3134
import org.locationtech.rasterframes.encoders.CatalystSerializer._
3235
import org.locationtech.rasterframes.model.{LazyCRS, TileContext}
3336
import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef, RasterSource}
@@ -94,6 +97,15 @@ object DynamicExtractors {
9497
(v: Any) => v.asInstanceOf[InternalRow].to[CRS]
9598
}
9699

100+
lazy val extentLikeExtractor: PartialFunction[DataType, Any ⇒ Extent] = {
101+
case t if org.apache.spark.sql.rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
102+
(input: Any) => JTSTypes.GeometryTypeInstance.deserialize(input).getEnvelopeInternal
103+
case t if t.conformsTo[Extent] =>
104+
(input: Any) => input.asInstanceOf[InternalRow].to[Extent]
105+
case t if t.conformsTo[Envelope] =>
106+
(input: Any) => Extent(input.asInstanceOf[InternalRow].to[Envelope])
107+
}
108+
97109
sealed trait TileOrNumberArg
98110
sealed trait NumberArg extends TileOrNumberArg
99111
case class TileArg(tile: Tile, ctx: Option[TileContext]) extends TileOrNumberArg

‎core/src/main/scala/org/locationtech/rasterframes/expressions/package.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -134,6 +134,8 @@ package object expressions {
134134
registry.registerExpression[RenderPNG.RenderCompositePNG]("rf_render_png")
135135
registry.registerExpression[RGBComposite]("rf_rgb_composite")
136136

137+
registry.registerExpression[XZ2Indexer]("rf_spatial_index")
138+
137139
registry.registerExpression[transformers.ReprojectGeometry]("st_reproject")
138140
}
139141
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* This software is licensed under the Apache 2 license, quoted below.
3+
*
4+
* Copyright 2019 Astraea, Inc.
5+
*
6+
* Licensed under the Apache License, Version 2.0 (the "License"); you may not
7+
* use this file except in compliance with the License. You may obtain a copy of
8+
* the License at
9+
*
10+
* [http://www.apache.org/licenses/LICENSE-2.0]
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15+
* License for the specific language governing permissions and limitations under
16+
* the License.
17+
*
18+
* SPDX-License-Identifier: Apache-2.0
19+
*
20+
*/
21+
22+
package org.locationtech.rasterframes.expressions.transformers
23+
24+
import geotrellis.proj4.LatLng
25+
import geotrellis.vector.Extent
26+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
27+
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
28+
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
29+
import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, Expression, ExpressionDescription}
30+
import org.apache.spark.sql.jts.JTSTypes
31+
import org.apache.spark.sql.rf.RasterSourceUDT
32+
import org.apache.spark.sql.types.{DataType, LongType}
33+
import org.apache.spark.sql.{Column, TypedColumn, rf}
34+
import org.locationtech.geomesa.curve.XZ2SFC
35+
import org.locationtech.jts.geom.{Envelope, Geometry}
36+
import org.locationtech.rasterframes.encoders.CatalystSerializer._
37+
import org.locationtech.rasterframes.expressions.DynamicExtractors._
38+
import org.locationtech.rasterframes.expressions.accessors.GetCRS
39+
import org.locationtech.rasterframes.expressions.row
40+
import org.locationtech.rasterframes.jts.ReprojectionTransformer
41+
import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
42+
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
43+
44+
/**
45+
* Constructs a XZ2 index in WGS84 from either a Geometry, Extent, ProjectedRasterTile, or RasterSource
46+
* This function is useful for [range partitioning](http://spark.apache.org/docs/latest/api/python/pyspark.sql.html?highlight=registerjava#pyspark.sql.DataFrame.repartitionByRange).
47+
* Also see: https://www.geomesa.org/documentation/user/datastores/index_overview.html
48+
*
49+
* @param left geometry-like column
50+
* @param right CRS column
51+
* @param indexResolution resolution level of the space filling curve -
52+
* i.e. how many times the space will be recursively quartered
53+
* 1-18 is typical.
54+
*/
55+
@ExpressionDescription(
56+
usage = "_FUNC_(geom, crs) - Constructs a XZ2 index in WGS84/EPSG:4326",
57+
arguments = """
58+
Arguments:
59+
* geom - Geometry or item with Geometry: Extent, ProjectedRasterTile, or RasterSource
60+
* crs - the native CRS of the `geom` column
61+
"""
62+
)
63+
case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Short)
64+
extends BinaryExpression with CodegenFallback {
65+
66+
override def nodeName: String = "rf_spatial_index"
67+
68+
override def dataType: DataType = LongType
69+
70+
override def checkInputDataTypes(): TypeCheckResult = {
71+
if (!extentLikeExtractor.orElse(projectedRasterLikeExtractor).isDefinedAt(left.dataType))
72+
TypeCheckFailure(s"Input type '${left.dataType}' does not look like something with an Extent or something with one.")
73+
else if(!crsExtractor.isDefinedAt(right.dataType))
74+
TypeCheckFailure(s"Input type '${right.dataType}' does not look like something with a CRS.")
75+
else TypeCheckSuccess
76+
}
77+
78+
private lazy val indexer = XZ2SFC(indexResolution)
79+
80+
override protected def nullSafeEval(leftInput: Any, rightInput: Any): Any = {
81+
val crs = crsExtractor(right.dataType)(rightInput)
82+
83+
val coords = left.dataType match {
84+
case t if rf.WithTypeConformity(t).conformsTo(JTSTypes.GeometryTypeInstance) =>
85+
JTSTypes.GeometryTypeInstance.deserialize(leftInput)
86+
case t if t.conformsTo[Extent] =>
87+
row(leftInput).to[Extent]
88+
case t if t.conformsTo[Envelope] =>
89+
row(leftInput).to[Envelope]
90+
case _: RasterSourceUDT ⇒
91+
row(leftInput).to[RasterSource](RasterSourceUDT.rasterSourceSerializer).extent
92+
case t if t.conformsTo[ProjectedRasterTile] =>
93+
row(leftInput).to[ProjectedRasterTile].extent
94+
case t if t.conformsTo[RasterRef] =>
95+
row(leftInput).to[RasterRef].extent
96+
}
97+
98+
// If no transformation is needed then just normalize to an Envelope
99+
val env = if(crs == LatLng) coords match {
100+
case e: Extent => e.jtsEnvelope
101+
case g: Geometry => g.getEnvelopeInternal
102+
case e: Envelope => e
103+
}
104+
// Otherwise convert to geometry, transform, and get envelope
105+
else {
106+
val trans = new ReprojectionTransformer(crs, LatLng)
107+
coords match {
108+
case e: Extent => trans(e).getEnvelopeInternal
109+
case g: Geometry => trans(g).getEnvelopeInternal
110+
case e: Envelope => trans(e).getEnvelopeInternal
111+
}
112+
}
113+
114+
val index = indexer.index(
115+
env.getMinX, env.getMinY, env.getMaxX, env.getMaxY,
116+
lenient = false
117+
)
118+
index
119+
}
120+
}
121+
122+
object XZ2Indexer {
123+
import org.locationtech.rasterframes.encoders.SparkBasicEncoders.longEnc
124+
def apply(targetExtent: Column, targetCRS: Column, indexResolution: Short): TypedColumn[Any, Long] =
125+
new Column(new XZ2Indexer(targetExtent.expr, targetCRS.expr, indexResolution)).as[Long]
126+
def apply(targetExtent: Column, targetCRS: Column): TypedColumn[Any, Long] =
127+
new Column(new XZ2Indexer(targetExtent.expr, targetCRS.expr, 18)).as[Long]
128+
def apply(targetExtent: Column, indexResolution: Short = 18): TypedColumn[Any, Long] =
129+
new Column(new XZ2Indexer(targetExtent.expr, GetCRS(targetExtent.expr), indexResolution)).as[Long]
130+
}

‎core/src/main/scala/org/locationtech/rasterframes/jts/ReprojectionTransformer.scala‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@
2121

2222
package org.locationtech.rasterframes.jts
2323

24-
import org.locationtech.jts.geom.{CoordinateSequence, Geometry}
24+
import org.locationtech.jts.geom.{CoordinateSequence, Envelope, Geometry, GeometryFactory}
2525
import org.locationtech.jts.geom.util.GeometryTransformer
2626
import geotrellis.proj4.CRS
27+
import geotrellis.vector.Extent
2728

2829
/**
2930
* JTS Geometry reprojection transformation routine.
@@ -32,6 +33,12 @@ import geotrellis.proj4.CRS
3233
*/
3334
class ReprojectionTransformer(src: CRS, dst: CRS) extends GeometryTransformer {
3435
lazy val transform = geotrellis.proj4.Transform(src, dst)
36+
@transient
37+
private lazy val gf = new GeometryFactory()
38+
def apply(geometry: Geometry): Geometry = transform(geometry)
39+
def apply(extent: Extent): Geometry = transform(extent.jtsGeom)
40+
def apply(env: Envelope): Geometry = transform(gf.toGeometry(env))
41+
3542
override def transformCoordinates(coords: CoordinateSequence, parent: Geometry): CoordinateSequence = {
3643
val fact = parent.getFactory
3744
val retval = fact.getCoordinateSequenceFactory.create(coords)

‎core/src/test/scala/org/locationtech/rasterframes/GeometryFunctionsSpec.scala‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,7 @@ class GeometryFunctionsSpec extends TestEnvironment with TestData with StandardC
131131
val wm4 = sql("SELECT st_reproject(ll, '+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs', 'EPSG:3857') AS wm4 from geom")
132132
.as[Geometry].first()
133133
wm4 should matchGeom(webMercator, 0.00001)
134-
135-
// TODO: See comment in `org.locationtech.rasterframes.expressions.register` for
136-
// TODO: what needs to happen to support this.
137-
//checkDocs("st_reproject")
134+
checkDocs("st_reproject")
138135
}
139136
}
140137

core/src/test/scala/org/locationtech/rasterframes/expressions/ProjectedLayerMetadataAggregateTest.scala renamed to core/src/test/scala/org/locationtech/rasterframes/expressions/ProjectedLayerMetadataAggregateSpec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.locationtech.rasterframes.encoders.serialized_literal
3030
import org.locationtech.rasterframes.expressions.aggregates.ProjectedLayerMetadataAggregate
3131
import org.locationtech.rasterframes.model.TileDimensions
3232

33-
class ProjectedLayerMetadataAggregateTest extends TestEnvironment {
33+
class ProjectedLayerMetadataAggregateSpec extends TestEnvironment {
3434

3535
import spark.implicits._
3636

0 commit comments

Comments
 (0)