Skip to content

Commit 56e04b9

Browse files
metasimmetasim
committed
Incremental progress commit toward fixing tests.
1 parent e2b5572 commit 56e04b9

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+183
-201
lines changed

‎bench/src/main/scala/org/locationtech/rasterframes/bench/RasterRefBench.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ import org.locationtech.rasterframes._
2929
import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs
3030
import org.locationtech.rasterframes.expressions.transformers.RasterRefToTile
3131
import org.locationtech.rasterframes.model.TileDimensions
32-
import org.locationtech.rasterframes.ref.RasterSource
32+
import org.locationtech.rasterframes.ref.RFRasterSource
3333
import org.openjdk.jmh.annotations._
3434

3535
@BenchmarkMode(Array(Mode.AverageTime))
@@ -43,8 +43,8 @@ class RasterRefBench extends SparkEnv with LazyLogging {
4343

4444
@Setup(Level.Trial)
4545
def setupData(): Unit = {
46-
val r1 = RasterSource(remoteCOGSingleband1)
47-
val r2 = RasterSource(remoteCOGSingleband2)
46+
val r1 = RFRasterSource(remoteCOGSingleband1)
47+
val r2 = RFRasterSource(remoteCOGSingleband2)
4848

4949
singleDF = Seq((r1, r2)).toDF("B1", "B2")
5050
.select(RasterRefToTile(RasterSourceToRasterRefs(Some(TileDimensions(r1.dimensions)), Seq(0), $"B1", $"B2")))

‎bench/src/main/scala/org/locationtech/rasterframes/bench/TileEncodeBench.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import geotrellis.raster.Tile
3030
import geotrellis.vector.Extent
3131
import org.apache.spark.sql.catalyst.InternalRow
3232
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
33-
import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
33+
import org.locationtech.rasterframes.ref.{RasterRef, RFRasterSource}
3434
import org.openjdk.jmh.annotations._
3535

3636
@BenchmarkMode(Array(Mode.AverageTime))
@@ -56,7 +56,7 @@ class TileEncodeBench extends SparkEnv {
5656
case "rasterRef" ⇒
5757
val baseCOG = "https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/149/039/LC08_L1TP_149039_20170411_20170415_01_T1/LC08_L1TP_149039_20170411_20170415_01_T1_B1.TIF"
5858
val extent = Extent(253785.0, 3235185.0, 485115.0, 3471015.0)
59-
tile = RasterRefTile(RasterRef(RasterSource(URI.create(baseCOG)), 0, Some(extent), None))
59+
tile = RasterRefTile(RasterRef(RFRasterSource(URI.create(baseCOG)), 0, Some(extent), None))
6060
case _ ⇒
6161
tile = randomTile(tileSize, tileSize, cellTypeName)
6262
}

‎build.sbt‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@ lazy val core = project
5656
`spray-json`,
5757
geomesa("z3").value,
5858
geomesa("spark-jts").value,
59-
// `geotrellis-contrib-vlm`,
60-
// `geotrellis-contrib-gdal`,
6159
spark("core").value % Provided,
6260
spark("mllib").value % Provided,
6361
spark("sql").value % Provided,
@@ -72,14 +70,27 @@ lazy val core = project
7270
scaffeine,
7371
scalatest
7472
),
73+
/** https://github.com/lucidworks/spark-solr/issues/179
74+
* Thanks @pomadchin for the tip! */
75+
dependencyOverrides ++= {
76+
val deps = Seq(
77+
"com.fasterxml.jackson.core" % "jackson-core" % "2.6.7",
78+
"com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7",
79+
"com.fasterxml.jackson.core" % "jackson-annotations" % "2.6.7"
80+
)
81+
CrossVersion.partialVersion(scalaVersion.value) match {
82+
// if Scala 2.12+ is used
83+
case Some((2, scalaMajor)) if scalaMajor >= 12 => deps
84+
case _ => deps :+ "com.fasterxml.jackson.module" %% "jackson-module-scala" % "2.6.7"
85+
}
86+
},
7587
buildInfoKeys ++= Seq[BuildInfoKey](
76-
moduleName, version, scalaVersion, sbtVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion
88+
version, scalaVersion, rfGeoTrellisVersion, rfGeoMesaVersion, rfSparkVersion
7789
),
7890
buildInfoPackage := "org.locationtech.rasterframes",
7991
buildInfoObject := "RFBuildInfo",
8092
buildInfoOptions := Seq(
8193
BuildInfoOption.ToMap,
82-
BuildInfoOption.BuildTime,
8394
BuildInfoOption.ToJson
8495
)
8596
)

‎core/src/it/scala/org/locationtech/rasterframes/ref/RasterRefIT.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ class RasterRefIT extends TestEnvironment {
3535
def scene(idx: Int) = URI.create(s"https://landsat-pds.s3.us-west-2.amazonaws.com" +
3636
s"/c1/L8/176/039/LC08_L1TP_176039_20190703_20190718_01_T1/LC08_L1TP_176039_20190703_20190718_01_T1_B$idx.TIF")
3737

38-
val redScene = RasterSource(scene(4))
38+
val redScene = RFRasterSource(scene(4))
3939
// [west, south, east, north]
4040
val area = Extent(31.115, 29.963, 31.148, 29.99).reproject(LatLng, redScene.crs)
4141

4242
val red = RasterRef(redScene, 0, Some(area), None)
43-
val green = RasterRef(RasterSource(scene(3)), 0, Some(area), None)
44-
val blue = RasterRef(RasterSource(scene(2)), 0, Some(area), None)
43+
val green = RasterRef(RFRasterSource(scene(3)), 0, Some(area), None)
44+
val blue = RasterRef(RFRasterSource(scene(2)), 0, Some(area), None)
4545

4646
val rf = Seq((red, green, blue)).toDF("red", "green", "blue")
4747
val df = rf.select(

‎core/src/it/scala/org/locationtech/rasterframes/ref/RasterSourceIT.scala‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,10 @@ class RasterSourceIT extends TestEnvironment with TestData {
4444
val bURI = new URI(
4545
"https://s3-us-west-2.amazonaws.com/landsat-pds/c1/L8/016/034/LC08_L1TP_016034_20181003_20181003_01_RT/LC08_L1TP_016034_20181003_20181003_01_RT_B2.TIF")
4646
val red = time("read B4") {
47-
RasterSource(rURI).readAll()
47+
RFRasterSource(rURI).readAll()
4848
}
4949
val blue = time("read B2") {
50-
RasterSource(bURI).readAll()
50+
RFRasterSource(bURI).readAll()
5151
}
5252
time("test empty") {
5353
red should not be empty
@@ -69,47 +69,47 @@ class RasterSourceIT extends TestEnvironment with TestData {
6969

7070

7171
it("should read JPEG2000 scene") {
72-
RasterSource(localSentinel).readAll().flatMap(_.tile.statisticsDouble).size should be(64)
72+
RFRasterSource(localSentinel).readAll().flatMap(_.tile.statisticsDouble).size should be(64)
7373
}
7474

7575
it("should read small MRF scene with one band converted from MODIS HDF") {
7676
val (expectedTileCount, _) = expectedTileCountAndBands(2400, 2400)
77-
RasterSource(modisConvertedMrfPath).readAll().flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
77+
RFRasterSource(modisConvertedMrfPath).readAll().flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
7878
}
7979

8080
it("should read remote HTTP MRF scene") {
8181
val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4)
82-
RasterSource(remoteHttpMrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
82+
RFRasterSource(remoteHttpMrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
8383
}
8484

8585
it("should read remote S3 MRF scene") {
8686
val (expectedTileCount, bands) = expectedTileCountAndBands(6257, 7584, 4)
87-
RasterSource(remoteS3MrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
87+
RFRasterSource(remoteS3MrfPath).readAll(bands = bands).flatMap(_.tile.statisticsDouble).size should be (expectedTileCount)
8888
}
8989
}
9090
} else {
9191
describe("GDAL missing error support") {
9292
it("should throw exception reading JPEG2000 scene") {
9393
intercept[IllegalArgumentException] {
94-
RasterSource(localSentinel)
94+
RFRasterSource(localSentinel)
9595
}
9696
}
9797

9898
it("should throw exception reading MRF scene with one band converted from MODIS HDF") {
9999
intercept[IllegalArgumentException] {
100-
RasterSource(modisConvertedMrfPath)
100+
RFRasterSource(modisConvertedMrfPath)
101101
}
102102
}
103103

104104
it("should throw exception reading remote HTTP MRF scene") {
105105
intercept[IllegalArgumentException] {
106-
RasterSource(remoteHttpMrfPath)
106+
RFRasterSource(remoteHttpMrfPath)
107107
}
108108
}
109109

110110
it("should throw exception reading remote S3 MRF scene") {
111111
intercept[IllegalArgumentException] {
112-
RasterSource(remoteS3MrfPath)
112+
RFRasterSource(remoteS3MrfPath)
113113
}
114114
}
115115
}

‎core/src/main/scala/org/apache/spark/sql/rf/RasterSourceUDT.scala‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ import org.locationtech.rasterframes.encoders.CatalystSerializer._
2727
import org.apache.spark.sql.catalyst.InternalRow
2828
import org.apache.spark.sql.types.{DataType, UDTRegistration, UserDefinedType, _}
2929
import org.locationtech.rasterframes.encoders.CatalystSerializer
30-
import org.locationtech.rasterframes.ref.RasterSource
30+
import org.locationtech.rasterframes.ref.RFRasterSource
3131
import org.locationtech.rasterframes.util.KryoSupport
3232

3333
/**
@@ -36,25 +36,25 @@ import org.locationtech.rasterframes.util.KryoSupport
3636
* @since 9/5/18
3737
*/
3838
@SQLUserDefinedType(udt = classOf[RasterSourceUDT])
39-
class RasterSourceUDT extends UserDefinedType[RasterSource] {
39+
class RasterSourceUDT extends UserDefinedType[RFRasterSource] {
4040
import RasterSourceUDT._
4141
override def typeName = "rf_rastersource"
4242

4343
override def pyUDT: String = "pyrasterframes.rf_types.RasterSourceUDT"
4444

45-
def userClass: Class[RasterSource] = classOf[RasterSource]
45+
def userClass: Class[RFRasterSource] = classOf[RFRasterSource]
4646

47-
override def sqlType: DataType = schemaOf[RasterSource]
47+
override def sqlType: DataType = schemaOf[RFRasterSource]
4848

49-
override def serialize(obj: RasterSource): InternalRow =
49+
override def serialize(obj: RFRasterSource): InternalRow =
5050
Option(obj)
5151
.map(_.toInternalRow)
5252
.orNull
5353

54-
override def deserialize(datum: Any): RasterSource =
54+
override def deserialize(datum: Any): RFRasterSource =
5555
Option(datum)
5656
.collect {
57-
case ir: InternalRow ⇒ ir.to[RasterSource]
57+
case ir: InternalRow ⇒ ir.to[RFRasterSource]
5858
}
5959
.orNull
6060

@@ -66,24 +66,24 @@ class RasterSourceUDT extends UserDefinedType[RasterSource] {
6666
}
6767

6868
object RasterSourceUDT {
69-
UDTRegistration.register(classOf[RasterSource].getName, classOf[RasterSourceUDT].getName)
69+
UDTRegistration.register(classOf[RFRasterSource].getName, classOf[RasterSourceUDT].getName)
7070

7171
/** Deserialize a byte array, also used inside the Python API */
72-
def from(byteArray: Array[Byte]): RasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RasterSource]
72+
def from(byteArray: Array[Byte]): RFRasterSource = CatalystSerializer.CatalystIO.rowIO.create(byteArray).to[RFRasterSource]
7373

74-
implicit val rasterSourceSerializer: CatalystSerializer[RasterSource] = new CatalystSerializer[RasterSource] {
74+
implicit val rasterSourceSerializer: CatalystSerializer[RFRasterSource] = new CatalystSerializer[RFRasterSource] {
7575

7676
override val schema: StructType = StructType(Seq(
7777
StructField("raster_source_kryo", BinaryType, false)
7878
))
7979

80-
override def to[R](t: RasterSource, io: CatalystIO[R]): R = {
80+
override def to[R](t: RFRasterSource, io: CatalystIO[R]): R = {
8181
val buf = KryoSupport.serialize(t)
8282
io.create(buf.array())
8383
}
8484

85-
override def from[R](row: R, io: CatalystIO[R]): RasterSource = {
86-
KryoSupport.deserialize[RasterSource](ByteBuffer.wrap(io.getByteArray(row, 0)))
85+
override def from[R](row: R, io: CatalystIO[R]): RFRasterSource = {
86+
KryoSupport.deserialize[RFRasterSource](ByteBuffer.wrap(io.getByteArray(row, 0)))
8787
}
8888
}
8989
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/DynamicExtractors.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ import org.apache.spark.unsafe.types.UTF8String
3333
import org.locationtech.jts.geom.Envelope
3434
import org.locationtech.rasterframes.encoders.CatalystSerializer._
3535
import org.locationtech.rasterframes.model.{LazyCRS, TileContext}
36-
import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef, RasterSource}
36+
import org.locationtech.rasterframes.ref.{ProjectedRasterLike, RasterRef, RFRasterSource}
3737
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
3838

3939
private[rasterframes]
@@ -71,7 +71,7 @@ object DynamicExtractors {
7171
/** Partial function for pulling a ProjectedRasterLike an input row. */
7272
lazy val projectedRasterLikeExtractor: PartialFunction[DataType, InternalRow ⇒ ProjectedRasterLike] = {
7373
case _: RasterSourceUDT ⇒
74-
(row: InternalRow) => row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer)
74+
(row: InternalRow) => row.to[RFRasterSource](RasterSourceUDT.rasterSourceSerializer)
7575
case t if t.conformsTo[ProjectedRasterTile] =>
7676
(row: InternalRow) => row.to[ProjectedRasterTile]
7777
case t if t.conformsTo[RasterRef] =>
@@ -83,7 +83,7 @@ object DynamicExtractors {
8383
case _: TileUDT =>
8484
(row: InternalRow) => row.to[Tile](TileUDT.tileSerializer)
8585
case _: RasterSourceUDT =>
86-
(row: InternalRow) => row.to[RasterSource](RasterSourceUDT.rasterSourceSerializer)
86+
(row: InternalRow) => row.to[RFRasterSource](RasterSourceUDT.rasterSourceSerializer)
8787
case t if t.conformsTo[RasterRef] ⇒
8888
(row: InternalRow) => row.to[RasterRef]
8989
case t if t.conformsTo[ProjectedRasterTile] =>

‎core/src/main/scala/org/locationtech/rasterframes/expressions/generators/RasterSourceToRasterRefs.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ import org.apache.spark.sql.{Column, TypedColumn}
3131
import org.locationtech.rasterframes.encoders.CatalystSerializer._
3232
import org.locationtech.rasterframes.expressions.generators.RasterSourceToRasterRefs.bandNames
3333
import org.locationtech.rasterframes.model.TileDimensions
34-
import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
34+
import org.locationtech.rasterframes.ref.{RasterRef, RFRasterSource}
3535
import org.locationtech.rasterframes.util._
3636
import org.locationtech.rasterframes.RasterSourceType
3737

@@ -55,7 +55,7 @@ case class RasterSourceToRasterRefs(children: Seq[Expression], bandIndexes: Seq[
5555
name <- bandNames(basename, bandIndexes)
5656
} yield StructField(name, schemaOf[RasterRef], true))
5757

58-
private def band2ref(src: RasterSource, e: Option[(GridBounds[Int], Extent)])(b: Int): RasterRef =
58+
private def band2ref(src: RFRasterSource, e: Option[(GridBounds[Int], Extent)])(b: Int): RasterRef =
5959
if (b < src.bandCount) RasterRef(src, b, e.map(_._2), e.map(_._1)) else null
6060

6161

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/URIToRasterSource.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ import org.apache.spark.sql.types.{DataType, StringType}
3030
import org.apache.spark.sql.{Column, TypedColumn}
3131
import org.apache.spark.unsafe.types.UTF8String
3232
import org.locationtech.rasterframes.RasterSourceType
33-
import org.locationtech.rasterframes.ref.RasterSource
33+
import org.locationtech.rasterframes.ref.RFRasterSource
3434
import org.slf4j.LoggerFactory
3535

3636
/**
@@ -53,12 +53,12 @@ case class URIToRasterSource(override val child: Expression)
5353
override protected def nullSafeEval(input: Any): Any = {
5454
val uriString = input.asInstanceOf[UTF8String].toString
5555
val uri = URI.create(uriString)
56-
val ref = RasterSource(uri)
56+
val ref = RFRasterSource(uri)
5757
RasterSourceType.serialize(ref)
5858
}
5959
}
6060

6161
object URIToRasterSource {
62-
def apply(rasterURI: Column): TypedColumn[Any, RasterSource] =
63-
new Column(new URIToRasterSource(rasterURI.expr)).as[RasterSource]
62+
def apply(rasterURI: Column): TypedColumn[Any, RFRasterSource] =
63+
new Column(new URIToRasterSource(rasterURI.expr)).as[RFRasterSource]
6464
}

‎core/src/main/scala/org/locationtech/rasterframes/expressions/transformers/XZ2Indexer.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import org.locationtech.rasterframes.expressions.DynamicExtractors._
3838
import org.locationtech.rasterframes.expressions.accessors.GetCRS
3939
import org.locationtech.rasterframes.expressions.row
4040
import org.locationtech.rasterframes.jts.ReprojectionTransformer
41-
import org.locationtech.rasterframes.ref.{RasterRef, RasterSource}
41+
import org.locationtech.rasterframes.ref.{RasterRef, RFRasterSource}
4242
import org.locationtech.rasterframes.tiles.ProjectedRasterTile
4343

4444
/**
@@ -88,7 +88,7 @@ case class XZ2Indexer(left: Expression, right: Expression, indexResolution: Shor
8888
case t if t.conformsTo[Envelope] =>
8989
row(leftInput).to[Envelope]
9090
case _: RasterSourceUDT ⇒
91-
row(leftInput).to[RasterSource](RasterSourceUDT.rasterSourceSerializer).extent
91+
row(leftInput).to[RFRasterSource](RasterSourceUDT.rasterSourceSerializer).extent
9292
case t if t.conformsTo[ProjectedRasterTile] =>
9393
row(leftInput).to[ProjectedRasterTile].extent
9494
case t if t.conformsTo[RasterRef] =>

0 commit comments

Comments
 (0)