Merge pull request #367 from s22s/fix/366

Added abiltiy to pass a crs specification string to `rf_crs`.

Author: metasim

core/src/main/resources/reference.conf

@@ -14,6 +14,9 @@ vlm.gdal {
       AWS_REQUEST_PAYER = "requester"
       GDAL_DISABLE_READDIR_ON_OPEN = "YES"
       CPL_VSIL_CURL_ALLOWED_EXTENSIONS = ".tif,.tiff,.jp2,.mrf,.idx,.lrc,.mrf.aux.xml,.vrt"
+      GDAL_CACHEMAX = 512
+      GDAL_PAM_ENABLED = "NO"
+      CPL_VSIL_CURL_CHUNK_SIZE = 1000000
     }
     // set this to `false` if CPL_DEBUG is `ON`
     useExceptions = true

core/src/main/scala/org/locationtech/rasterframes/expressions/accessors/GetCRS.scala

@@ -21,33 +21,53 @@
 
 package org.locationtech.rasterframes.expressions.accessors
 
-import org.locationtech.rasterframes.encoders.CatalystSerializer._
-import org.locationtech.rasterframes.encoders.StandardEncoders.crsEncoder
-import org.locationtech.rasterframes.expressions.OnTileContextExpression
 import geotrellis.proj4.CRS
 import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
+import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{TypeCheckFailure, TypeCheckSuccess}
 import org.apache.spark.sql.catalyst.expressions._
 import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.types.DataType
+import org.apache.spark.sql.types.{DataType, StringType}
 import org.apache.spark.sql.{Column, TypedColumn}
-import org.locationtech.rasterframes.model.TileContext
+import org.apache.spark.unsafe.types.UTF8String
+import org.locationtech.rasterframes.encoders.CatalystSerializer._
+import org.locationtech.rasterframes.encoders.StandardEncoders.crsEncoder
+import org.locationtech.rasterframes.expressions.DynamicExtractors.projectedRasterLikeExtractor
+import org.locationtech.rasterframes.model.LazyCRS
 
 /**
  * Expression to extract the CRS out of a RasterRef or ProjectedRasterTile column.
  *
  * @since 9/9/18
  */
 @ExpressionDescription(
-  usage = "_FUNC_(raster) - Fetches the CRS of a ProjectedRasterTile or RasterSource.",
+  usage = "_FUNC_(raster) - Fetches the CRS of a ProjectedRasterTile or RasterSource, or converts a proj4 string column.",
   examples = """
     Examples:
       > SELECT _FUNC_(raster);
          ....
   """)
-case class GetCRS(child: Expression) extends OnTileContextExpression with CodegenFallback {
+case class GetCRS(child: Expression) extends UnaryExpression with CodegenFallback {
   override def dataType: DataType = schemaOf[CRS]
   override def nodeName: String = "rf_crs"
-  override def eval(ctx: TileContext): InternalRow = ctx.crs.toInternalRow
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+    if (child.dataType != StringType && !projectedRasterLikeExtractor.isDefinedAt(child.dataType)) {
+      TypeCheckFailure(s"Input type '${child.dataType}' does not conform to `String` or `ProjectedRasterLike`.")
+    }
+    else TypeCheckSuccess
+  }
+
+  override protected def nullSafeEval(input: Any): Any = {
+    input match {
+      case s: UTF8String => LazyCRS(s.toString).toInternalRow
+      case row: InternalRow ⇒
+        val prl = projectedRasterLikeExtractor(child.dataType)(row)
+        prl.crs.toInternalRow
+      case o ⇒ throw new IllegalArgumentException(s"Unsupported input type: $o")
+    }
+  }
+
 }
 
 object GetCRS {

core/src/main/scala/org/locationtech/rasterframes/model/LazyCRS.scala

@@ -21,10 +21,11 @@
 
 package org.locationtech.rasterframes.model
 
-import LazyCRS.EncodedCRS
 import com.github.blemale.scaffeine.Scaffeine
 import geotrellis.proj4.CRS
 import org.locationtech.proj4j.CoordinateReferenceSystem
+import org.locationtech.rasterframes.encoders.CatalystSerializer
+import org.locationtech.rasterframes.model.LazyCRS.EncodedCRS
 
 class LazyCRS(val encoded: EncodedCRS) extends CRS {
   private lazy val delegate = LazyCRS.cache.get(encoded)
@@ -68,4 +69,6 @@ object LazyCRS {
     else throw new IllegalArgumentException(
       "crs string must be either EPSG code, +proj string, or OGC WKT")
   }
+
+  implicit val crsSererializer: CatalystSerializer[LazyCRS] = CatalystSerializer.crsSerializer.asInstanceOf[CatalystSerializer[LazyCRS]]
 }

core/src/test/scala/org/locationtech/rasterframes/RasterFunctionsSpec.scala

@@ -318,6 +318,11 @@ class RasterFunctionsSpec extends TestEnvironment with RasterMatchers {
       checkDocs("rf_crs")
     }
 
+    it("should parse a CRS from string") {
+      val e = Seq(crs.toProj4String).toDF("crs").select(rf_crs($"crs")).first()
+      e should be (crs)
+    }
+
     it("should get the Geometry of a ProjectedRasterTile") {
       val g = Seq(randPRT).toDF("tile").select(rf_geometry($"tile")).first()
       g should be (extent.jtsGeom)

core/src/test/scala/org/locationtech/rasterframes/TestEnvironment.scala

@@ -20,12 +20,10 @@
  */
 package org.locationtech.rasterframes
 
-import java.io.File
 import java.nio.file.{Files, Path}
 
 import com.typesafe.scalalogging.LazyLogging
 import geotrellis.raster.testkit.RasterMatchers
-import org.apache.hadoop.fs.FileUtil
 import org.apache.spark.sql._
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.types.StructType

docs/src/main/paradox/release-notes.md

@@ -2,6 +2,10 @@
 
 ## 0.8.x
 
+### 0.8.3
+
+* Updated `rf_crs` to accept string columns containing CRS specifications. ([#366](https://github.com/locationtech/rasterframes/issues/366))
+
 ### 0.8.2
 
 * Added ability to pass config options to convenience PySpark session constructor. ([#361](https://github.com/locationtech/rasterframes/issues/361)) 

pyrasterframes/src/main/python/docs/reference.pymd

@@ -98,8 +98,9 @@ Fetches the extent (bounding box or envelope) of a `ProjectedRasterTile` or `Ras
 
     Struct rf_crs(ProjectedRasterTile proj_raster)
     Struct rf_crs(RasterSource proj_raster)
+    Struct rf_crs(String crs_spec)
 
-Fetch CRS structure representing the coordinate reference system of a `ProjectedRasterTile` or `RasterSource` type tile columns.
+Fetch CRS structure representing the coordinate reference system of a `ProjectedRasterTile` or `RasterSource` type tile columns, or from a column of strings in the form supported by @ref:[`rf_mk_crs`](reference.md#rf-mk-crs). 
 
 ### rf_mk_crs