bumped dev version

metasim · echeipesh · commit ff00b4137976 · 2022-12-14T13:11:36.000-05:00
Spark 3.2

Lets get it compiled, spark 2 support is well out the window anyway.
diff --git a/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala b/core/src/main/scala/org/apache/spark/sql/rf/VersionShims.scala
@@ -1,21 +1,17 @@
 package org.apache.spark.sql.rf
 
 import java.lang.reflect.Constructor
-
-import org.apache.spark.sql.AnalysisException
-import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
-import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, FunctionRegistryBase}
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.{FUNC_ALIAS, FunctionBuilder}
 import org.apache.spark.sql.catalyst.catalog.CatalogTable
 import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, InvokeLike}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionDescription, ExpressionInfo}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, Expression, ExpressionInfo}
 import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
 import org.apache.spark.sql.execution.datasources.LogicalRelation
 import org.apache.spark.sql.sources.BaseRelation
 import org.apache.spark.sql.types.DataType
 
 import scala.reflect._
-import scala.util.{Failure, Success, Try}
 
 /**
  * Collection of Spark version compatibility adapters.
@@ -27,18 +23,6 @@ object VersionShims {
     val lrClazz = classOf[LogicalRelation]
     val ctor = lrClazz.getConstructors.head.asInstanceOf[Constructor[LogicalRelation]]
     ctor.getParameterTypes.length match {
-      // In Spark 2.1.0 the signature looks like this:
-      //
-      // case class LogicalRelation(
-      //   relation: BaseRelation,
-      //   expectedOutputAttributes: Option[Seq[Attribute]] = None,
-      //   catalogTable: Option[CatalogTable] = None)
-      //   extends LeafNode with MultiInstanceRelation
-      // In Spark 2.2.0 it's like this:
-      // case class LogicalRelation(
-      //   relation: BaseRelation,
-      //   output: Seq[AttributeReference],
-      //   catalogTable: Option[CatalogTable])
       case 3 =>
         val arg2: Seq[AttributeReference] = lr.output
         val arg3: Option[CatalogTable] = lr.catalogTable
@@ -49,14 +33,6 @@ object VersionShims {
           ctor.newInstance(base, arg2, arg3)
         }
 
-      // In Spark 2.3.0 this signature is this:
-      //
-      // case class LogicalRelation(
-      //   relation: BaseRelation,
-      //   output: Seq[AttributeReference],
-      //   catalogTable: Option[CatalogTable],
-      //   override val isStreaming: Boolean)
-      //   extends LeafNode with MultiInstanceRelation {
       case 4 =>
         val arg2: Seq[AttributeReference] = lr.output
         val arg3: Option[CatalogTable] = lr.catalogTable
@@ -75,25 +51,8 @@ object VersionShims {
     val ctor = classOf[Invoke].getConstructors.head
     val TRUE = Boolean.box(true)
     ctor.getParameterTypes.length match {
-      // In Spark 2.1.0 the signature looks like this:
-      //
-      // case class Invoke(
-      //   targetObject: Expression,
-      //   functionName: String,
-      //   dataType: DataType,
-      //   arguments: Seq[Expression] = Nil,
-      //   propagateNull: Boolean = true) extends InvokeLike
       case 5 =>
         ctor.newInstance(targetObject, functionName, dataType, Nil, TRUE).asInstanceOf[InvokeLike]
-      // In spark 2.2.0 the signature looks like this:
-      //
-      // case class Invoke(
-      //   targetObject: Expression,
-      //   functionName: String,
-      //   dataType: DataType,
-      //   arguments: Seq[Expression] = Nil,
-      //   propagateNull: Boolean = true,
-      //   returnNullable : Boolean = true) extends InvokeLike
       case 6 =>
         ctor.newInstance(targetObject, functionName, dataType, Nil, TRUE, TRUE).asInstanceOf[InvokeLike]
 
@@ -125,68 +84,18 @@ object VersionShims {
         }
     }
 
-    // Much of the code herein is copied from org.apache.spark.sql.catalyst.analysis.FunctionRegistry
-    def registerExpression[T <: Expression: ClassTag](name: String): Unit = {
-      val clazz = classTag[T].runtimeClass
-
-      def expressionInfo: ExpressionInfo = {
-        val df = clazz.getAnnotation(classOf[ExpressionDescription])
-        if (df != null) {
-          if (df.extended().isEmpty) {
-            new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.arguments(), df.examples(), df.note(), df.group(), df.since(), df.deprecated())
-          } else {
-            // This exists for the backward compatibility with old `ExpressionDescription`s defining
-            // the extended description in `extended()`.
-            new ExpressionInfo(clazz.getCanonicalName, null, name, df.usage(), df.extended())
-          }
-        } else {
-          new ExpressionInfo(clazz.getCanonicalName, name)
-        }
+    def registerExpression[T <: Expression : ClassTag](
+      name: String,
+      setAlias: Boolean = false,
+      since: Option[String] = None
+    ): (String, (ExpressionInfo, FunctionBuilder)) = {
+      val (expressionInfo, builder) = FunctionRegistryBase.build[T](name, since)
+      val newBuilder = (expressions: Seq[Expression]) => {
+        val expr = builder(expressions)
+        if (setAlias) expr.setTagValue(FUNC_ALIAS, name)
+        expr
       }
-      def findBuilder: FunctionBuilder = {
-        val constructors = clazz.getConstructors
-        // See if we can find a constructor that accepts Seq[Expression]
-        val varargCtor = constructors.find(_.getParameterTypes.toSeq == Seq(classOf[Seq[_]]))
-        val builder = (expressions: Seq[Expression]) => {
-          if (varargCtor.isDefined) {
-            // If there is an apply method that accepts Seq[Expression], use that one.
-            Try(varargCtor.get.newInstance(expressions).asInstanceOf[Expression]) match {
-              case Success(e) => e
-              case Failure(e) =>
-                // the exception is an invocation exception. To get a meaningful message, we need the
-                // cause.
-                throw new AnalysisException(e.getCause.getMessage)
-            }
-          } else {
-            // Otherwise, find a constructor method that matches the number of arguments, and use that.
-            val params = Seq.fill(expressions.size)(classOf[Expression])
-            val f = constructors.find(_.getParameterTypes.toSeq == params).getOrElse {
-              val validParametersCount = constructors
-                .filter(_.getParameterTypes.forall(_ == classOf[Expression]))
-                .map(_.getParameterCount).distinct.sorted
-              val expectedNumberOfParameters = if (validParametersCount.length == 1) {
-                validParametersCount.head.toString
-              } else {
-                validParametersCount.init.mkString("one of ", ", ", " and ") +
-                  validParametersCount.last
-              }
-              throw new AnalysisException(s"Invalid number of arguments for function ${clazz.getSimpleName}. " +
-                s"Expected: $expectedNumberOfParameters; Found: ${params.length}")
-            }
-            Try(f.newInstance(expressions : _*).asInstanceOf[Expression]) match {
-              case Success(e) => e
-              case Failure(e) =>
-                // the exception is an invocation exception. To get a meaningful message, we need the
-                // cause.
-                throw new AnalysisException(e.getCause.getMessage)
-            }
-          }
-        }
-
-        builder
-      }
-
-      registry.registerFunction(FunctionIdentifier(name), expressionInfo, findBuilder)
+      (name, (expressionInfo, newBuilder))
     }
   }
 }
diff --git a/project/RFDependenciesPlugin.scala b/project/RFDependenciesPlugin.scala
@@ -53,10 +53,10 @@ object RFDependenciesPlugin extends AutoPlugin {
     val `spray-json` = "io.spray" %%  "spray-json" % "1.3.6"
     val `scala-logging` = "com.typesafe.scala-logging" %% "scala-logging" % "3.9.4"
     val stac4s = "com.azavea.stac4s" %% "client" % "0.7.2"
-    val sttpCatsCe2 = "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats-ce2" % "3.5.1"
-    val frameless = "org.typelevel" %% "frameless-dataset-spark31" % "0.11.1"
-    val framelessRefined = "org.typelevel" %% "frameless-refined-spark31" % "0.11.1"
-    val `better-files` = "com.github.pathikrit" %% "better-files" % "3.9.1" 
+    val sttpCatsCe2 = "com.softwaremill.sttp.client3" %% "async-http-client-backend-cats-ce2" % "3.3.15"
+    val frameless = "org.typelevel" %% "frameless-dataset-spark31" % "0.12.0"
+    val framelessRefined = "org.typelevel" %% "frameless-refined-spark31" % "0.12.0"
+    val `better-files` = "com.github.pathikrit" %% "better-files" % "3.9.1" % Test
   }
   import autoImport._
 
@@ -70,7 +70,7 @@ object RFDependenciesPlugin extends AutoPlugin {
       "jitpack" at "https://jitpack.io"
     ),
     // NB: Make sure to update the Spark version in pyrasterframes/python/setup.py
-    rfSparkVersion := "3.1.3",
+    rfSparkVersion := "3.2.1",
     rfGeoTrellisVersion := "3.6.1",
     rfGeoMesaVersion := "3.2.0",
     excludeDependencies += "log4j" % "log4j"
diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.5.5
+sbt.version=1.6.2
