Skip to content

Commit ac10171

Browse files
kiszkcloud-fan
authored andcommitted
[SPARK-22500][SQL] Fix 64KB JVM bytecode limit problem with cast
## What changes were proposed in this pull request? This PR changes `cast` code generation to place generated code for expression for fields of a structure into separated methods if these size could be large. ## How was this patch tested? Added new test cases into `CastSuite` Author: Kazuaki Ishizaki <[email protected]> Closes #19730 from kiszk/SPARK-22500.
1 parent b96f61b commit ac10171

File tree

2 files changed

+28
-2
lines changed
  • sql/catalyst/src
    • main/scala/org/apache/spark/sql/catalyst/expressions
    • test/scala/org/apache/spark/sql/catalyst/expressions

2 files changed

+28
-2
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1039,13 +1039,21 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String
10391039
}
10401040
}
10411041
"""
1042-
}.mkString("\n")
1042+
}
1043+
val fieldsEvalCodes = if (ctx.INPUT_ROW != null && ctx.currentVars == null) {
1044+
ctx.splitExpressions(
1045+
expressions = fieldsEvalCode,
1046+
funcName = "castStruct",
1047+
arguments = ("InternalRow", tmpRow) :: (rowClass, result) :: Nil)
1048+
} else {
1049+
fieldsEvalCode.mkString("\n")
1050+
}
10431051

10441052
(c, evPrim, evNull) =>
10451053
s"""
10461054
final $rowClass $result = new $rowClass(${fieldsCasts.length});
10471055
final InternalRow $tmpRow = $c;
1048-
$fieldsEvalCode
1056+
$fieldsEvalCodes
10491057
$evPrim = $result;
10501058
"""
10511059
}

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -827,4 +827,22 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
827827

828828
checkEvaluation(cast(Literal.create(input, from), to), input)
829829
}
830+
831+
test("SPARK-22500: cast for struct should not generate codes beyond 64KB") {
832+
val N = 250
833+
834+
val fromInner = new StructType(
835+
(1 to N).map(i => StructField(s"s$i", DoubleType)).toArray)
836+
val toInner = new StructType(
837+
(1 to N).map(i => StructField(s"i$i", IntegerType)).toArray)
838+
val inputInner = Row.fromSeq((1 to N).map(i => i + 0.5))
839+
val outputInner = Row.fromSeq((1 to N))
840+
val fromOuter = new StructType(
841+
(1 to N).map(i => StructField(s"s$i", fromInner)).toArray)
842+
val toOuter = new StructType(
843+
(1 to N).map(i => StructField(s"s$i", toInner)).toArray)
844+
val inputOuter = Row.fromSeq((1 to N).map(_ => inputInner))
845+
val outputOuter = Row.fromSeq((1 to N).map(_ => outputInner))
846+
checkEvaluation(cast(Literal.create(inputOuter, fromOuter), toOuter), outputOuter)
847+
}
830848
}

0 commit comments

Comments
 (0)