Skip to content

Commit b479aee

Browse files
richardc-dbrichardc-db
committed
xcompile defaults
1 parent a58bf6c commit b479aee

File tree

9 files changed

+223
-15
lines changed

9 files changed

+223
-15
lines changed

‎build.sbt‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,25 @@ def crossSparkProjectSettings(): Seq[Setting[_]] = getSparkVersion() match {
165165
case SPARK_MASTER_VERSION => Seq()
166166
}
167167

168+
/**
169+
* Java-/Scala-/Uni-Doc settings aren't working yet against Spark Master.
170+
1) delta-spark on Spark Master uses JDK 17. delta-iceberg uses JDK 8 or 11. For some reason,
171+
generating delta-spark unidoc compiles delta-iceberg
172+
2) delta-spark unidoc fails to compile. spark 3.5 is on its classpath. likely due to iceberg
173+
issue above.
174+
*/
175+
def crossDeltaSparkProjectSettings(): Seq[Setting[_]] = getSparkVersion() match {
176+
case LATEST_RELEASED_SPARK_VERSION => Seq(
177+
// Java-/Scala-/Uni-Doc Settings
178+
scalacOptions ++= Seq(
179+
"-P:genjavadoc:strictVisibility=true" // hide package private types and methods in javadoc
180+
),
181+
unidocSourceFilePatterns := Seq(SourceFilePattern("io/delta/tables/", "io/delta/exceptions/"))
182+
)
183+
184+
case SPARK_MASTER_VERSION => Seq()
185+
}
186+
168187
/**
169188
* Note: we cannot access sparkVersion.value here, since that can only be used within a task or
170189
* setting macro.
@@ -315,7 +334,7 @@ lazy val spark = (project in file("spark"))
315334
sparkMimaSettings,
316335
releaseSettings,
317336
crossSparkSettings(),
318-
crossSparkProjectSettings(),
337+
crossDeltaSparkProjectSettings(),
319338
libraryDependencies ++= Seq(
320339
// Adding test classifier seems to break transitive resolution of the core dependencies
321340
"org.apache.spark" %% "spark-hive" % sparkVersion.value % "provided",

‎kernel/kernel-defaults/src/test/scala-spark-3.5/DeltaExcludedBySparkVersionTestMixinShims.scala‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
* Copyright (2024) The Delta Lake Project Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.delta.kernel.defaults
18+
19+
import org.scalatest.funsuite.AnyFunSuite
20+
21+
import io.delta.kernel.defaults.utils.TestUtils
22+
23+
trait DeltaExcludedBySparkVersionTestMixinShims extends AnyFunSuite with TestUtils {
24+
/**
25+
* Tests that are meant for Delta compiled against Spark Latest Release only. Executed since this
26+
* is the Spark Latest Release shim.
27+
*/
28+
protected def testSparkLatestOnly(
29+
testName: String, testTags: org.scalatest.Tag*)
30+
(testFun: => Any)
31+
(implicit pos: org.scalactic.source.Position): Unit = {
32+
test(testName, testTags: _*)(testFun)(pos)
33+
}
34+
35+
/**
36+
* Tests that are meant for Delta compiled against Spark Master Release only. Ignored since this
37+
* is the Spark Latest Release shim.
38+
*/
39+
protected def testSparkMasterOnly(
40+
testName: String, testTags: org.scalatest.Tag*)
41+
(testFun: => Any)
42+
(implicit pos: org.scalactic.source.Position): Unit = {
43+
ignore(testName, testTags: _*)(testFun)(pos)
44+
}
45+
}

‎kernel/kernel-defaults/src/test/scala-spark-3.5/shims/VariantShims.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Copyright (2024) The Delta Lake Project Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.delta.kernel.defaults
18+
19+
import io.delta.kernel.defaults.internal.data.value.DefaultVariantValue
20+
21+
import org.apache.spark.sql.Row
22+
import org.apache.spark.sql.types.DataType
23+
24+
object VariantShims {
25+
26+
/**
27+
* Spark's variant type is implemented for Spark 4.0 and is not implemented in Spark 3.5. Thus,
28+
* any Spark 3.5 DataType cannot be a variant type.
29+
*/
30+
def isVariantType(dt: DataType): Boolean = false
31+
32+
/**
33+
* Converts Spark's variant value to Kernel Default's variant value for testing.
34+
* This method should not be called when depending on Spark 3.5 because Spark 3.5 cannot create
35+
* variants.
36+
*/
37+
def convertToKernelVariant(v: Any): DefaultVariantValue =
38+
throw new UnsupportedOperationException("Not supported")
39+
40+
/**
41+
* Retrieves a Spark variant from a Spark row and converts it to Kernel Default's variant value
42+
* for testing.
43+
*
44+
* Should not be called when testing using Spark 3.5.
45+
*/
46+
def getVariantAndConvertToKernel(r: Row, ordinal: Int): DefaultVariantValue =
47+
throw new UnsupportedOperationException("Not supported")
48+
49+
/**
50+
* Returns Spark's variant type singleton. This should not be called when testing with Spark 3.5.
51+
*/
52+
def getSparkVariantType(): DataType = throw new UnsupportedOperationException("Not supported")
53+
}

‎kernel/kernel-defaults/src/test/scala-spark-master/DeltaExcludedBySparkVersionTestMixinShims.scala‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Copyright (2021) The Delta Lake Project Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.delta.kernel.defaults
18+
19+
import org.scalatest.funsuite.AnyFunSuite
20+
21+
import io.delta.kernel.defaults.utils.TestUtils
22+
23+
trait DeltaExcludedBySparkVersionTestMixinShims extends AnyFunSuite with TestUtils {
24+
25+
/**
26+
* Tests that are meant for Delta compiled against Spark Latest Release only. Ignored since this
27+
* is the Spark Master shim.
28+
*/
29+
protected def testSparkLatestOnly(
30+
testName: String, testTags: org.scalatest.Tag*)
31+
(testFun: => Any)
32+
(implicit pos: org.scalactic.source.Position): Unit = {
33+
ignore(testName + " (Spark Latest Release Only)", testTags: _*)(testFun)(pos)
34+
}
35+
36+
/**
37+
* Tests that are meant for Delta compiled against Spark Master (4.0+). Executed since this is the
38+
* Spark Master shim.
39+
*/
40+
protected def testSparkMasterOnly(
41+
testName: String, testTags: org.scalatest.Tag*)
42+
(testFun: => Any)
43+
(implicit pos: org.scalactic.source.Position): Unit = {
44+
test(testName, testTags: _*)(testFun)(pos)
45+
}
46+
47+
}

‎kernel/kernel-defaults/src/test/scala-spark-master/shims/VariantShims.scala‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (2024) The Delta Lake Project Authors.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.delta.kernel.defaults
18+
19+
import io.delta.kernel.defaults.internal.data.value.DefaultVariantValue
20+
21+
import org.apache.spark.sql.Row
22+
import org.apache.spark.sql.types.{DataType, VariantType}
23+
import org.apache.spark.unsafe.types.VariantVal
24+
25+
object VariantShims {
26+
27+
/** Spark's variant type is only implemented in Spark 4.0 and above. */
28+
def isVariantType(dt: DataType): Boolean = dt.isInstanceOf[VariantType]
29+
30+
/** Converts Spark's variant value to Kernel Default's variant value for testing. */
31+
def convertToKernelVariant(v: Any): DefaultVariantValue = {
32+
val sparkVariant = v.asInstanceOf[VariantVal]
33+
new DefaultVariantValue(sparkVariant.getValue(), sparkVariant.getMetadata())
34+
}
35+
36+
/**
37+
* Retrieves a Spark variant from a Spark row and converts it to Kernel Default's variant value
38+
* for testing.
39+
*/
40+
def getVariantAndConvertToKernel(r: Row, ordinal: Int): DefaultVariantValue = {
41+
val sparkVariant = r.getAs[VariantVal](ordinal)
42+
new DefaultVariantValue(sparkVariant.getValue(), sparkVariant.getMetadata())
43+
}
44+
45+
def getSparkVariantType(): DataType = VariantType
46+
}

‎kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/DeltaTableWritesSuite.scala‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,6 @@ import io.delta.kernel.utils.{CloseableIterable, CloseableIterator}
4242

4343
import java.util.Optional
4444
import scala.collection.JavaConverters._
45-
import scala.collection.immutable.Seq
4645

4746
class DeltaTableWritesSuite extends DeltaTableWriteSuiteBase with ParquetSuiteBase {
4847
val OBJ_MAPPER = new ObjectMapper()
@@ -508,7 +507,7 @@ class DeltaTableWritesSuite extends DeltaTableWriteSuiteBase with ParquetSuiteBa
508507
val parquetAllTypes = goldenTablePath("parquet-all-types")
509508
val schema = removeUnsupportedTypes(tableSchema(parquetAllTypes))
510509

511-
val data = readTableUsingKernel(engine, parquetAllTypes, schema).toSeq
510+
val data = readTableUsingKernel(engine, parquetAllTypes, schema)
512511
val dataWithPartInfo = Seq(Map.empty[String, Literal] -> data)
513512

514513
appendData(engine, tblPath, isNewTable = true, schema, Seq.empty, dataWithPartInfo)
@@ -584,7 +583,7 @@ class DeltaTableWritesSuite extends DeltaTableWriteSuiteBase with ParquetSuiteBa
584583
}.toMap
585584
}
586585

587-
val data = readTableUsingKernel(engine, parquetAllTypes, schema).toSeq
586+
val data = readTableUsingKernel(engine, parquetAllTypes, schema)
588587

589588
// From the above table read data, convert each row as a new batch with partition info
590589
// Take the values of the partitionCols from the data and create a new batch with the

‎kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/ScanSuite.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,10 @@ import io.delta.kernel.internal.util.Utils.singletonCloseableIterator
5151
import io.delta.kernel.internal.util.Utils.toCloseableIterator
5252
import io.delta.kernel.defaults.utils.{ExpressionTestUtils, TestUtils}
5353

54-
class ScanSuite extends AnyFunSuite with TestUtils with ExpressionTestUtils with SQLHelper {
54+
class ScanSuite extends AnyFunSuite
55+
with ExpressionTestUtils
56+
with SQLHelper
57+
with DeltaExcludedBySparkVersionTestMixinShims {
5558

5659
import io.delta.kernel.defaults.ScanSuite._
5760

@@ -1596,7 +1599,7 @@ class ScanSuite extends AnyFunSuite with TestUtils with ExpressionTestUtils with
15961599
}
15971600

15981601
private def testReadWithVariant(testName: String)(df: => DataFrame): Unit = {
1599-
test(testName) {
1602+
testSparkMasterOnly(testName) {
16001603
withTable("test_table") {
16011604
df.write
16021605
.format("delta")

‎kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestRow.scala‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,8 @@ import scala.collection.JavaConverters._
1919
import scala.collection.mutable.{Seq => MutableSeq}
2020
import org.apache.spark.sql.{types => sparktypes}
2121
import org.apache.spark.sql.{Row => SparkRow}
22-
import org.apache.spark.unsafe.types.VariantVal
2322
import io.delta.kernel.data.{ArrayValue, ColumnVector, MapValue, Row}
24-
import io.delta.kernel.defaults.internal.data.value.DefaultVariantValue
23+
import io.delta.kernel.defaults.VariantShims
2524
import io.delta.kernel.types._
2625

2726
import java.sql.Timestamp
@@ -144,9 +143,7 @@ object TestRow {
144143
decodeCellValue(mapType.keyType, k) -> decodeCellValue(mapType.valueType, v)
145144
}
146145
case _: sparktypes.StructType => TestRow(obj.asInstanceOf[SparkRow])
147-
case _: sparktypes.VariantType =>
148-
val sparkVariant = obj.asInstanceOf[VariantVal]
149-
new DefaultVariantValue(sparkVariant.getValue(), sparkVariant.getMetadata())
146+
case t if VariantShims.isVariantType(t) => VariantShims.convertToKernelVariant(obj)
150147
case _ => throw new UnsupportedOperationException("unrecognized data type")
151148
}
152149
}
@@ -180,9 +177,7 @@ object TestRow {
180177
decodeCellValue(mapType.keyType, k) -> decodeCellValue(mapType.valueType, v)
181178
}
182179
case _: sparktypes.StructType => TestRow(row.getStruct(i))
183-
case _: sparktypes.VariantType =>
184-
val sparkVariant = row.getAs[VariantVal](i)
185-
new DefaultVariantValue(sparkVariant.getValue(), sparkVariant.getMetadata())
180+
case t if VariantShims.isVariantType(t) => VariantShims.getVariantAndConvertToKernel(row, i)
186181
case _ => throw new UnsupportedOperationException("unrecognized data type")
187182
}
188183
})

‎kernel/kernel-defaults/src/test/scala/io/delta/kernel/defaults/utils/TestUtils.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import scala.collection.mutable.ArrayBuffer
2424
import io.delta.golden.GoldenTableUtils
2525
import io.delta.kernel.{Scan, Snapshot, Table}
2626
import io.delta.kernel.data.{ColumnVector, ColumnarBatch, FilteredColumnarBatch, MapValue, Row}
27+
import io.delta.kernel.defaults.VariantShims
2728
import io.delta.kernel.defaults.engine.DefaultEngine
2829
import io.delta.kernel.defaults.internal.data.vector.DefaultGenericVector
2930
import io.delta.kernel.engine.Engine
@@ -685,7 +686,7 @@ trait TestUtils extends Assertions with SQLHelper {
685686
field.isNullable
686687
)
687688
}.toSeq)
688-
case VariantType.VARIANT => sparktypes.DataTypes.VariantType
689+
case VariantType.VARIANT => VariantShims.getSparkVariantType()
689690
}
690691
}
691692

0 commit comments

Comments
 (0)