Skip to content

Commit 3bf9704

Browse files
authored
[Spark] Add IdentityColumnsTableFeature (delta-io#2859)
## Description This PR is part of delta-io#1959 In this PR, we introduce the IdentityColumnsTableFeature to test-only so that we can start developing with it. Note, we do not add support to minWriterVersion 6 yet to properties.defaults.minWriterVersion because that will enable the table feature outside of testing. ## How was this patch tested? Existing tests pass. ## Does this PR introduce _any_ user-facing changes? No, this is a test-only change.
1 parent 8b419e7 commit 3bf9704

File tree

3 files changed

+16
-34
lines changed

3 files changed

+16
-34
lines changed

spark/src/main/scala/org/apache/spark/sql/delta/TableFeature.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -354,6 +354,8 @@ object TableFeature {
354354
TestFeatureWithDependency,
355355
TestFeatureWithTransitiveDependency,
356356
TestWriterFeatureWithTransitiveDependency,
357+
// Identity columns are under development and only available in testing.
358+
IdentityColumnsTableFeature,
357359
// managed-commits are under development and only available in testing.
358360
ManagedCommitTableFeature,
359361
InCommitTimestampTableFeature,
@@ -482,6 +484,16 @@ object ColumnMappingTableFeature
482484
}
483485
}
484486

487+
object IdentityColumnsTableFeature
488+
extends LegacyWriterFeature(name = "identityColumns", minWriterVersion = 6)
489+
with FeatureAutomaticallyEnabledByMetadata {
490+
override def metadataRequiresFeatureToBeEnabled(
491+
metadata: Metadata,
492+
spark: SparkSession): Boolean = {
493+
ColumnWithDefaultExprUtils.hasIdentityColumn(metadata.schema)
494+
}
495+
}
496+
485497
object TimestampNTZTableFeature extends ReaderWriterFeature(name = "timestampNtz")
486498
with FeatureAutomaticallyEnabledByMetadata {
487499
override def metadataRequiresFeatureToBeEnabled(

spark/src/test/scala/org/apache/spark/sql/delta/DeltaProtocolVersionSuite.scala

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -2417,40 +2417,6 @@ trait DeltaProtocolVersionSuiteBase extends QueryTest
24172417
}
24182418
}
24192419

2420-
test("can't write to a table with identity columns (legacy protocol)") {
2421-
withTempDir { dir =>
2422-
val writerVersion = 6
2423-
createTableWithProtocol(Protocol(1, writerVersion), dir)
2424-
2425-
checkAnswer(
2426-
sql(s"SELECT * FROM delta.`${dir.getCanonicalPath}`"),
2427-
spark.range(0).toDF)
2428-
assert(intercept[InvalidProtocolVersionException] {
2429-
sql(s"INSERT INTO delta.`${dir.getCanonicalPath}` VALUES (9)")
2430-
}.getMessage.contains(s" and writer version 6"))
2431-
}
2432-
}
2433-
2434-
test("can't write to a table with identity columns (table features)") {
2435-
withTempDir { dir =>
2436-
val featureName = "identityColumns"
2437-
createTableWithProtocol(
2438-
Protocol(
2439-
TABLE_FEATURES_MIN_READER_VERSION,
2440-
TABLE_FEATURES_MIN_WRITER_VERSION,
2441-
readerFeatures = Some(Set.empty),
2442-
writerFeatures = Some(Set(featureName))),
2443-
dir)
2444-
2445-
checkAnswer(
2446-
sql(s"SELECT * FROM delta.`${dir.getCanonicalPath}`"),
2447-
spark.range(0).toDF)
2448-
assert(intercept[DeltaTableFeatureException] {
2449-
sql(s"INSERT INTO delta.`${dir.getCanonicalPath}` VALUES (9)")
2450-
}.getMessage.contains(s"""unsupported by Delta Lake "${io.delta.VERSION}": $featureName"""))
2451-
}
2452-
}
2453-
24542420
def protocolWithFeatures(
24552421
readerFeatures: Seq[TableFeature] = Seq.empty,
24562422
writerFeatures: Seq[TableFeature] = Seq.empty): Protocol = {

spark/src/test/scala/org/apache/spark/sql/delta/DeltaTableFeatureSuite.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ class DeltaTableFeatureSuite
146146
CheckConstraintsTableFeature,
147147
ChangeDataFeedTableFeature,
148148
GeneratedColumnsTableFeature,
149+
IdentityColumnsTableFeature,
149150
TestLegacyWriterFeature,
150151
TestLegacyReaderWriterFeature,
151152
TestRemovableLegacyWriterFeature,
@@ -215,6 +216,7 @@ class DeltaTableFeatureSuite
215216
ChangeDataFeedTableFeature,
216217
CheckConstraintsTableFeature,
217218
GeneratedColumnsTableFeature,
219+
IdentityColumnsTableFeature,
218220
TestLegacyWriterFeature,
219221
TestLegacyReaderWriterFeature,
220222
TestRemovableLegacyWriterFeature,
@@ -243,6 +245,7 @@ class DeltaTableFeatureSuite
243245
CheckConstraintsTableFeature,
244246
ChangeDataFeedTableFeature,
245247
GeneratedColumnsTableFeature,
248+
IdentityColumnsTableFeature,
246249
ColumnMappingTableFeature,
247250
TestLegacyWriterFeature,
248251
TestLegacyReaderWriterFeature,
@@ -257,6 +260,7 @@ class DeltaTableFeatureSuite
257260
CheckConstraintsTableFeature,
258261
ChangeDataFeedTableFeature,
259262
GeneratedColumnsTableFeature,
263+
IdentityColumnsTableFeature,
260264
ColumnMappingTableFeature,
261265
TestLegacyWriterFeature,
262266
TestLegacyReaderWriterFeature,

0 commit comments

Comments
 (0)