Skip to content

Commit 89b4b7d

Browse files
committed
[SPARK-52151] DataFrameWriterV2.overwrite should handle condition
### What changes were proposed in this pull request? This PR aims to fix `DataFrameWriterV2.overwrite` to handle the `condition` parameter. ### Why are the changes needed? This was missed at the initial implementation. ### Does this PR introduce _any_ user-facing change? Yes, but this is a bug fix. ### How was this patch tested? Pass the CIs. Apache Iceberg CI will be added soon. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #149 from dongjoon-hyun/SPARK-52151. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent dfe880e commit 89b4b7d

File tree

1 file changed

+8
-2
lines changed

1 file changed

+8
-2
lines changed

Sources/SparkConnect/DataFrameWriterV2.swift

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public actor DataFrameWriterV2: Sendable {
109109
/// output table.
110110
/// - Parameter condition: A filter condition.
111111
public func overwrite(condition: String) async throws {
112-
try await executeWriteOperation(.overwrite)
112+
try await executeWriteOperation(.overwrite, condition)
113113
}
114114

115115
/// Overwrite all partition for which the ``DataFrame`` contains at least one row with the contents
@@ -120,7 +120,10 @@ public actor DataFrameWriterV2: Sendable {
120120
try await executeWriteOperation(.overwritePartitions)
121121
}
122122

123-
private func executeWriteOperation(_ mode: WriteOperationV2.Mode) async throws {
123+
private func executeWriteOperation(
124+
_ mode: WriteOperationV2.Mode,
125+
_ overwriteCondition: String? = nil
126+
) async throws {
124127
var write = WriteOperationV2()
125128

126129
let plan = await self.df.getPlan() as! Plan
@@ -139,6 +142,9 @@ public actor DataFrameWriterV2: Sendable {
139142
for property in self.tableProperties.toStringDictionary() {
140143
write.tableProperties[property.key] = property.value
141144
}
145+
if let overwriteCondition {
146+
write.overwriteCondition = overwriteCondition.toExpression
147+
}
142148
write.mode = mode
143149

144150
var command = Spark_Connect_Command()

0 commit comments

Comments
 (0)