From 08f0d97d564c60f45469040d89f9dfe390e88083 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Wed, 14 May 2025 18:51:11 -0700 Subject: [PATCH] [SPARK-52151] `DataFrameWriterV2.overwrite` should handle `condition` --- Sources/SparkConnect/DataFrameWriterV2.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Sources/SparkConnect/DataFrameWriterV2.swift b/Sources/SparkConnect/DataFrameWriterV2.swift index 55b8503..6deac16 100644 --- a/Sources/SparkConnect/DataFrameWriterV2.swift +++ b/Sources/SparkConnect/DataFrameWriterV2.swift @@ -109,7 +109,7 @@ public actor DataFrameWriterV2: Sendable { /// output table. /// - Parameter condition: A filter condition. public func overwrite(condition: String) async throws { - try await executeWriteOperation(.overwrite) + try await executeWriteOperation(.overwrite, condition) } /// Overwrite all partition for which the ``DataFrame`` contains at least one row with the contents @@ -120,7 +120,10 @@ public actor DataFrameWriterV2: Sendable { try await executeWriteOperation(.overwritePartitions) } - private func executeWriteOperation(_ mode: WriteOperationV2.Mode) async throws { + private func executeWriteOperation( + _ mode: WriteOperationV2.Mode, + _ overwriteCondition: String? = nil + ) async throws { var write = WriteOperationV2() let plan = await self.df.getPlan() as! Plan @@ -139,6 +142,9 @@ public actor DataFrameWriterV2: Sendable { for property in self.tableProperties.toStringDictionary() { write.tableProperties[property.key] = property.value } + if let overwriteCondition { + write.overwriteCondition = overwriteCondition.toExpression + } write.mode = mode var command = Spark_Connect_Command()