Skip to content

Commit c629439

Browse files
AFFogartyAFFogarty
authored
Add Delta Lake version annotations to Microsoft.Spark.Extensions.Delta APIs. (#632)
1 parent 38531de commit c629439

File tree

6 files changed

+83
-0
lines changed

6 files changed

+83
-0
lines changed

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/Attributes.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using System;
6+
7+
namespace Microsoft.Spark.Extensions.Delta
8+
{
9+
/// <summary>
10+
/// Custom attribute to denote the Delta Lake version in which an API is introduced.
11+
/// </summary>
12+
[AttributeUsage(AttributeTargets.All)]
13+
public sealed class DeltaLakeSinceAttribute : VersionAttribute
14+
{
15+
/// <summary>
16+
/// Constructor for DeltaLakeSinceAttribute class.
17+
/// </summary>
18+
/// <param name="version">Delta Lake version</param>
19+
public DeltaLakeSinceAttribute(string version)
20+
: base(version)
21+
{
22+
}
23+
}
24+
}

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/DeltaLakeVersions.cs‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
namespace Microsoft.Spark.Extensions.Delta
6+
{
7+
internal static class DeltaLakeVersions
8+
{
9+
internal const string V0_1_0 = "0.1.0";
10+
internal const string V0_2_0 = "0.2.0";
11+
internal const string V0_3_0 = "0.3.0";
12+
internal const string V0_4_0 = "0.4.0";
13+
internal const string V0_5_0 = "0.5.0";
14+
internal const string V0_6_0 = "0.6.0";
15+
internal const string V0_6_1 = "0.6.1";
16+
}
17+
}

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/Tables/DeltaMergeBuilder.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ namespace Microsoft.Spark.Extensions.Delta.Tables
7575
/// </code>
7676
/// </example>
7777
/// </summary>
78+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
7879
public class DeltaMergeBuilder : IJvmObjectReferenceProvider
7980
{
8081
private readonly JvmObjectReference _jvmObject;
@@ -92,6 +93,7 @@ internal DeltaMergeBuilder(JvmObjectReference jvmObject)
9293
/// delete the matched target table row with the source row.
9394
/// </summary>
9495
/// <returns>DeltaMergeMatchedActionBuilder object.</returns>
96+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
9597
public DeltaMergeMatchedActionBuilder WhenMatched() =>
9698
new DeltaMergeMatchedActionBuilder(
9799
(JvmObjectReference)_jvmObject.Invoke("whenMatched"));
@@ -104,6 +106,7 @@ public DeltaMergeMatchedActionBuilder WhenMatched() =>
104106
/// </summary>
105107
/// <param name="condition">Boolean expression as a SQL formatted string.</param>
106108
/// <returns>DeltaMergeMatchedActionBuilder object.</returns>
109+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
107110
public DeltaMergeMatchedActionBuilder WhenMatched(string condition) =>
108111
new DeltaMergeMatchedActionBuilder(
109112
(JvmObjectReference)_jvmObject.Invoke("whenMatched", condition));
@@ -116,6 +119,7 @@ public DeltaMergeMatchedActionBuilder WhenMatched(string condition) =>
116119
/// </summary>
117120
/// <param name="condition">Boolean expression as a Column object.</param>
118121
/// <returns>DeltaMergeMatchedActionBuilder object.</returns>
122+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
119123
public DeltaMergeMatchedActionBuilder WhenMatched(Column condition) =>
120124
new DeltaMergeMatchedActionBuilder(
121125
(JvmObjectReference)_jvmObject.Invoke("whenMatched", condition));
@@ -126,6 +130,7 @@ public DeltaMergeMatchedActionBuilder WhenMatched(Column condition) =>
126130
/// new sourced row into the target table.
127131
/// </summary>
128132
/// <returns>DeltaMergeNotMatchedActionBuilder object.</returns>
133+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
129134
public DeltaMergeNotMatchedActionBuilder WhenNotMatched() =>
130135
new DeltaMergeNotMatchedActionBuilder(
131136
(JvmObjectReference)_jvmObject.Invoke("whenNotMatched"));
@@ -137,6 +142,7 @@ public DeltaMergeNotMatchedActionBuilder WhenNotMatched() =>
137142
/// </summary>
138143
/// <param name="condition">Boolean expression as a SQL formatted string.</param>
139144
/// <returns>DeltaMergeNotMatchedActionBuilder object.</returns>
145+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
140146
public DeltaMergeNotMatchedActionBuilder WhenNotMatched(string condition) =>
141147
new DeltaMergeNotMatchedActionBuilder(
142148
(JvmObjectReference)_jvmObject.Invoke("whenNotMatched", condition));
@@ -148,13 +154,15 @@ public DeltaMergeNotMatchedActionBuilder WhenNotMatched(string condition) =>
148154
/// </summary>
149155
/// <param name="condition">Boolean expression as a Column object.</param>
150156
/// <returns>DeltaMergeNotMatchedActionBuilder object.</returns>
157+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
151158
public DeltaMergeNotMatchedActionBuilder WhenNotMatched(Column condition) =>
152159
new DeltaMergeNotMatchedActionBuilder(
153160
(JvmObjectReference)_jvmObject.Invoke("whenNotMatched", condition));
154161

155162
/// <summary>
156163
/// Execute the merge operation based on the built matched and not matched actions.
157164
/// </summary>
165+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
158166
public void Execute() => _jvmObject.Invoke("execute");
159167
}
160168
}

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/Tables/DeltaMergeMatchedActionBuilder.cs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ namespace Microsoft.Spark.Extensions.Delta.Tables
1212
/// Builder class to specify the actions to perform when a target table row has matched a
1313
/// source row based on the given merge condition and optional match condition.
1414
/// </summary>
15+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
1516
public class DeltaMergeMatchedActionBuilder : IJvmObjectReferenceProvider
1617
{
1718
private readonly JvmObjectReference _jvmObject;
@@ -29,6 +30,7 @@ internal DeltaMergeMatchedActionBuilder(JvmObjectReference jvmObject)
2930
/// <param name="set">Rules to update a row as amap between target column names and
3031
/// corresponding update expressions as Column objects.</param>
3132
/// <returns>DeltaMergeBuilder object.</returns>
33+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
3234
public DeltaMergeBuilder Update(Dictionary<string, Column> set) =>
3335
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("update", set));
3436

@@ -38,6 +40,7 @@ public DeltaMergeBuilder Update(Dictionary<string, Column> set) =>
3840
/// <param name="set">Rules to update a row as a map between target column names and
3941
/// corresponding update expressions as SQL formatted strings.</param>
4042
/// <returns>DeltaMergeBuilder object.</returns>
43+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
4144
public DeltaMergeBuilder UpdateExpr(Dictionary<string, string> set) =>
4245
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("updateExpr", set));
4346

@@ -46,13 +49,15 @@ public DeltaMergeBuilder UpdateExpr(Dictionary<string, string> set) =>
4649
/// columns in the source row.
4750
/// </summary>
4851
/// <returns>DeltaMergeBuilder object.</returns>
52+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
4953
public DeltaMergeBuilder UpdateAll() =>
5054
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("updateAll"));
5155

5256
/// <summary>
5357
/// Delete a matched row from the table.
5458
/// </summary>
5559
/// <returns>DeltaMergeBuilder object.</returns>
60+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
5661
public DeltaMergeBuilder Delete() =>
5762
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("delete"));
5863
}

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/Tables/DeltaMergeNotMatchedActionBuilder.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ namespace Microsoft.Spark.Extensions.Delta.Tables
1515
///
1616
/// See <see cref="DeltaMergeBuilder"/> for more information.
1717
/// </summary>
18+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
1819
public class DeltaMergeNotMatchedActionBuilder : IJvmObjectReferenceProvider
1920
{
2021
private readonly JvmObjectReference _jvmObject;
@@ -32,6 +33,7 @@ internal DeltaMergeNotMatchedActionBuilder(JvmObjectReference jvmObject)
3233
/// <param name="values">Rules to insert a row as a map between target column names and
3334
/// corresponding expressions as Column objects.</param>
3435
/// <returns>DeltaMergeBuilder object.</returns>
36+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
3537
public DeltaMergeBuilder Insert(Dictionary<string, Column> values) =>
3638
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("insert", values));
3739

@@ -41,6 +43,7 @@ public DeltaMergeBuilder Insert(Dictionary<string, Column> values) =>
4143
/// <param name="values">Rules to insert a row as a map between target column names and
4244
/// corresponding expressions as SQL formatted strings.</param>
4345
/// <returns>DeltaMergeBuilder object.</returns>
46+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
4447
public DeltaMergeBuilder InsertExpr(Dictionary<string, string> values) =>
4548
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("insertExpr", values));
4649

@@ -49,6 +52,7 @@ public DeltaMergeBuilder InsertExpr(Dictionary<string, string> values) =>
4952
/// corresponding columns in the source row.
5053
/// </summary>
5154
/// <returns>DeltaMergeBuilder object.</returns>
55+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
5256
public DeltaMergeBuilder InsertAll() =>
5357
new DeltaMergeBuilder((JvmObjectReference)_jvmObject.Invoke("insertAll"));
5458
}

‎src/csharp/Extensions/Microsoft.Spark.Extensions.Delta/Tables/DeltaTable.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ namespace Microsoft.Spark.Extensions.Delta.Tables
1818
/// DeltaTable.ForPath(sparkSession, pathToTheDeltaTable)
1919
/// </code>
2020
/// </summary>
21+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
2122
public class DeltaTable : IJvmObjectReferenceProvider
2223
{
2324
private readonly JvmObjectReference _jvmObject;
@@ -56,6 +57,7 @@ internal DeltaTable(JvmObjectReference jvmObject)
5657
/// <param name="identifier">String used to identify the parquet table.</param>
5758
/// <param name="partitionSchema">StructType representing the partition schema.</param>
5859
/// <returns>The converted DeltaTable.</returns>
60+
[DeltaLakeSince(DeltaLakeVersions.V0_4_0)]
5961
public static DeltaTable ConvertToDelta(
6062
SparkSession spark,
6163
string identifier,
@@ -86,6 +88,7 @@ public static DeltaTable ConvertToDelta(
8688
/// <param name="identifier">String used to identify the parquet table.</param>
8789
/// <param name="partitionSchema">String representing the partition schema.</param>
8890
/// <returns>The converted DeltaTable.</returns>
91+
[DeltaLakeSince(DeltaLakeVersions.V0_4_0)]
8992
public static DeltaTable ConvertToDelta(
9093
SparkSession spark,
9194
string identifier,
@@ -114,6 +117,7 @@ public static DeltaTable ConvertToDelta(
114117
/// <param name="spark">The relevant session.</param>
115118
/// <param name="identifier">String used to identify the parquet table.</param>
116119
/// <returns>The converted DeltaTable.</returns>
120+
[DeltaLakeSince(DeltaLakeVersions.V0_4_0)]
117121
public static DeltaTable ConvertToDelta(SparkSession spark, string identifier) =>
118122
new DeltaTable(
119123
(JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod(
@@ -131,6 +135,7 @@ public static DeltaTable ConvertToDelta(SparkSession spark, string identifier) =
131135
/// </summary>
132136
/// <param name="path">The path to the data.</param>
133137
/// <returns>DeltaTable loaded from the path.</returns>
138+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
134139
public static DeltaTable ForPath(string path) =>
135140
new DeltaTable(
136141
(JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod(
@@ -145,6 +150,7 @@ public static DeltaTable ForPath(string path) =>
145150
/// <param name="sparkSession">The active SparkSession.</param>
146151
/// <param name="path">The path to the data.</param>
147152
/// <returns>DeltaTable loaded from the path.</returns>
153+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
148154
public static DeltaTable ForPath(SparkSession sparkSession, string path) =>
149155
new DeltaTable(
150156
(JvmObjectReference)SparkEnvironment.JvmBridge.CallStaticJavaMethod(
@@ -165,6 +171,7 @@ public static DeltaTable ForPath(SparkSession sparkSession, string path) =>
165171
/// <param name="sparkSession">The relevant session.</param>
166172
/// <param name="identifier">String that identifies the table, e.g. path to table.</param>
167173
/// <returns>True if the table is a DeltaTable.</returns>
174+
[DeltaLakeSince(DeltaLakeVersions.V0_4_0)]
168175
public static bool IsDeltaTable(SparkSession sparkSession, string identifier) =>
169176
(bool)SparkEnvironment.JvmBridge.CallStaticJavaMethod(
170177
s_deltaTableClassName,
@@ -187,6 +194,7 @@ public static bool IsDeltaTable(SparkSession sparkSession, string identifier) =>
187194
/// </summary>
188195
/// <param name="identifier">String that identifies the table, e.g. path to table.</param>
189196
/// <returns>True if the table is a DeltaTable.</returns>
197+
[DeltaLakeSince(DeltaLakeVersions.V0_4_0)]
190198
public static bool IsDeltaTable(string identifier) =>
191199
(bool)SparkEnvironment.JvmBridge.CallStaticJavaMethod(
192200
s_deltaTableClassName,
@@ -199,6 +207,7 @@ public static bool IsDeltaTable(string identifier) =>
199207
/// </summary>
200208
/// <param name="alias">The table alias.</param>
201209
/// <returns>Aliased DeltaTable.</returns>
210+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
202211
public DeltaTable As(string alias) =>
203212
new DeltaTable((JvmObjectReference)_jvmObject.Invoke("as", alias));
204213

@@ -208,13 +217,15 @@ public DeltaTable As(string alias) =>
208217
/// </summary>
209218
/// <param name="alias">The table alias.</param>
210219
/// <returns>Aliased DeltaTable.</returns>
220+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
211221
public DeltaTable Alias(string alias) =>
212222
new DeltaTable((JvmObjectReference)_jvmObject.Invoke("alias", alias));
213223

214224
/// <summary>
215225
/// Get a DataFrame (that is, Dataset[Row]) representation of this Delta table.
216226
/// </summary>
217227
/// <returns>DataFrame representation of Delta table.</returns>
228+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
218229
public DataFrame ToDF() => new DataFrame((JvmObjectReference)_jvmObject.Invoke("toDF"));
219230

220231
/// <summary>
@@ -226,6 +237,7 @@ public DeltaTable Alias(string alias) =>
226237
/// table for reading versions earlier than this will be preserved and the rest of them
227238
/// will be deleted.</param>
228239
/// <returns>Vacuumed DataFrame.</returns>
240+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
229241
public DataFrame Vacuum(double retentionHours) =>
230242
new DataFrame((JvmObjectReference)_jvmObject.Invoke("vacuum", retentionHours));
231243

@@ -237,6 +249,7 @@ public DataFrame Vacuum(double retentionHours) =>
237249
/// Note: This will use the default retention period of 7 days.
238250
/// </summary>
239251
/// <returns>Vacuumed DataFrame.</returns>
252+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
240253
public DataFrame Vacuum() =>
241254
new DataFrame((JvmObjectReference)_jvmObject.Invoke("vacuum"));
242255

@@ -246,6 +259,7 @@ public DataFrame Vacuum() =>
246259
/// </summary>
247260
/// <param name="limit">The number of previous commands to get history for.</param>
248261
/// <returns>History DataFrame.</returns>
262+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
249263
public DataFrame History(int limit) =>
250264
new DataFrame((JvmObjectReference)_jvmObject.Invoke("history", limit));
251265

@@ -254,6 +268,7 @@ public DataFrame History(int limit) =>
254268
/// information is in reverse chronological order.
255269
/// </summary>
256270
/// <returns>History DataFrame</returns>
271+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
257272
public DataFrame History() =>
258273
new DataFrame((JvmObjectReference)_jvmObject.Invoke("history"));
259274

@@ -265,23 +280,27 @@ public DataFrame History() =>
265280
/// - "symlink_format_manifest" : This will generate manifests in symlink format
266281
/// for Presto and Athena read support.
267282
/// See the online documentation for more information.</param>
283+
[DeltaLakeSince(DeltaLakeVersions.V0_5_0)]
268284
public void Generate(string mode) => _jvmObject.Invoke("generate", mode);
269285

270286
/// <summary>
271287
/// Delete data from the table that match the given <c>condition</c>.
272288
/// </summary>
273289
/// <param name="condition">Boolean SQL expression.</param>
290+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
274291
public void Delete(string condition) => _jvmObject.Invoke("delete", condition);
275292

276293
/// <summary>
277294
/// Delete data from the table that match the given <c>condition</c>.
278295
/// </summary>
279296
/// <param name="condition">Boolean SQL expression.</param>
297+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
280298
public void Delete(Column condition) => _jvmObject.Invoke("delete", condition);
281299

282300
/// <summary>
283301
/// Delete data from the table.
284302
/// </summary>
303+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
285304
public void Delete() => _jvmObject.Invoke("delete");
286305

287306
/// <summary>
@@ -297,6 +316,7 @@ public DataFrame History() =>
297316
/// </example>
298317
/// <param name="set">Pules to update a row as a Scala map between target column names
299318
/// and corresponding update expressions as Column objects.</param>
319+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
300320
public void Update(Dictionary<string, Column> set) => _jvmObject.Invoke("update", set);
301321

302322
/// <summary>
@@ -317,6 +337,7 @@ public DataFrame History() =>
317337
/// to update.</param>
318338
/// <param name="set">Rules to update a row as a Scala map between target column names and
319339
/// corresponding update expressions as Column objects.</param>
340+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
320341
public void Update(Column condition, Dictionary<string, Column> set) =>
321342
_jvmObject.Invoke("update", condition, set);
322343

@@ -334,6 +355,7 @@ public void Update(Column condition, Dictionary<string, Column> set) =>
334355
/// </example>
335356
/// <param name="set">Rules to update a row as a Scala map between target column names and
336357
/// corresponding update expressions as SQL formatted strings.</param>
358+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
337359
public void UpdateExpr(Dictionary<string, string> set) =>
338360
_jvmObject.Invoke("updateExpr", set);
339361

@@ -355,6 +377,7 @@ public void UpdateExpr(Dictionary<string, string> set) =>
355377
/// which rows to update.</param>
356378
/// <param name="set">Rules to update a row as a map between target column names and
357379
/// corresponding update expressions as SQL formatted strings.</param>
380+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
358381
public void UpdateExpr(string condition, Dictionary<string, string> set) =>
359382
_jvmObject.Invoke("updateExpr", condition, set);
360383

@@ -395,6 +418,7 @@ public void UpdateExpr(string condition, Dictionary<string, string> set) =>
395418
/// <param name="source">Source Dataframe to be merged.</param>
396419
/// <param name="condition">Boolean expression as SQL formatted string.</param>
397420
/// <returns>DeltaMergeBuilder</returns>
421+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
398422
public DeltaMergeBuilder Merge(DataFrame source, string condition) =>
399423
new DeltaMergeBuilder(
400424
(JvmObjectReference)_jvmObject.Invoke(
@@ -436,6 +460,7 @@ public DeltaMergeBuilder Merge(DataFrame source, string condition) =>
436460
/// <param name="source">Source Dataframe to be merged.</param>
437461
/// <param name="condition">Coolean expression as a Column object</param>
438462
/// <returns>DeltaMergeBuilder</returns>
463+
[DeltaLakeSince(DeltaLakeVersions.V0_3_0)]
439464
public DeltaMergeBuilder Merge(DataFrame source, Column condition) =>
440465
new DeltaMergeBuilder(
441466
(JvmObjectReference)_jvmObject.Invoke(

0 commit comments

Comments
 (0)