Skip to content

Commit 2ceed6f

Browse files
srowendongjoon-hyun
srowen
authored and
dongjoon-hyun
committed
[SPARK-29392][CORE][SQL][FOLLOWUP] Avoid deprecated (in 2.13) Symbol syntax 'foo in favor of simpler expression, where it generated deprecation warnings
### What changes were proposed in this pull request? Where it generates a deprecation warning in Scala 2.13, replace Symbol shorthand syntax `'foo` with an equivalent. ### Why are the changes needed? Symbol syntax `'foo` is deprecated in Scala 2.13. The lines changed below otherwise generate about 440 warnings when building for 2.13. The previous PR directly replaced many usages with `Symbol("foo")`. But it's also used to specify Columns via implicit conversion (`.select('foo)`) or even where simple Strings are used (`.as('foo)`), as it's kind of an abstraction for interned Strings. While I find this syntax confusing and would like to deprecate it, here I just replaced it where it generates a build warning (not sure why all occurrences don't): `$"foo"` or just `"foo"`. ### Does this PR introduce any user-facing change? Should not change behavior. ### How was this patch tested? Existing tests. Closes apache#26748 from srowen/SPARK-29392.2. Authored-by: Sean Owen <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent a2102c8 commit 2ceed6f

File tree

21 files changed

+594
-589
lines changed

21 files changed

+594
-589
lines changed

‎mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,9 @@ class LinearSVCSuite extends MLTest with DefaultReadWriteTest {
5959
// Dataset for testing SparseVector
6060
val toSparse: Vector => SparseVector = _.asInstanceOf[DenseVector].toSparse
6161
val sparse = udf(toSparse)
62-
smallSparseBinaryDataset = smallBinaryDataset.withColumn("features", sparse('features))
63-
smallSparseValidationDataset = smallValidationDataset.withColumn("features", sparse('features))
64-
62+
smallSparseBinaryDataset = smallBinaryDataset.withColumn("features", sparse($"features"))
63+
smallSparseValidationDataset =
64+
smallValidationDataset.withColumn("features", sparse($"features"))
6565
}
6666

6767
/**

‎mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,14 +161,14 @@ class PowerIterationClusteringSuite extends SparkFunSuite
161161
}
162162

163163
test("test default weight") {
164-
val dataWithoutWeight = data.sample(0.5, 1L).select('src, 'dst)
164+
val dataWithoutWeight = data.sample(0.5, 1L).select("src", "dst")
165165

166166
val assignments = new PowerIterationClustering()
167167
.setK(2)
168168
.setMaxIter(40)
169169
.assignClusters(dataWithoutWeight)
170170
val localAssignments = assignments
171-
.select('id, 'cluster)
171+
.select("id", "cluster")
172172
.as[(Long, Int)].collect().toSet
173173

174174
val dataWithWeightOne = dataWithoutWeight.withColumn("weight", lit(1.0))
@@ -178,7 +178,7 @@ class PowerIterationClusteringSuite extends SparkFunSuite
178178
.setMaxIter(40)
179179
.assignClusters(dataWithWeightOne)
180180
val localAssignments2 = assignments2
181-
.select('id, 'cluster)
181+
.select("id", "cluster")
182182
.as[(Long, Int)].collect().toSet
183183

184184
assert(localAssignments === localAssignments2)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/DistributionSuite.scala

Lines changed: 61 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -59,12 +59,12 @@ class DistributionSuite extends SparkFunSuite {
5959
true)
6060

6161
checkSatisfied(
62-
HashPartitioning(Seq('a), 10),
62+
HashPartitioning(Seq($"a"), 10),
6363
UnspecifiedDistribution,
6464
true)
6565

6666
checkSatisfied(
67-
RangePartitioning(Seq('a.asc), 10),
67+
RangePartitioning(Seq($"a".asc), 10),
6868
UnspecifiedDistribution,
6969
true)
7070

@@ -101,22 +101,22 @@ class DistributionSuite extends SparkFunSuite {
101101
true)
102102

103103
checkSatisfied(
104-
HashPartitioning(Seq('a), 1),
104+
HashPartitioning(Seq($"a"), 1),
105105
AllTuples,
106106
true)
107107

108108
checkSatisfied(
109-
HashPartitioning(Seq('a), 10),
109+
HashPartitioning(Seq($"a"), 10),
110110
AllTuples,
111111
false)
112112

113113
checkSatisfied(
114-
RangePartitioning(Seq('a.asc), 1),
114+
RangePartitioning(Seq($"a".asc), 1),
115115
AllTuples,
116116
true)
117117

118118
checkSatisfied(
119-
RangePartitioning(Seq('a.asc), 10),
119+
RangePartitioning(Seq($"a".asc), 10),
120120
AllTuples,
121121
false)
122122

@@ -130,17 +130,17 @@ class DistributionSuite extends SparkFunSuite {
130130
// SinglePartition can satisfy all the distributions except `BroadcastDistribution`
131131
checkSatisfied(
132132
SinglePartition,
133-
ClusteredDistribution(Seq('a, 'b, 'c)),
133+
ClusteredDistribution(Seq($"a", $"b", $"c")),
134134
true)
135135

136136
checkSatisfied(
137137
SinglePartition,
138-
HashClusteredDistribution(Seq('a, 'b, 'c)),
138+
HashClusteredDistribution(Seq($"a", $"b", $"c")),
139139
true)
140140

141141
checkSatisfied(
142142
SinglePartition,
143-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc)),
143+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc)),
144144
true)
145145

146146
checkSatisfied(
@@ -153,154 +153,154 @@ class DistributionSuite extends SparkFunSuite {
153153
// HashPartitioning can satisfy ClusteredDistribution iff its hash expressions are a subset of
154154
// the required clustering expressions.
155155
checkSatisfied(
156-
HashPartitioning(Seq('a, 'b, 'c), 10),
157-
ClusteredDistribution(Seq('a, 'b, 'c)),
156+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
157+
ClusteredDistribution(Seq($"a", $"b", $"c")),
158158
true)
159159

160160
checkSatisfied(
161-
HashPartitioning(Seq('b, 'c), 10),
162-
ClusteredDistribution(Seq('a, 'b, 'c)),
161+
HashPartitioning(Seq($"b", $"c"), 10),
162+
ClusteredDistribution(Seq($"a", $"b", $"c")),
163163
true)
164164

165165
checkSatisfied(
166-
HashPartitioning(Seq('a, 'b, 'c), 10),
167-
ClusteredDistribution(Seq('b, 'c)),
166+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
167+
ClusteredDistribution(Seq($"b", $"c")),
168168
false)
169169

170170
checkSatisfied(
171-
HashPartitioning(Seq('a, 'b, 'c), 10),
172-
ClusteredDistribution(Seq('d, 'e)),
171+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
172+
ClusteredDistribution(Seq($"d", $"e")),
173173
false)
174174

175175
// HashPartitioning can satisfy HashClusteredDistribution iff its hash expressions are exactly
176176
// same with the required hash clustering expressions.
177177
checkSatisfied(
178-
HashPartitioning(Seq('a, 'b, 'c), 10),
179-
HashClusteredDistribution(Seq('a, 'b, 'c)),
178+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
179+
HashClusteredDistribution(Seq($"a", $"b", $"c")),
180180
true)
181181

182182
checkSatisfied(
183-
HashPartitioning(Seq('c, 'b, 'a), 10),
184-
HashClusteredDistribution(Seq('a, 'b, 'c)),
183+
HashPartitioning(Seq($"c", $"b", $"a"), 10),
184+
HashClusteredDistribution(Seq($"a", $"b", $"c")),
185185
false)
186186

187187
checkSatisfied(
188-
HashPartitioning(Seq('a, 'b), 10),
189-
HashClusteredDistribution(Seq('a, 'b, 'c)),
188+
HashPartitioning(Seq($"a", $"b"), 10),
189+
HashClusteredDistribution(Seq($"a", $"b", $"c")),
190190
false)
191191

192192
// HashPartitioning cannot satisfy OrderedDistribution
193193
checkSatisfied(
194-
HashPartitioning(Seq('a, 'b, 'c), 10),
195-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc)),
194+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
195+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc)),
196196
false)
197197

198198
checkSatisfied(
199-
HashPartitioning(Seq('a, 'b, 'c), 1),
200-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc)),
199+
HashPartitioning(Seq($"a", $"b", $"c"), 1),
200+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc)),
201201
false) // TODO: this can be relaxed.
202202

203203
checkSatisfied(
204-
HashPartitioning(Seq('b, 'c), 10),
205-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc)),
204+
HashPartitioning(Seq($"b", $"c"), 10),
205+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc)),
206206
false)
207207
}
208208

209209
test("RangePartitioning is the output partitioning") {
210210
// RangePartitioning can satisfy OrderedDistribution iff its ordering is a prefix
211211
// of the required ordering, or the required ordering is a prefix of its ordering.
212212
checkSatisfied(
213-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
214-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc)),
213+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
214+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc)),
215215
true)
216216

217217
checkSatisfied(
218-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
219-
OrderedDistribution(Seq('a.asc, 'b.asc)),
218+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
219+
OrderedDistribution(Seq($"a".asc, $"b".asc)),
220220
true)
221221

222222
checkSatisfied(
223-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
224-
OrderedDistribution(Seq('a.asc, 'b.asc, 'c.asc, 'd.desc)),
223+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
224+
OrderedDistribution(Seq($"a".asc, $"b".asc, $"c".asc, 'd.desc)),
225225
true)
226226

227227
// TODO: We can have an optimization to first sort the dataset
228228
// by a.asc and then sort b, and c in a partition. This optimization
229229
// should tradeoff the benefit of a less number of Exchange operators
230230
// and the parallelism.
231231
checkSatisfied(
232-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
233-
OrderedDistribution(Seq('a.asc, 'b.desc, 'c.asc)),
232+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
233+
OrderedDistribution(Seq($"a".asc, $"b".desc, $"c".asc)),
234234
false)
235235

236236
checkSatisfied(
237-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
238-
OrderedDistribution(Seq('b.asc, 'a.asc)),
237+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
238+
OrderedDistribution(Seq($"b".asc, $"a".asc)),
239239
false)
240240

241241
checkSatisfied(
242-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
243-
OrderedDistribution(Seq('a.asc, 'b.asc, 'd.desc)),
242+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
243+
OrderedDistribution(Seq($"a".asc, $"b".asc, 'd.desc)),
244244
false)
245245

246246
// RangePartitioning can satisfy ClusteredDistribution iff its ordering expressions are a subset
247247
// of the required clustering expressions.
248248
checkSatisfied(
249-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
250-
ClusteredDistribution(Seq('a, 'b, 'c)),
249+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
250+
ClusteredDistribution(Seq($"a", $"b", $"c")),
251251
true)
252252

253253
checkSatisfied(
254-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
255-
ClusteredDistribution(Seq('c, 'b, 'a)),
254+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
255+
ClusteredDistribution(Seq($"c", $"b", $"a")),
256256
true)
257257

258258
checkSatisfied(
259-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
260-
ClusteredDistribution(Seq('b, 'c, 'a, 'd)),
259+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
260+
ClusteredDistribution(Seq($"b", $"c", $"a", $"d")),
261261
true)
262262

263263
checkSatisfied(
264-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
265-
ClusteredDistribution(Seq('a, 'b)),
264+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
265+
ClusteredDistribution(Seq($"a", $"b")),
266266
false)
267267

268268
checkSatisfied(
269-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
270-
ClusteredDistribution(Seq('c, 'd)),
269+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
270+
ClusteredDistribution(Seq($"c", $"d")),
271271
false)
272272

273273
// RangePartitioning cannot satisfy HashClusteredDistribution
274274
checkSatisfied(
275-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
276-
HashClusteredDistribution(Seq('a, 'b, 'c)),
275+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
276+
HashClusteredDistribution(Seq($"a", $"b", $"c")),
277277
false)
278278
}
279279

280280
test("Partitioning.numPartitions must match Distribution.requiredNumPartitions to satisfy it") {
281281
checkSatisfied(
282282
SinglePartition,
283-
ClusteredDistribution(Seq('a, 'b, 'c), Some(10)),
283+
ClusteredDistribution(Seq($"a", $"b", $"c"), Some(10)),
284284
false)
285285

286286
checkSatisfied(
287287
SinglePartition,
288-
HashClusteredDistribution(Seq('a, 'b, 'c), Some(10)),
288+
HashClusteredDistribution(Seq($"a", $"b", $"c"), Some(10)),
289289
false)
290290

291291
checkSatisfied(
292-
HashPartitioning(Seq('a, 'b, 'c), 10),
293-
ClusteredDistribution(Seq('a, 'b, 'c), Some(5)),
292+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
293+
ClusteredDistribution(Seq($"a", $"b", $"c"), Some(5)),
294294
false)
295295

296296
checkSatisfied(
297-
HashPartitioning(Seq('a, 'b, 'c), 10),
298-
HashClusteredDistribution(Seq('a, 'b, 'c), Some(5)),
297+
HashPartitioning(Seq($"a", $"b", $"c"), 10),
298+
HashClusteredDistribution(Seq($"a", $"b", $"c"), Some(5)),
299299
false)
300300

301301
checkSatisfied(
302-
RangePartitioning(Seq('a.asc, 'b.asc, 'c.asc), 10),
303-
ClusteredDistribution(Seq('a, 'b, 'c), Some(5)),
302+
RangePartitioning(Seq($"a".asc, $"b".asc, $"c".asc), 10),
303+
ClusteredDistribution(Seq($"a", $"b", $"c"), Some(5)),
304304
false)
305305
}
306306
}

‎sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
9696

9797
test("cache temp table") {
9898
withTempView("tempTable") {
99-
testData.select('key).createOrReplaceTempView("tempTable")
99+
testData.select("key").createOrReplaceTempView("tempTable")
100100
assertCached(sql("SELECT COUNT(*) FROM tempTable"), 0)
101101
spark.catalog.cacheTable("tempTable")
102102
assertCached(sql("SELECT COUNT(*) FROM tempTable"))
@@ -127,8 +127,8 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
127127
}
128128

129129
test("uncaching temp table") {
130-
testData.select('key).createOrReplaceTempView("tempTable1")
131-
testData.select('key).createOrReplaceTempView("tempTable2")
130+
testData.select("key").createOrReplaceTempView("tempTable1")
131+
testData.select("key").createOrReplaceTempView("tempTable2")
132132
spark.catalog.cacheTable("tempTable1")
133133

134134
assertCached(sql("SELECT COUNT(*) FROM tempTable1"))
@@ -361,15 +361,15 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
361361
}
362362

363363
test("Drops temporary table") {
364-
testData.select('key).createOrReplaceTempView("t1")
364+
testData.select("key").createOrReplaceTempView("t1")
365365
spark.table("t1")
366366
spark.catalog.dropTempView("t1")
367367
intercept[AnalysisException](spark.table("t1"))
368368
}
369369

370370
test("Drops cached temporary table") {
371-
testData.select('key).createOrReplaceTempView("t1")
372-
testData.select('key).createOrReplaceTempView("t2")
371+
testData.select("key").createOrReplaceTempView("t1")
372+
testData.select("key").createOrReplaceTempView("t2")
373373
spark.catalog.cacheTable("t1")
374374

375375
assert(spark.catalog.isCached("t1"))
@@ -859,7 +859,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
859859

860860
test("SPARK-23880 table cache should be lazy and don't trigger any jobs") {
861861
val cachedData = checkIfNoJobTriggered {
862-
spark.range(1002).filter('id > 1000).orderBy('id.desc).cache()
862+
spark.range(1002).filter($"id" > 1000).orderBy($"id".desc).cache()
863863
}
864864
assert(cachedData.collect === Seq(1001))
865865
}
@@ -891,7 +891,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
891891

892892
test("SPARK-24596 Non-cascading Cache Invalidation - drop persistent view") {
893893
withTable("t") {
894-
spark.range(1, 10).toDF("key").withColumn("value", 'key * 2)
894+
spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
895895
.write.format("json").saveAsTable("t")
896896
withView("t1") {
897897
withTempView("t2") {
@@ -911,7 +911,7 @@ class CachedTableSuite extends QueryTest with SQLTestUtils with SharedSparkSessi
911911

912912
test("SPARK-24596 Non-cascading Cache Invalidation - uncache table") {
913913
withTable("t") {
914-
spark.range(1, 10).toDF("key").withColumn("value", 'key * 2)
914+
spark.range(1, 10).toDF("key").withColumn("value", $"key" * 2)
915915
.write.format("json").saveAsTable("t")
916916
withTempView("t1", "t2") {
917917
sql("CACHE TABLE t")

0 commit comments

Comments
 (0)