Skip to content

Commit 27f9f7f

Browse files
derrickburnsclaude
andcommitted
Fix Scala 2.13 compilation errors
- Fix ambiguous getAs method references by adding explicit type parameters - CLARA.scala: Specify Vector type and index for getAs calls - KMedoids.scala: Specify Vector type and index for getAs calls - SoftKMeans.scala: Specify Vector/Double types and column names for getAs calls - Fix tuple pattern matching error in KMedoids.scala by simplifying assignment - Remove unused imports to clean up compiler warnings - Remove unused scala.language.implicitConversions from compat package - Remove unused Vectors import from KMedoids.scala - Remove unused Random and DoubleType imports from CoresetKMeans.scala - Remove unused spark.implicits import from PersistenceRoundTripCoresetKMeans.scala All changes maintain API compatibility and preserve existing functionality. Verified compilation with both Scala 2.13.14 and 2.12.18 against Spark 3.5.1. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
1 parent 8975970 commit 27f9f7f

File tree

6 files changed

+10
-16
lines changed

6 files changed

+10
-16
lines changed

src/main/scala-2.13/com/massivedatascience/clusterer/compat/package.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
package com.massivedatascience.clusterer
22

3-
import scala.language.implicitConversions
4-
53
// Scala 2.13: Provide .par extension method via compat package
64
package object compat {
75
implicit class ParOps[A, CC[X] <: Iterable[X]](private val coll: CC[A]) extends AnyVal {

src/main/scala/com/massivedatascience/clusterer/ml/CLARA.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ class CLARA(override val uid: String)
8585
.select($(featuresCol))
8686
.rdd
8787
.map { row =>
88-
row.getAs
88+
row.getAs[Vector](0)
8989
}
9090
.collect()
9191

src/main/scala/com/massivedatascience/clusterer/ml/CoresetKMeans.scala

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,7 @@ import org.apache.spark.ml.param.ParamMap
77
import org.apache.spark.ml.util.{ DefaultParamsReadable, DefaultParamsWritable, Identifiable }
88
import org.apache.spark.sql.{ DataFrame, Dataset }
99
import org.apache.spark.sql.functions._
10-
import org.apache.spark.sql.types.{ DoubleType, StructType }
11-
import scala.util.Random
10+
import org.apache.spark.sql.types.StructType
1211

1312
/** Core-set based K-Means clustering with pluggable Bregman divergences.
1413
*

src/main/scala/com/massivedatascience/clusterer/ml/KMedoids.scala

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ package com.massivedatascience.clusterer.ml
22

33
import org.apache.spark.internal.Logging
44
import org.apache.spark.ml.Estimator
5-
import org.apache.spark.ml.linalg.{ Vector, Vectors }
5+
import org.apache.spark.ml.linalg.Vector
66
import org.apache.spark.ml.param._
77
import org.apache.spark.ml.param.shared._
88
import org.apache.spark.ml.util._
@@ -102,7 +102,7 @@ class KMedoids(override val uid: String)
102102
.select($(featuresCol))
103103
.rdd
104104
.map { row =>
105-
row.getAs
105+
row.getAs[Vector](0)
106106
}
107107
.collect()
108108

@@ -351,8 +351,7 @@ class KMedoids(override val uid: String)
351351
val (_, _, costHistory) = swapPhaseWithHistory(data, initialMedoidIndices, maxIter, distFn)
352352
// last state is encoded in the returned medoid indices by swapPhaseWithHistory
353353
// but we need to recompute those indices; reuse the logic:
354-
val distFnLocal = distFn
355-
val (_, _, _) = (distFnLocal, costHistory) // keep params used, avoid warnings
354+
val _ = (distFn, costHistory) // keep params used, avoid warnings
356355
// For API compatibility, just re-run a lightweight swap without history:
357356
val n = data.length
358357
val k = initialMedoidIndices.length
@@ -568,7 +567,7 @@ class KMedoidsModel(
568567
df.select($(featuresCol))
569568
.rdd
570569
.map { row =>
571-
val features = row.getAs
570+
val features = row.getAs[Vector](0)
572571
val meds = bcMedoids.value
573572
meds.map(m => distFn(features, m)).min
574573
}

src/main/scala/com/massivedatascience/clusterer/ml/SoftKMeans.scala

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,7 @@ class SoftKMeans(override val uid: String)
232232
.sample(withReplacement = false, fraction, seed)
233233
.limit(k)
234234
.collect()
235-
.map(_.getAs.toArray)
235+
.map(_.getAs[org.apache.spark.ml.linalg.Vector](0).toArray)
236236
}
237237

238238
/** Compute soft assignment probabilities for all points (Boltzmann distribution). */
@@ -280,10 +280,10 @@ class SoftKMeans(override val uid: String)
280280
var totalWeight = 0.0
281281

282282
membershipData.foreach { row =>
283-
val features = row.getAs.toArray
284-
val probs = row.getAs.toArray
283+
val features = row.getAs[org.apache.spark.ml.linalg.Vector](featCol).toArray
284+
val probs = row.getAs[org.apache.spark.ml.linalg.Vector]($(probabilityCol)).toArray
285285
val clusterProb = probs(clusterId)
286-
val w = weightColOpt.map(_ => row.getAs).getOrElse(1.0)
286+
val w = weightColOpt.map(col => row.getAs[Double](col)).getOrElse(1.0)
287287
val finalWeight = clusterProb * w
288288

289289
if (weightedSum == null) weightedSum = Array.fill(features.length)(0.0)

src/main/scala/examples/PersistenceRoundTripCoresetKMeans.scala

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ object PersistenceRoundTripCoresetKMeans {
2222
val spark =
2323
SparkSession.builder().appName("CoresetKMeans Persistence").master("local[*]").getOrCreate()
2424

25-
import spark.implicits._
26-
2725
mode match {
2826
case "save" => saveModel(spark, path)
2927
case "load" => loadModel(spark, path)

0 commit comments

Comments
 (0)