Skip to content

Commit 57a9fc9

Browse files
committed
[SPARK-53832][K8S] Make KubernetesClientUtils Java-friendly
### What changes were proposed in this pull request? This PR aims to make `KubernetesClientUtils` more Java-friendly by adding three additional APIs. All the other public APIs are already Java-friendly. | Scala Version | New Java-friendly Version | | - | - | | `buildConfigMap` (Since 3.1.0) | `buildConfigMapJava` (Since 4.1.0) | | `buildKeyToPathObjects` (Since 3.1.0) | `buildKeyToPathObjectsJava` (Since 4.1.0) | | `buildSparkConfDirFilesMap` (Since 3.1.1) | `buildSparkConfDirFilesMapJava` (Since 4.1.0) | ### Why are the changes needed? Java-based downstream project like `Apache Spark K8s Operator` can take advantage of this improvement. ### Does this PR introduce _any_ user-facing change? No behavior change. ### How was this patch tested? Pass the CIs with the newly added test case. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #52542 from dongjoon-hyun/SPARK-53832. Authored-by: Dongjoon Hyun <[email protected]> Signed-off-by: Dongjoon Hyun <[email protected]>
1 parent ee15beb commit 57a9fc9

File tree

2 files changed

+53
-0
lines changed

2 files changed

+53
-0
lines changed

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtils.scala

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package org.apache.spark.deploy.k8s.submit
1919

2020
import java.io.{File, StringWriter}
2121
import java.nio.charset.MalformedInputException
22+
import java.util.{List => JList, Map => JMap}
2223
import java.util.Properties
2324

2425
import scala.collection.mutable
@@ -70,6 +71,18 @@ object KubernetesClientUtils extends Logging {
7071
propertiesWriter.toString
7172
}
7273

74+
/**
75+
* Build, file -> 'file's content' map of all the selected files in SPARK_CONF_DIR.
76+
* (Java-friendly)
77+
*/
78+
@Since("4.1.0")
79+
def buildSparkConfDirFilesMapJava(
80+
configMapName: String,
81+
sparkConf: SparkConf,
82+
resolvedPropertiesMap: JMap[String, String]): JMap[String, String] = synchronized {
83+
buildSparkConfDirFilesMap(configMapName, sparkConf, resolvedPropertiesMap.asScala.toMap).asJava
84+
}
85+
7386
/**
7487
* Build, file -> 'file's content' map of all the selected files in SPARK_CONF_DIR.
7588
*/
@@ -89,6 +102,11 @@ object KubernetesClientUtils extends Logging {
89102
}
90103
}
91104

105+
@Since("4.1.0")
106+
def buildKeyToPathObjectsJava(confFilesMap: JMap[String, String]): JList[KeyToPath] = {
107+
buildKeyToPathObjects(confFilesMap.asScala.toMap).asJava
108+
}
109+
92110
@Since("3.1.0")
93111
def buildKeyToPathObjects(confFilesMap: Map[String, String]): Seq[KeyToPath] = {
94112
confFilesMap.map {
@@ -98,6 +116,16 @@ object KubernetesClientUtils extends Logging {
98116
}.toList.sortBy(x => x.getKey) // List is sorted to make mocking based tests work
99117
}
100118

119+
/**
120+
* Build a ConfigMap that will hold the content for environment variable SPARK_CONF_DIR
121+
* on remote pods. (Java-friendly)
122+
*/
123+
@Since("4.1.0")
124+
def buildConfigMapJava(configMapName: String, confFileMap: JMap[String, String],
125+
withLabels: JMap[String, String]): ConfigMap = {
126+
buildConfigMap(configMapName, confFileMap.asScala.toMap, withLabels.asScala.toMap)
127+
}
128+
101129
/**
102130
* Build a Config Map that will hold the content for environment variable SPARK_CONF_DIR
103131
* on remote pods.

resource-managers/kubernetes/core/src/test/scala/org/apache/spark/deploy/k8s/submit/KubernetesClientUtilsSuite.scala

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package org.apache.spark.deploy.k8s.submit
2020
import java.io.File
2121
import java.nio.charset.StandardCharsets
2222
import java.nio.file.Files
23+
import java.util.{HashMap => JHashMap}
2324
import java.util.UUID
2425

2526
import scala.jdk.CollectionConverters._
@@ -103,4 +104,28 @@ class KubernetesClientUtilsSuite extends SparkFunSuite with BeforeAndAfter {
103104
.build()
104105
assert(outputConfigMap === expectedConfigMap)
105106
}
107+
108+
test("SPARK-53832: verify that configmap built as expected va Java-friendly APIs") {
109+
val configMapName = s"configmap-name-${UUID.randomUUID.toString}"
110+
val configMapNameSpace = s"configmap-namespace-${UUID.randomUUID.toString}"
111+
val properties = new JHashMap[String, String]()
112+
properties.put(Config.KUBERNETES_NAMESPACE.key, configMapNameSpace)
113+
val sparkConf =
114+
testSetup(properties.asScala.toMap.map(f => f._1 -> f._2.getBytes(StandardCharsets.UTF_8)))
115+
val confFileMap =
116+
KubernetesClientUtils.buildSparkConfDirFilesMapJava(configMapName, sparkConf, properties)
117+
val outputConfigMap =
118+
KubernetesClientUtils.buildConfigMapJava(configMapName, confFileMap, properties)
119+
val expectedConfigMap =
120+
new ConfigMapBuilder()
121+
.withNewMetadata()
122+
.withName(configMapName)
123+
.withNamespace(configMapNameSpace)
124+
.withLabels(properties)
125+
.endMetadata()
126+
.withImmutable(true)
127+
.addToData(confFileMap)
128+
.build()
129+
assert(outputConfigMap === expectedConfigMap)
130+
}
106131
}

0 commit comments

Comments
 (0)