Skip to content

Commit 0f2a38a

Browse files
shkhrgptakshayrai
shkhrgpt
authored and
akshayrai
committed
Updates Spark configuration heuristic severity calculations (#229)
1 parent fa166d3 commit 0f2a38a

File tree

2 files changed

+143
-26
lines changed

2 files changed

+143
-26
lines changed

‎app/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristic.scala‎

Lines changed: 39 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package com.linkedin.drelephant.spark.heuristics
1818

19+
import java.util.ArrayList
20+
1921
import com.linkedin.drelephant.math.Statistics
2022

2123
import scala.collection.JavaConverters
@@ -42,9 +44,6 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo
4244
Option(heuristicConfigurationData.getParamMap.get(SERIALIZER_IF_NON_NULL_RECOMMENDATION_KEY))
4345
.getOrElse(DEFAULT_SERIALIZER_IF_NON_NULL_RECOMMENDATION)
4446

45-
val serializerIfNonNullSeverityIfRecommendationUnmet: Severity =
46-
DEFAULT_SERIALIZER_IF_NON_NULL_SEVERITY_IF_RECOMMENDATION_UNMET
47-
4847
override def getHeuristicConfData(): HeuristicConfigurationData = heuristicConfigurationData
4948

5049
override def apply(data: SparkApplicationData): HeuristicResult = {
@@ -75,17 +74,27 @@ class ConfigurationHeuristic(private val heuristicConfigurationData: HeuristicCo
7574
evaluator.applicationDuration.toString + " Seconds"
7675
),
7776
new HeuristicResultDetails(
78-
SPARK_SERIALIZER_KEY,
79-
formatProperty(evaluator.serializer)
77+
SPARK_DYNAMIC_ALLOCATION_ENABLED,
78+
formatProperty(evaluator.isDynamicAllocationEnabled.map(_.toString))
8079
)
8180
)
81+
// Constructing a mutable ArrayList for resultDetails, otherwise addResultDetail method HeuristicResult cannot be used.
82+
val mutableResultDetailsArrayList = new ArrayList(resultDetails.asJava)
8283
val result = new HeuristicResult(
8384
heuristicConfigurationData.getClassName,
8485
heuristicConfigurationData.getHeuristicName,
8586
evaluator.severity,
8687
0,
87-
resultDetails.asJava
88+
mutableResultDetailsArrayList
8889
)
90+
if (evaluator.serializerSeverity != Severity.NONE) {
91+
result.addResultDetail(SPARK_SERIALIZER_KEY, formatProperty(evaluator.serializer),
92+
"KyroSerializer is Not Enabled.")
93+
}
94+
if (evaluator.shuffleAndDynamicAllocationSeverity != Severity.NONE) {
95+
result.addResultDetail(SPARK_SHUFFLE_SERVICE_ENABLED, formatProperty(evaluator.isShuffleServiceEnabled.map(_.toString)),
96+
"Spark shuffle service is not enabled.")
97+
}
8998
result
9099
}
91100
}
@@ -102,6 +111,8 @@ object ConfigurationHeuristic {
102111
val SPARK_EXECUTOR_CORES_KEY = "spark.executor.cores"
103112
val SPARK_SERIALIZER_KEY = "spark.serializer"
104113
val SPARK_APPLICATION_DURATION = "spark.application.duration"
114+
val SPARK_SHUFFLE_SERVICE_ENABLED = "spark.shuffle.service.enabled"
115+
val SPARK_DYNAMIC_ALLOCATION_ENABLED = "spark.dynamicAllocation.enabled"
105116

106117
class Evaluator(configurationHeuristic: ConfigurationHeuristic, data: SparkApplicationData) {
107118
lazy val appConfigurationProperties: Map[String, String] =
@@ -127,18 +138,34 @@ object ConfigurationHeuristic {
127138

128139
lazy val serializer: Option[String] = getProperty(SPARK_SERIALIZER_KEY)
129140

141+
/**
142+
* If the serializer is either not configured or not equal to KryoSerializer, then the severity will be moderate.
143+
*/
144+
130145
lazy val serializerSeverity: Severity = serializer match {
131-
case None => Severity.NONE
146+
case None => Severity.MODERATE
132147
case Some(`serializerIfNonNullRecommendation`) => Severity.NONE
133-
case Some(_) => serializerIfNonNullSeverityIfRecommendationUnmet
148+
case Some(_) => DEFAULT_SERIALIZER_IF_NON_NULL_SEVERITY_IF_RECOMMENDATION_UNMET
134149
}
135150

136-
lazy val severity: Severity = serializerSeverity
151+
/**
152+
* The following logic computes severity based on shuffle service and dynamic allocation flags.
153+
* If dynamic allocation is disabled, then the severity will be MODERATE if shuffle service is disabled or not specified.
154+
* If dynamic allocation is enabled, then the severity will be SEVERE if shuffle service is disabled or not specified.
155+
*/
137156

138-
private val serializerIfNonNullRecommendation: String = configurationHeuristic.serializerIfNonNullRecommendation
157+
lazy val isDynamicAllocationEnabled: Option[Boolean] = Some(getProperty(SPARK_DYNAMIC_ALLOCATION_ENABLED).exists(_.toBoolean == true))
158+
lazy val isShuffleServiceEnabled: Option[Boolean] = Some(getProperty(SPARK_SHUFFLE_SERVICE_ENABLED).exists(_.toBoolean == true))
159+
160+
lazy val shuffleAndDynamicAllocationSeverity = (isDynamicAllocationEnabled, isShuffleServiceEnabled) match {
161+
case (_, Some(true)) => Severity.NONE
162+
case (Some(false), Some(false)) => Severity.MODERATE
163+
case (Some(true), Some(false)) => Severity.SEVERE
164+
}
139165

140-
private val serializerIfNonNullSeverityIfRecommendationUnmet: Severity =
141-
configurationHeuristic.serializerIfNonNullSeverityIfRecommendationUnmet
166+
lazy val severity: Severity = Severity.max(serializerSeverity, shuffleAndDynamicAllocationSeverity)
167+
168+
private val serializerIfNonNullRecommendation: String = configurationHeuristic.serializerIfNonNullRecommendation
142169

143170
private def getProperty(key: String): Option[String] = appConfigurationProperties.get(key)
144171
}

‎test/com/linkedin/drelephant/spark/heuristics/ConfigurationHeuristicTest.scala‎

Lines changed: 104 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -41,20 +41,26 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers {
4141

4242
val configurationHeuristic = new ConfigurationHeuristic(heuristicConfigurationData)
4343

44-
describe(".apply") {
44+
describe("apply with NO Severity") {
4545
val configurationProperties = Map(
4646
"spark.serializer" -> "org.apache.spark.serializer.KryoSerializer",
4747
"spark.storage.memoryFraction" -> "0.3",
4848
"spark.driver.memory" -> "2G",
4949
"spark.executor.instances" -> "900",
5050
"spark.executor.memory" -> "1g",
51-
"spark.shuffle.memoryFraction" -> "0.5"
51+
"spark.shuffle.memoryFraction" -> "0.5",
52+
"spark.shuffle.service.enabled" -> "true",
53+
"spark.dynamicAllocation.enabled" -> "true"
5254
)
5355

5456
val data = newFakeSparkApplicationData(configurationProperties)
5557
val heuristicResult = configurationHeuristic.apply(data)
5658
val heuristicResultDetails = heuristicResult.getHeuristicResultDetails
5759

60+
it("returns the size of result details") {
61+
heuristicResultDetails.size() should be(6)
62+
}
63+
5864
it("returns the severity") {
5965
heuristicResult.getSeverity should be(Severity.NONE)
6066
}
@@ -89,10 +95,50 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers {
8995
details.getValue should include("10")
9096
}
9197

92-
it("returns the serializer") {
98+
it("returns the dynamic allocation flag") {
99+
val details = heuristicResultDetails.get(5)
100+
details.getName should include("spark.dynamicAllocation.enabled")
101+
details.getValue should be("true")
102+
}
103+
}
104+
105+
describe("apply with Severity") {
106+
val configurationProperties = Map(
107+
"spark.serializer" -> "dummySerializer",
108+
"spark.shuffle.service.enabled" -> "false",
109+
"spark.dynamicAllocation.enabled" -> "true"
110+
)
111+
112+
val data = newFakeSparkApplicationData(configurationProperties)
113+
val heuristicResult = configurationHeuristic.apply(data)
114+
val heuristicResultDetails = heuristicResult.getHeuristicResultDetails
115+
116+
it("returns the size of result details") {
117+
heuristicResultDetails.size() should be(8)
118+
}
119+
120+
it("returns the severity") {
121+
heuristicResult.getSeverity should be(Severity.SEVERE)
122+
}
123+
124+
it("returns the dynamic allocation flag") {
93125
val details = heuristicResultDetails.get(5)
126+
details.getName should include("spark.dynamicAllocation.enabled")
127+
details.getValue should be("true")
128+
}
129+
130+
it("returns the serializer") {
131+
val details = heuristicResultDetails.get(6)
94132
details.getName should include("spark.serializer")
95-
details.getValue should be("org.apache.spark.serializer.KryoSerializer")
133+
details.getValue should be("dummySerializer")
134+
details.getDetails should be("KyroSerializer is Not Enabled.")
135+
}
136+
137+
it("returns the shuffle service flag") {
138+
val details = heuristicResultDetails.get(7)
139+
details.getName should include("spark.shuffle.service.enabled")
140+
details.getValue should be("false")
141+
details.getDetails should be("Spark shuffle service is not enabled.")
96142
}
97143
}
98144

@@ -148,34 +194,78 @@ class ConfigurationHeuristicTest extends FunSpec with Matchers {
148194
evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer"))
149195
}
150196

151-
it("has no serializer when it's absent") {
197+
it("has no serializer, dynamic allocation flag, and shuffle flag when they are absent") {
152198
val evaluator = newEvaluatorWithConfigurationProperties(Map.empty)
153199
evaluator.serializer should be(None)
200+
evaluator.isDynamicAllocationEnabled should be(Some(false))
201+
evaluator.isShuffleServiceEnabled should be(Some(false))
202+
evaluator.serializerSeverity should be(Severity.MODERATE)
203+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE)
204+
evaluator.severity should be(Severity.MODERATE)
154205
}
155206

156-
it("has the severity of the serializer setting when it matches our recommendation") {
207+
it("has no dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") {
157208
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer"))
209+
evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer"))
210+
evaluator.isDynamicAllocationEnabled should be(Some(false))
211+
evaluator.isShuffleServiceEnabled should be(Some(false))
158212
evaluator.serializerSeverity should be(Severity.NONE)
213+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE)
214+
evaluator.severity should be(Severity.MODERATE)
159215
}
160216

161-
it("has the severity of the serializer setting when it doesn't match our recommendation and is non-null") {
217+
it("has no dynamic allocation flag and shuffle flag, and serializer setting doesn't match our recommendation and is non-null") {
162218
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.FooSerializer"))
219+
evaluator.serializer should be(Some("org.apache.spark.serializer.FooSerializer"))
220+
evaluator.isDynamicAllocationEnabled should be(Some(false))
221+
evaluator.isShuffleServiceEnabled should be(Some(false))
163222
evaluator.serializerSeverity should be(Severity.MODERATE)
223+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE)
224+
evaluator.severity should be(Severity.MODERATE)
164225
}
165226

166-
it("has the severity of the serializer setting when it is null") {
167-
val evaluator = newEvaluatorWithConfigurationProperties(Map.empty)
227+
it("true dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") {
228+
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer",
229+
"spark.shuffle.service.enabled" -> "true", "spark.dynamicAllocation.enabled" -> "true"))
230+
evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer"))
231+
evaluator.isDynamicAllocationEnabled should be(Some(true))
232+
evaluator.isShuffleServiceEnabled should be(Some(true))
168233
evaluator.serializerSeverity should be(Severity.NONE)
234+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.NONE)
235+
evaluator.severity should be(Severity.NONE)
169236
}
170237

171-
it("computes the overall severity when there are some issues") {
172-
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.FooSerializer"))
238+
it("true dynamic allocation flag and shuffle flag, and serializer setting is absent") {
239+
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.shuffle.service.enabled" -> "true",
240+
"spark.dynamicAllocation.enabled" -> "true"))
241+
evaluator.serializer should be(None)
242+
evaluator.isDynamicAllocationEnabled should be(Some(true))
243+
evaluator.isShuffleServiceEnabled should be(Some(true))
244+
evaluator.serializerSeverity should be(Severity.MODERATE)
245+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.NONE)
173246
evaluator.severity should be(Severity.MODERATE)
174247
}
175248

176-
it("computes the overall severity when there are no issues") {
177-
val evaluator = newEvaluatorWithConfigurationProperties(Map.empty)
178-
evaluator.severity should be(Severity.NONE)
249+
it("true dynamic allocation flag and false shuffle flag, and serializer setting matches our recommendation") {
250+
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer",
251+
"spark.shuffle.service.enabled" -> "false", "spark.dynamicAllocation.enabled" -> "true"))
252+
evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer"))
253+
evaluator.isDynamicAllocationEnabled should be(Some(true))
254+
evaluator.isShuffleServiceEnabled should be(Some(false))
255+
evaluator.serializerSeverity should be(Severity.NONE)
256+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.SEVERE)
257+
evaluator.severity should be(Severity.SEVERE)
258+
}
259+
260+
it("false dynamic allocation flag and shuffle flag, and serializer setting matches our recommendation") {
261+
val evaluator = newEvaluatorWithConfigurationProperties(Map("spark.serializer" -> "org.apache.spark.serializer.KryoSerializer",
262+
"spark.shuffle.service.enabled" -> "false", "spark.dynamicAllocation.enabled" -> "false"))
263+
evaluator.serializer should be(Some("org.apache.spark.serializer.KryoSerializer"))
264+
evaluator.isDynamicAllocationEnabled should be(Some(false))
265+
evaluator.isShuffleServiceEnabled should be(Some(false))
266+
evaluator.serializerSeverity should be(Severity.NONE)
267+
evaluator.shuffleAndDynamicAllocationSeverity should be(Severity.MODERATE)
268+
evaluator.severity should be(Severity.MODERATE)
179269
}
180270
}
181271
}

0 commit comments

Comments
 (0)