Skip to content

Commit f538b56

Browse files
committed
Address suggestion, remove the spark prefix to be common
1 parent 827ee04 commit f538b56

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

client-spark/spark3/src/main/java/org/apache/spark/shuffle/DelegationRssShuffleManager.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,11 @@ private boolean tryAccessCluster() {
123123
Map<String, String> extraProperties = Maps.newHashMap();
124124
extraProperties.put(
125125
ACCESS_INFO_REQUIRED_SHUFFLE_NODES_NUM, String.valueOf(assignmentShuffleNodesNum));
126-
extraProperties.putAll(RssSparkConfig.sparkConfToMap(sparkConf));
126+
// Put all spark conf into extra properties, except which length is longer than 100
127+
// to avoid extra properties too long.
128+
RssSparkConfig.toRssConf(sparkConf).getAll().stream()
129+
.filter(entry -> StringUtils.length((String) entry.getValue()) < 100)
130+
.forEach(entry -> extraProperties.put(entry.getKey(), (String) entry.getValue()));
127131

128132
Set<String> assignmentTags = RssSparkShuffleUtils.getAssignmentTags(sparkConf);
129133
try {

0 commit comments

Comments
 (0)