File tree Expand file tree Collapse file tree 1 file changed +5
-1
lines changed
client-spark/spark3/src/main/java/org/apache/spark/shuffle Expand file tree Collapse file tree 1 file changed +5
-1
lines changed Original file line number Diff line number Diff line change @@ -123,7 +123,11 @@ private boolean tryAccessCluster() {
123123 Map <String , String > extraProperties = Maps .newHashMap ();
124124 extraProperties .put (
125125 ACCESS_INFO_REQUIRED_SHUFFLE_NODES_NUM , String .valueOf (assignmentShuffleNodesNum ));
126- extraProperties .putAll (RssSparkConfig .sparkConfToMap (sparkConf ));
126+ // Put all spark conf into extra properties, except which length is longer than 100
127+ // to avoid extra properties too long.
128+ RssSparkConfig .toRssConf (sparkConf ).getAll ().stream ()
129+ .filter (entry -> StringUtils .length ((String ) entry .getValue ()) < 100 )
130+ .forEach (entry -> extraProperties .put (entry .getKey (), (String ) entry .getValue ()));
127131
128132 Set <String > assignmentTags = RssSparkShuffleUtils .getAssignmentTags (sparkConf );
129133 try {
You can’t perform that action at this time.
0 commit comments