Skip to content

Commit 8b33485

Browse files
committed
Fix the Spark session creating issue of Spark 2.3
Zero executor isn't allowed in Spark 2.3 Signed-off-by: Wei Zhang <[email protected]>
1 parent 64f3d7d commit 8b33485

File tree

1 file changed

+33
-0
lines changed
  • Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/livy/interactive

1 file changed

+33
-0
lines changed

Utils/hdinsight-node-common/src/com/microsoft/azure/hdinsight/sdk/common/livy/interactive/Session.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,12 @@ public abstract class Session implements AutoCloseable, Closeable {
7474
@Nullable
7575
private List<String> lastLogs; // Last session logs
7676

77+
private int executorCores = 1; // Default cores per executor to create
78+
79+
private int executorNum = 2; // Default executor count to create
80+
81+
private int driverCores = 2; // Default driver cores to create
82+
7783
/*
7884
* Constructor
7985
*/
@@ -121,6 +127,30 @@ public int getId() throws SessionNotStartException {
121127
return id;
122128
}
123129

130+
public void setExecutorCores(int executorCores) {
131+
this.executorCores = executorCores;
132+
}
133+
134+
public int getExecutorCores() {
135+
return executorCores;
136+
}
137+
138+
public int getExecutorNum() {
139+
return executorNum;
140+
}
141+
142+
public void setExecutorNum(int executorNum) {
143+
this.executorNum = executorNum;
144+
}
145+
146+
public int getDriverCores() {
147+
return driverCores;
148+
}
149+
150+
public void setDriverCores(int driverCores) {
151+
this.driverCores = driverCores;
152+
}
153+
124154
public Observable<String> getAppId() {
125155
return appId != null ?
126156
Observable.just(appId) :
@@ -243,6 +273,9 @@ private PostSessions preparePostSessions() {
243273
PostSessions postBody = new PostSessions();
244274
postBody.setName(getName());
245275
postBody.setKind(getKind());
276+
postBody.setExecutorCores(getExecutorCores());
277+
postBody.setNumExecutors(getExecutorNum());
278+
postBody.setDriverCores(getDriverCores());
246279

247280
return postBody;
248281
}

0 commit comments

Comments
 (0)