Skip to content

Commit 13c5c1f

Browse files
wangyumsrowen
authored andcommitted
[SPARK-27180][BUILD][YARN] Fix testing issues with yarn module in Hadoop-3
## What changes were proposed in this pull request? Fix testing issues with `yarn` module in Hadoop-3: 1. Upgrade jersey-1 to `1.19` to fix ```Cause: java.lang.NoClassDefFoundError: com/sun/jersey/spi/container/servlet/ServletContainer```. 2. Copy `ServerSocketUtil` from hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java to fix ```java.lang.NoClassDefFoundError: org/apache/hadoop/net/ServerSocketUtil```. 3. Adapte `SessionHandler` from jetty-9.3.25.v20180904/jetty-server/src/main/java/org/eclipse/jetty/server/session/SessionHandler.java to fix ```java.lang.NoSuchMethodError: org.eclipse.jetty.server.session.SessionHandler.getSessionManager()Lorg/eclipse/jetty/server/SessionManager```. ## How was this patch tested? manual tests: ```shell build/sbt yarn/test -Pyarn build/sbt yarn/test -Phadoop-3.2 -Pyarn build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl resource-managers/yarn test -Pyarn build/mvn -Dtest=none -DwildcardSuites=org.apache.spark.deploy.yarn.YarnClusterSuite -pl resource-managers/yarn test -Pyarn -Phadoop-3.2 ``` Closes apache#24115 from wangyum/hadoop3-yarn. Authored-by: Yuming Wang <[email protected]> Signed-off-by: Sean Owen <[email protected]>
1 parent 57aff93 commit 13c5c1f

File tree

5 files changed

+521
-1
lines changed

5 files changed

+521
-1
lines changed

dev/.rat-excludes

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,3 +115,5 @@ structured-streaming/*
115115
kafka-source-initial-offset-version-2.1.0.bin
116116
kafka-source-initial-offset-future-version.bin
117117
vote.tmpl
118+
SessionManager.java
119+
SessionHandler.java

resource-managers/yarn/pom.xml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@
2929
<name>Spark Project YARN</name>
3030
<properties>
3131
<sbt.project.name>yarn</sbt.project.name>
32-
<jersey-1.version>1.9</jersey-1.version>
32+
<jersey-1.version>1.19</jersey-1.version>
3333
</properties>
3434

3535
<dependencies>
@@ -166,6 +166,12 @@
166166
<scope>test</scope>
167167
<version>${jersey-1.version}</version>
168168
</dependency>
169+
<dependency>
170+
<groupId>com.sun.jersey</groupId>
171+
<artifactId>jersey-servlet</artifactId>
172+
<scope>test</scope>
173+
<version>${jersey-1.version}</version>
174+
</dependency>
169175

170176
<!-- These dependencies are duplicated from core, because dependencies in the "provided"
171177
scope are not transitive.-->
Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.hadoop.net;
19+
20+
import org.slf4j.Logger;
21+
import org.slf4j.LoggerFactory;
22+
23+
import java.io.IOException;
24+
import java.net.ServerSocket;
25+
import java.util.Random;
26+
27+
/**
28+
* Copied from
29+
* hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
30+
* for Hadoop-3.x testing
31+
*/
32+
public class ServerSocketUtil {
33+
34+
private static final Logger LOG = LoggerFactory.getLogger(ServerSocketUtil.class);
35+
private static Random rand = new Random();
36+
37+
/**
38+
* Port scan & allocate is how most other apps find ports
39+
*
40+
* @param port given port
41+
* @param retries number of retries
42+
* @return
43+
* @throws IOException
44+
*/
45+
public static int getPort(int port, int retries) throws IOException {
46+
int tryPort = port;
47+
int tries = 0;
48+
while (true) {
49+
if (tries > 0 || tryPort == 0) {
50+
tryPort = port + rand.nextInt(65535 - port);
51+
}
52+
if (tryPort == 0) {
53+
continue;
54+
}
55+
try (ServerSocket s = new ServerSocket(tryPort)) {
56+
LOG.info("Using port " + tryPort);
57+
return tryPort;
58+
} catch (IOException e) {
59+
tries++;
60+
if (tries >= retries) {
61+
LOG.info("Port is already in use; giving up");
62+
throw e;
63+
} else {
64+
LOG.info("Port is already in use; trying again");
65+
}
66+
}
67+
}
68+
}
69+
70+
/**
71+
* Check whether port is available or not.
72+
*
73+
* @param port given port
74+
* @return
75+
*/
76+
private static boolean isPortAvailable(int port) {
77+
try (ServerSocket s = new ServerSocket(port)) {
78+
return true;
79+
} catch (IOException e) {
80+
return false;
81+
}
82+
}
83+
84+
/**
85+
* Wait till the port available.
86+
*
87+
* @param port given port
88+
* @param retries number of retries for given port
89+
* @return
90+
* @throws InterruptedException
91+
* @throws IOException
92+
*/
93+
public static int waitForPort(int port, int retries)
94+
throws InterruptedException, IOException {
95+
int tries = 0;
96+
while (true) {
97+
if (isPortAvailable(port)) {
98+
return port;
99+
} else {
100+
tries++;
101+
if (tries >= retries) {
102+
throw new IOException(
103+
"Port is already in use; giving up after " + tries + " times.");
104+
}
105+
Thread.sleep(1000);
106+
}
107+
}
108+
}
109+
110+
/**
111+
* Find the specified number of unique ports available.
112+
* The ports are all closed afterwards,
113+
* so other network services started may grab those same ports.
114+
*
115+
* @param numPorts number of required port nubmers
116+
* @return array of available port numbers
117+
* @throws IOException
118+
*/
119+
public static int[] getPorts(int numPorts) throws IOException {
120+
ServerSocket[] sockets = new ServerSocket[numPorts];
121+
int[] ports = new int[numPorts];
122+
for (int i = 0; i < numPorts; i++) {
123+
ServerSocket sock = new ServerSocket(0);
124+
sockets[i] = sock;
125+
ports[i] = sock.getLocalPort();
126+
}
127+
for (ServerSocket sock : sockets) {
128+
sock.close();
129+
}
130+
return ports;
131+
}
132+
}

0 commit comments

Comments
 (0)