NoSuchMethodError #44
Description
I am getting a NoSuchMethodError even though I have
in the path at the top. I know this is the conflict of classpaths, but cannot figure out where the conflict is
[task-result-getter-0] WARN org.apache.spark.scheduler.TaskSetManager - Lost task 1.0 in stage 0.0 (TID 1, hdfsu10.almaden.ibm.com): java.lang.NoSuchMethodError: org.apache.http.conn.ssl.SSLConnectionSocketFactory.(Ljavax/net/ssl/SSLContext;Ljavax/net/ssl/HostnameVerifier;)V
at com.amazonaws.http.conn.ssl.SdkTLSSocketFactory.(SdkTLSSocketFactory.java:56)
at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.getPreferredSocketFactory(ApacheConnectionManagerFactory.java:92)
at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.create(ApacheConnectionManagerFactory.java:65)
at com.amazonaws.http.apache.client.impl.ApacheConnectionManagerFactory.create(ApacheConnectionManagerFactory.java:58)
at com.amazonaws.http.apache.client.impl.ApacheHttpClientFactory.create(ApacheHttpClientFactory.java:50)
at com.amazonaws.http.apache.client.impl.ApacheHttpClientFactory.create(ApacheHttpClientFactory.java:38)
at com.amazonaws.http.AmazonHttpClient.(AmazonHttpClient.java:315)
at com.amazonaws.http.AmazonHttpClient.(AmazonHttpClient.java:299)
at com.amazonaws.AmazonWebServiceClient.(AmazonWebServiceClient.java:164)
at com.amazonaws.services.s3.AmazonS3Client.(AmazonS3Client.java:570)
at com.amazonaws.services.s3.AmazonS3Client.(AmazonS3Client.java:550)
Here is my code snippet
def main(args: Array[String]) {
val accessKey = "xxx"
val secretKey = "xxx"
val endPoint = "s3-api.us-geo.objectstorage.softlayer.net"
val nameNodeIP = "hdfs://9.1.70.248:9000";
val ressourceManagerIP = "spark://9.1.70.248:7077";
val lastJar = "/home/malladip/stocator-test.jar";
val jars = Array(lastJar,"lib/httpclient-4.5.2.jar","lib/aws-java-sdk-1.11.236.jar","lib/stocator-1.0.11-SNAPSHOT.jar","lib/guava-19.0.0.jar");
val (sc, conf) = initializeSpark(nameNodeIP, ressourceManagerIP, jars, false, "StocatorTest");
var hconf = sc.hadoopConfiguration;
hconf.set("fs.cos.impl", "com.ibm.stocator.fs.ObjectStoreFileSystem")
hconf.set("fs.stocator.scheme.list", "cos")
hconf.set("fs.stocator.cos.impl", "com.ibm.stocator.fs.cos.COSAPIClient")
hconf.set("fs.stocator.cos.scheme", "cos")
hconf.set("fs.cos.mycos.access.key", accessKey)
hconf.set("fs.cos.mycos.endpoint", endPoint)
hconf.set("fs.cos.mycos.secret.key", secretKey)
hconf.set("fs.cos.myCos.v2.signer.type","false")
val data = Array(1, 2, 3, 4, 5, 6, 7, 8)
val distData = sc.parallelize(data)
distData.saveAsTextFile("cos://mocktestdata.mycos/one1.txt")
}