High availability setup

Author: blootsvoets

src/main/java/org/radarcns/hdfs/RestructureAvroRecords.java

@@ -124,16 +124,16 @@ public static void main(String [] args) {
 
             builder.putHdfsConfig("dfs.nameservices", commandLineArgs.hdfsUri);
             builder.putHdfsConfig("dfs.ha.namenodes." + commandLineArgs.hdfsUri, commandLineArgs.hdfsHa);
-            builder.putHdfsConfig("dfs.namenode.rpc-address." + commandLineArgs.hdfsUri + "." + haNames[0], commandLineArgs.hdfsUri1);
-            builder.putHdfsConfig("dfs.namenode.rpc-address." + commandLineArgs.hdfsUri + "." + haNames[1], commandLineArgs.hdfsUri2);
+            builder.putHdfsConfig("dfs.namenode.rpc-address." + commandLineArgs.hdfsUri + "." + haNames[0], commandLineArgs.hdfsUri1 + ":8020");
+            builder.putHdfsConfig("dfs.namenode.rpc-address." + commandLineArgs.hdfsUri + "." + haNames[1], commandLineArgs.hdfsUri2 + ":8020");
             builder.putHdfsConfig("dfs.client.failover.proxy.provider." + commandLineArgs.hdfsUri,"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider");
         }
 
         RestructureAvroRecords restr = builder.build();
 
         try {
             for(String input : commandLineArgs.inputPaths) {
-                logger.info("In:  " + commandLineArgs.hdfsUri + input);
+                logger.info("In:  hdfs://" + commandLineArgs.hdfsUri + input);
                 logger.info("Out: " + commandLineArgs.outputDirectory);
                 restr.start(input);
             }
@@ -177,7 +177,6 @@ private RestructureAvroRecords(RestructureAvroRecords.Builder builder) {
 
     public void setInputWebHdfsURL(String fileSystemURL) {
         conf.set("fs.defaultFS", "hdfs://" + fileSystemURL);
-        conf.set("fs.default.name", conf.get("fs.defaultFS"));
     }
 
     public void setOutputPath(String path) {

src/main/java/org/radarcns/hdfs/util/commandline/CommandLineArgs.java

@@ -23,10 +23,10 @@ public class CommandLineArgs {
     @Parameter(names = { "-n", "--nameservice"}, description = "The HDFS name services to connect to. Eg - '<HOST>' for single configurations or <CLUSTER_ID> for high availability web services.", required = true, validateWith = { PathValidator.class })
     public String hdfsUri;
 
-    @Parameter(names = { "--namenode-1" }, description = "High availability HDFS name node to also connect to. Eg - '<HOST>:<RPC_PORT>'.", validateWith = { PathValidator.class })
+    @Parameter(names = { "--namenode-1" }, description = "High availability HDFS first name node hostname.", validateWith = { PathValidator.class })
     public String hdfsUri1;
 
-    @Parameter(names = { "--namenode-2" }, description = "High availability HDFS name node to also connect to. Eg - '<HOST>:<RPC_PORT>'.", validateWith = { PathValidator.class })
+    @Parameter(names = { "--namenode-2" }, description = "High availability HDFS second name node hostname. Eg - '<HOST>'.", validateWith = { PathValidator.class })
     public String hdfsUri2;
 
     @Parameter(names = { "--namenode-ha"}, description = "High availability HDFS name node names. Eg - 'nn1,nn2'.", validateWith = { PathValidator.class })