Multiple input paths

Author: yatharthranjan

README.md

@@ -18,27 +18,27 @@ Build jar from source with
 and find the output JAR file as `build/libs/restructurehdfs-0.3.3-all.jar`. Then run with:
 
 ```shell
-java -jar restructurehdfs-0.3.3-all.jar --hdfs-uri <webhdfs_url> --hdfs-root-directory <hdfs_topic_path> --output-directory <output_folder>
+java -jar restructurehdfs-0.3.3-all.jar --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 or you can use the short form as well like - 
 ```shell
-java -jar restructurehdfs-0.3.3-all.jar -u <webhdfs_url> -i <hdfs_topic_path> -o <output_folder>
+java -jar restructurehdfs-0.3.3-all.jar -u <webhdfs_url> -o <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 
 To display the usage and all available options you can use the help option as follows - 
 ```shell
 java -jar restructurehdfs-0.3.3-all.jar --help
 ```
-Note that the options preceded by the `*` in the above output are required to run the app.
+Note that the options preceded by the `*` in the above output are required to run the app. Also note that there can be multiple input paths from which to read the files. Eg - `/topicAndroidNew/topic1 /topicAndroidNew/topic2 ...`. At least one input path is required.
 
 By default, this will output the data in CSV format. If JSON format is preferred, use the following instead:
 ```shell
-java -jar restructurehdfs-0.3.3-all.jar --format json --hdfs-uri <webhdfs_url> --hdfs-root-directory <hdfs_topic_path> --output-directory <output_folder>
+java -jar restructurehdfs-0.3.3-all.jar --format json --hdfs-uri <webhdfs_url> --output-directory <output_folder>  <input_path_1> [<input_path_2> ...]
 ```
 
 Another option is to output the data in compressed form. All files will get the `gz` suffix, and can be decompressed with a GZIP decoder. Note that for a very small number of records, this may actually increase the file size.
 ```
-java -jar restructurehdfs-0.3.3-all.jar --compression gzip  --hdfs-uri <webhdfs_url> --hdfs-root-directory <hdfs_topic_path> --output-directory <output_folder>
+java -jar restructurehdfs-0.3.3-all.jar --compression gzip  --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 
 Finally, by default, files records are not deduplicated after writing. To enable this behaviour, specify the option `--deduplicate` or `-d`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.

src/main/java/org/radarcns/RestructureAvroRecords.java

@@ -72,14 +72,14 @@ public class RestructureAvroRecords {
 
     private long processedFileCount;
     private long processedRecordsCount;
-    private boolean useGzip;
-    private boolean doDeduplicate;
-
-    private static final CommandLineArgs commandLineArgs = new CommandLineArgs();
-    private static final JCommander parser = JCommander.newBuilder().addObject(commandLineArgs).build();
+    private final boolean useGzip;
+    private final boolean doDeduplicate;
 
     public static void main(String [] args) throws Exception {
 
+        final CommandLineArgs commandLineArgs = new CommandLineArgs();
+        final JCommander parser = JCommander.newBuilder().addObject(commandLineArgs).build();
+
         parser.setProgramName("hadoop jar restructurehdfs-all-0.3.3.jar");
         parser.parse(args);
 
@@ -90,9 +90,6 @@ public static void main(String [] args) throws Exception {
 
         logger.info(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()));
         logger.info("Starting...");
-        logger.info("In:  " + commandLineArgs.hdfsUri + commandLineArgs.hdfsRootDirectory);
-        logger.info("Out: " + commandLineArgs.outputDirectory);
-
 
         long time1 = System.currentTimeMillis();
 
@@ -103,7 +100,11 @@ public static void main(String [] args) throws Exception {
                 .build();
 
         try {
-            restr.start(commandLineArgs.hdfsRootDirectory);
+            for(String input : commandLineArgs.inputPaths) {
+                logger.info("In:  " + commandLineArgs.hdfsUri + input);
+                logger.info("Out: " + commandLineArgs.outputDirectory);
+                restr.start(input);
+            }
         } catch (IOException ex) {
             logger.error("Processing failed", ex);
         }

src/main/java/org/radarcns/util/commandline/CommandLineArgs.java

@@ -2,8 +2,14 @@
 
 import com.beust.jcommander.Parameter;
 
+import java.util.ArrayList;
+import java.util.List;
+
 public class CommandLineArgs {
 
+    @Parameter(description = "<input_path_1> [<input_path_2> ...]", variableArity = true, required = true)
+    public List<String> inputPaths = new ArrayList<>();
+
     @Parameter(names = { "-f", "--format" }, description = "Format to use when converting the files. JSON and CSV is available.")
     public String format = "csv";
 
@@ -17,9 +23,6 @@ public class CommandLineArgs {
     @Parameter(names = { "-u", "--hdfs-uri" }, description = "The HDFS uri to connect to. Eg - 'hdfs://<HOST>:<RPC_PORT>/<PATH>'.", required = true, validateWith = { HdfsUriValidator.class, PathValidator.class })
     public String hdfsUri;
 
-    @Parameter(names = { "-i", "--hdfs-root-directory" }, description = "The input HDFS root directory from which files are to be read. Eg - '/topicAndroidNew'", required = true, validateWith = PathValidator.class)
-    public String hdfsRootDirectory;
-
     @Parameter(names = { "-o", "--output-directory"}, description = "The output folder where the files are to be extracted.", required = true, validateWith = PathValidator.class)
     public String outputDirectory;