Jar compile and command line arguments

Author: Maxim Moinat

Readme.md

@@ -1,10 +1,4 @@
+To be run with:
 ```
-scp src/main/org.radarcns.restructureAvroSensorTopics.java [email protected]:/home/maxim/hadoop-test/
-
-javac -classpath "/opt/hadoop/share/hadoop/common/hadoop-common-2.7.1.jar:/opt/hadoop/share/hadoop/avro-tools/avro-tools-1.7.7.jar" -d wordcount_classes org.radarcns.restructureAvroSensorTopics.java
-
-
-jar -cvf restructureAvroSensorTopics.jar -C wordcount_classes/ .
-
-hadoop jar restructureAvroSensorTopics.jar org.radarcns.org.radarcns.restructureAvroSensorTopics
+hadoop jar restructurehdfs-all-0.1.0.jar webhdfs://radar-test.thehyve.net:50070 /topicAndroidPhoneNew/ output2
 ```

build.gradle

@@ -2,6 +2,7 @@ group 'org.radarcns.restructurehdfs'
 version '1.0-SNAPSHOT'
 
 apply plugin: 'java'
+version = '0.1.0'
 
 sourceCompatibility = 1.8
 
@@ -16,4 +17,16 @@ dependencies {
     compile 'org.apache.avro:avro:1.7.7'
     compile 'org.apache.avro:avro-mapred:1.7.7'
     compile 'com.fasterxml.jackson.core:jackson-core:2.8.7'
+}
+
+//create a single Jar with all dependencies
+task fatJar(type: Jar) {
+    manifest {
+        attributes 'Implementation-Title': 'Gradle Jar File Example',
+                'Implementation-Version': version,
+                'Main-Class': 'org.radarcns.restructureAvroRecords'
+    }
+    baseName = project.name + '-all'
+    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+    with jar
 }

src/main/java/org/radarcns/restructureAvroSensorTopics.java renamed to src/main/java/org/radarcns/restructureAvroRecords.java

@@ -16,7 +16,7 @@
 import java.text.SimpleDateFormat;
 import java.util.*;
 
-public class restructureAvroSensorTopics {
+public class restructureAvroRecords {
 
     String outputPath = ".";
     String OUTPUT_FILE_EXTENSION = "json";
@@ -25,16 +25,15 @@ public class restructureAvroSensorTopics {
     SimpleDateFormat dateFormatFileName = new SimpleDateFormat("yyyyMMdd_HH");
 
     public static void main(String [] args) throws Exception {
-        System.out.println("Started!");
 
-        restructureAvroSensorTopics res = new restructureAvroSensorTopics();
-        res.setInputWebHdfsURL("webhdfs://radar-test.thehyve.net:50070");
-        res.setOutputPath("output3/");
+        restructureAvroRecords restr = new restructureAvroRecords();
+        restr.setInputWebHdfsURL(args[0]); //"webhdfs://radar-test.thehyve.net:50070");
+        restr.setOutputPath(args[2]); //"output3/");
 
-        res.start("/topicAndroidPhoneNew/");
-//        restructureAvroSensorTopics.processTopic("/topicE4/android_empatica_e4_inter_beat_interval/partition=0/");
-//        restructureAvroSensorTopics.processAvroFile(new Path("/topicE4/android_empatica_e4_inter_beat_interval/partition=0/android_empatica_e4_inter_beat_interval+0+0000031485+0000031488.avro") );
-//        restructureAvroSensorTopics.processAvroFile(new Path("/testE4Time/android_phone_acceleration/partition=0/android_phone_acceleration+0+0000590000+0000599999.avro"),"wazaa" );
+        restr.start(args[1]); //"/topicAndroidPhoneNew/");
+//        restructureAvroRecords.processTopic("/topicE4/android_empatica_e4_inter_beat_interval/partition=0/");
+//        restructureAvroRecords.processAvroFile(new Path("/topicE4/android_empatica_e4_inter_beat_interval/partition=0/android_empatica_e4_inter_beat_interval+0+0000031485+0000031488.avro") );
+//        restructureAvroRecords.processAvroFile(new Path("/testE4Time/android_phone_acceleration/partition=0/android_phone_acceleration+0+0000590000+0000599999.avro"),"wazaa" );
     }
 
     public void setInputWebHdfsURL(String fileSystemURL) {
@@ -74,18 +73,13 @@ public void processTopic(Path topicPath) throws IOException {
 
         String topicName = topicPath.getName();
 
-        int fileCounter = 0;
         while (files.hasNext()) {
             LocatedFileStatus locatedFileStatus = files.next();
 
             System.out.println(locatedFileStatus.getPath());
 
             if (locatedFileStatus.isFile())
                 this.processAvroFile( locatedFileStatus.getPath(), topicName );
-            fileCounter += 1;
-
-            if (fileCounter > 1)
-                continue;
         }
     }
 
@@ -119,7 +113,7 @@ public void writeRecord(GenericRecord record, String topicName) throws IOExcepti
         GenericRecord valueField = (GenericRecord) record.get("valueField");
 
         // Make a timestamped filename YYYYMMDD_HH00.json
-        String outputFileName = this.createFilepathFromTimestamp( (Double) valueField.get("time"));
+        String outputFileName = this.createFilePathFromTimestamp( (Double) valueField.get("time"));
 
         // Clean user id and create final output pathname
         String userId = keyField.get("userId").toString().replaceAll("\\W+", "");
@@ -130,7 +124,7 @@ public void writeRecord(GenericRecord record, String topicName) throws IOExcepti
         this.appendToFile(dirName, outputFileName, data);
     }
 
-    public String createFilepathFromTimestamp(Double time) {
+    public String createFilePathFromTimestamp(Double time) {
         // Send all output to the Appendable object sb
         StringBuilder sb = new StringBuilder();
         Formatter formatter = new Formatter(sb, Locale.US);