Merge pull request #29 from RADAR-base/release-0.4.0

Release 0.4.0

Author: blootsvoets

.dockerignore

@@ -0,0 +1,6 @@
+.git/
+build/
+src/test/
+out/
+.gradle/
+.idea/

Dockerfile

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-alpine AS builder
+
+RUN mkdir /code
+WORKDIR /code
+
+ENV GRADLE_OPTS -Dorg.gradle.daemon=false
+
+COPY ./gradle /code/gradle
+COPY ./gradlew /code/
+RUN ./gradlew --version
+
+COPY ./build.gradle ./gradle.properties ./settings.gradle /code/
+
+RUN ./gradlew downloadDependencies copyDependencies startScripts
+
+COPY ./src /code/src
+
+RUN ./gradlew jar
+
+FROM openjdk:8-jre-alpine
+
+MAINTAINER Joris Borgdorff <[email protected]>
+
+LABEL description="RADAR-base HDFS data restructuring"
+
+COPY --from=builder /code/build/third-party/* /usr/lib/
+COPY --from=builder /code/build/scripts/* /usr/bin/
+COPY --from=builder /code/build/libs/* /usr/lib/
+
+ENTRYPOINT ["radar-hdfs-restructure"]

README.md

@@ -2,33 +2,61 @@
 
 [![Build Status](https://travis-ci.org/RADAR-base/Restructure-HDFS-topic.svg?branch=master)](https://travis-ci.org/RADAR-base/Restructure-HDFS-topic)
 
-Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/RADAR-CNS/RADAR-HDFS-Sink-Connector) is streamed to files based on sensor only. This package can transform that output to a local directory structure as follows: `userId/topic/date_hour.csv`. The date and hour is extracted from the `time` field of each record, and is formatted in UTC time.
+Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/RADAR-CNS/RADAR-HDFS-Sink-Connector) is streamed to files based on sensor only. This package can transform that output to a local directory structure as follows: `userId/topic/date_hour.csv`. The date and hour is extracted from the `time` field of each record, and is formatted in UTC time. This package is included in the [RADAR-Docker](https://github.com/RADAR-CNS/RADAR-Docker) repository, in the `dcompose/radar-cp-hadoop-stack/hdfs_restructure.sh` script.
 
-## Usage
+## Docker usage
+
+This package is available as docker image [`radarbase/radar-hdfs-restructure`](https://hub.docker.com/r/radarbase/radar-hdfs-restructure). The entrypoint of the image is the current application. So in all of the commands listed in usage, replace `radar-hdfs-restructure` with for example:
+```shell
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.4.0 -u hdfs://hdfs -o /output /myTopic
+```
+if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
 
-This package is included in the [RADAR-Docker](https://github.com/RADAR-CNS/RADAR-Docker) repository, in the `dcompose/radar-cp-hadoop-stack/hdfs_restructure.sh` script.
 
-## Advanced usage
+## Local build
 
-Build jar from source with
+This package requires at least Java JDK 8. Build the distribution with
 
 ```shell
 ./gradlew build
 ```
-and find the output JAR file as `build/libs/restructurehdfs-0.3.1-all.jar`. Then run with:
 
+and install the package into `/usr/local` with for example
 ```shell
-java -jar restructurehdfs-0.3.1-all.jar <webhdfs_url> <hdfs_topic_path> <output_folder>
+sudo mkdir -p /usr/local
+sudo tar -xzf build/distributions/radar-hdfs-restructure-0.4.0.tar.gz -C /usr/local --strip-components=1
 ```
 
-By default, this will output the data in CSV format. If JSON format is preferred, use the following instead:
+Now the `radar-hdfs-restructure` command should be available.
+
+## Command line usage
+
+When the application is installed, it can be used as follows:
+
+```shell
+radar-hdfs-restructure --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
-java -Dorg.radarcns.format=json -jar restructurehdfs-0.3.1-all.jar <webhdfs_url> <hdfs_topic_path> <output_folder>
+or you can use the short form as well like - 
+```shell
+radar-hdfs-restructure -u <webhdfs_url> -o <output_folder> <input_path_1> [<input_path_2> ...]
+```
+
+To display the usage and all available options you can use the help option as follows - 
+```shell
+radar-hdfs-restructure --help
+```
+Note that the options preceded by the `*` in the above output are required to run the app. Also note that there can be multiple input paths from which to read the files. Eg - `/topicAndroidNew/topic1 /topicAndroidNew/topic2 ...`. At least one input path is required.
+
+By default, this will output the data in CSV format. If JSON format is preferred, use the following instead:
+```shell
+radar-hdfs-restructure --format json --hdfs-uri <webhdfs_url> --output-directory <output_folder>  <input_path_1> [<input_path_2> ...]
 ```
 
 Another option is to output the data in compressed form. All files will get the `gz` suffix, and can be decompressed with a GZIP decoder. Note that for a very small number of records, this may actually increase the file size.
 ```
-java -Dorg.radarcns.compress=gzip -jar restructurehdfs-0.3.1-all.jar <webhdfs_url> <hdfs_topic_path> <output_folder>
+radar-hdfs-restructure --compression gzip  --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 
-Finally, by default, files records are not deduplicated after writing. To enable this behaviour, specify the option `-Dorg.radarcns.deduplicate=true`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.
+By default, files records are not deduplicated after writing. To enable this behaviour, specify the option `--deduplicate` or `-d`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.
+
+Finally, while processing, files are staged to a temporary directory and moved to the output directory afterwards. This has the advantage of less chance of data corruption, but it may result in slower performance. Disable staging using the `--no-stage` option.

build.gradle

@@ -2,60 +2,70 @@ apply plugin: 'java'
 apply plugin: 'application'
 
 group 'org.radarcns.restructurehdfs'
-version '0.3.2'
-mainClassName = 'org.radarcns.RestructureAvroRecords'
-
-run {
-    args = ['webhdfs://localhost:50070', '/topicAndroidNew/android_phone_sensor_acceleration', "${projectDir}/data"]
-}
+version '0.4.0'
+mainClassName = 'org.radarcns.hdfs.RestructureAvroRecords'
 
 sourceCompatibility = '1.8'
 targetCompatibility = '1.8'
 
-ext.avroVersion = '1.8.2'
-ext.jacksonVersion = '2.8.9'
-ext.hadoopVersion = '2.7.3'
+ext {
+    avroVersion = '1.8.2'
+    jacksonVersion = '2.9.6'
+    hadoopVersion = '2.7.6'
+    jCommanderVersion = '1.72'
+}
 
 repositories {
     jcenter()
 }
 
 dependencies {
-    compile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
-    compile group: 'org.apache.avro', name: 'avro', version: avroVersion
-    compile group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
-    compile group: 'com.fasterxml.jackson.core' , name: 'jackson-databind', version: jacksonVersion
-    compile group: 'com.fasterxml.jackson.dataformat' , name: 'jackson-dataformat-csv', version: jacksonVersion
+    implementation group: 'org.apache.avro', name: 'avro', version: avroVersion
+    implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind', version: jacksonVersion
+    implementation group: 'com.fasterxml.jackson.dataformat' , name: 'jackson-dataformat-csv', version: jacksonVersion
+    implementation group: 'com.beust', name: 'jcommander', version: jCommanderVersion
 
-    runtime group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion
+    implementation group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
+    implementation group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
 
-    testCompile group: 'junit', name: 'junit', version: '4.12'
+    runtimeOnly group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion
+
+    testImplementation group: 'junit', name: 'junit', version: '4.12'
 }
 
-//create a single Jar with all dependencies
-task fatJar(type: Jar) {
-    dependsOn configurations.runtime
+jar {
     manifest {
-        attributes 'Implementation-Title': 'radar-restructure-hdfs',
-                'Implementation-Version': version,
-                'Main-Class': mainClassName
+        attributes 'Implementation-Title': 'RADAR-base HDFS data restructuring',
+                'Implementation-Version': version
     }
-    baseName = project.name + '-all'
-    from {
-        configurations.runtime.collect {
-            it.isDirectory() ? it : zipTree(it)
+}
+
+distributions {
+    main {
+        contents {
+            into ("share/${project.name}") {
+                from 'README.md', 'LICENSE'
+            }
         }
-    } {
-        exclude 'META-INF/*'
     }
-    with jar
 }
 
-artifacts {
-    archives fatJar
+tasks.withType(Tar){
+    compression = Compression.GZIP
+    extension = 'tar.gz'
+}
+
+task downloadDependencies {
+    description "Pre-downloads dependencies"
+    configurations.compileClasspath.files
+    configurations.runtimeClasspath.files
+}
+
+task copyDependencies(type: Copy) {
+    from configurations.runtimeClasspath.files
+    into "${buildDir}/third-party/"
 }
 
-task wrapper(type: Wrapper) {
-    gradleVersion = '4.1'
-    distributionType 'all'
+wrapper {
+    gradleVersion '4.8'
 }

gradle.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip

gradle/wrapper/gradle-wrapper.jar

@@ -1,2 +1 @@
-rootProject.name = 'restructurehdfs'
-
+rootProject.name = 'radar-hdfs-restructure'

gradle/wrapper/gradle-wrapper.properties

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import org.apache.commons.collections.MapIterator;
 import org.apache.commons.collections.keyvalue.MultiKey;

settings.gradle

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;