Merge branch 'dev' into release-0.4.0

Author: blootsvoets

.dockerignore

@@ -0,0 +1,6 @@
+.git/
+build/
+src/test/
+out/
+.gradle/
+.idea/

Dockerfile

@@ -0,0 +1,42 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM openjdk:8-alpine AS builder
+
+RUN mkdir /code
+WORKDIR /code
+
+ENV GRADLE_OPTS -Dorg.gradle.daemon=false
+
+COPY ./gradle /code/gradle
+COPY ./gradlew /code/
+RUN ./gradlew --version
+
+COPY ./build.gradle ./gradle.properties ./settings.gradle /code/
+
+RUN ./gradlew downloadDependencies copyDependencies startScripts
+
+COPY ./src /code/src
+
+RUN ./gradlew jar
+
+FROM openjdk:8-jre-alpine
+
+MAINTAINER Joris Borgdorff <[email protected]>
+
+LABEL description="RADAR-base HDFS data restructuring"
+
+COPY --from=builder /code/build/third-party/* /usr/lib/
+COPY --from=builder /code/build/scripts/* /usr/bin/
+COPY --from=builder /code/build/libs/* /usr/lib/
+
+ENTRYPOINT ["radar-hdfs-restructure"]

README.md

@@ -2,43 +2,59 @@
 
 [![Build Status](https://travis-ci.org/RADAR-base/Restructure-HDFS-topic.svg?branch=master)](https://travis-ci.org/RADAR-base/Restructure-HDFS-topic)
 
-Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/RADAR-base/RADAR-HDFS-Sink-Connector) is streamed to files based on sensor only. This package can transform that output to a local directory structure as follows: `userId/topic/date_hour.csv`. The date and hour is extracted from the `time` field of each record, and is formatted in UTC time.
+Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/RADAR-CNS/RADAR-HDFS-Sink-Connector) is streamed to files based on sensor only. This package can transform that output to a local directory structure as follows: `userId/topic/date_hour.csv`. The date and hour is extracted from the `time` field of each record, and is formatted in UTC time. This package is included in the [RADAR-Docker](https://github.com/RADAR-CNS/RADAR-Docker) repository, in the `dcompose/radar-cp-hadoop-stack/hdfs_restructure.sh` script.
 
-## Usage
+## Docker usage
+
+This package is available as docker image [`radarbase/radar-hdfs-restructure`](https://hub.docker.com/r/radarbase/radar-hdfs-restructure). The entrypoint of the image is the current application. So in all of the commands listed in usage, replace `radar-hdfs-restructure` with for example:
+```shell
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.4.0 -u hdfs://hdfs -o /output /myTopic
+```
+if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
 
-This package is included in the [RADAR-Docker](https://github.com/RADAR-base/RADAR-Docker) repository, in the `dcompose/radar-cp-hadoop-stack/hdfs_restructure.sh` script.
 
-## Advanced usage
+## Local build
 
-Build jar from source with
+This package requires at least Java JDK 8. Build the distribution with
 
 ```shell
 ./gradlew build
 ```
-and find the output JAR file as `build/libs/restructurehdfs-0.4.0-all.jar`. Then run with:
+
+and install the package into `/usr/local` with for example
+```shell
+sudo mkdir -p /usr/local
+sudo tar -xzf build/distributions/radar-hdfs-restructure-0.4.0.tar.gz -C /usr/local --strip-components=1
+```
+
+Now the `radar-hdfs-restructure` command should be available.
+
+## Command line usage
+
+When the application is installed, it can be used as follows:
 
 ```shell
-java -jar restructurehdfs-0.4.0-all.jar --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
+radar-hdfs-restructure --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 or you can use the short form as well like - 
 ```shell
-java -jar restructurehdfs-0.4.0-all.jar -u <webhdfs_url> -o <output_folder> <input_path_1> [<input_path_2> ...]
+radar-hdfs-restructure -u <webhdfs_url> -o <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 
 To display the usage and all available options you can use the help option as follows - 
 ```shell
-java -jar restructurehdfs-0.4.0-all.jar --help
+radar-hdfs-restructure --help
 ```
 Note that the options preceded by the `*` in the above output are required to run the app. Also note that there can be multiple input paths from which to read the files. Eg - `/topicAndroidNew/topic1 /topicAndroidNew/topic2 ...`. At least one input path is required.
 
 By default, this will output the data in CSV format. If JSON format is preferred, use the following instead:
 ```shell
-java -jar restructurehdfs-0.4.0-all.jar --format json --hdfs-uri <webhdfs_url> --output-directory <output_folder>  <input_path_1> [<input_path_2> ...]
+radar-hdfs-restructure --format json --hdfs-uri <webhdfs_url> --output-directory <output_folder>  <input_path_1> [<input_path_2> ...]
 ```
 
 Another option is to output the data in compressed form. All files will get the `gz` suffix, and can be decompressed with a GZIP decoder. Note that for a very small number of records, this may actually increase the file size.
 ```
-java -jar restructurehdfs-0.4.0-all.jar --compression gzip  --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
+radar-hdfs-restructure --compression gzip  --hdfs-uri <webhdfs_url> --output-directory <output_folder> <input_path_1> [<input_path_2> ...]
 ```
 
 By default, files records are not deduplicated after writing. To enable this behaviour, specify the option `--deduplicate` or `-d`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.

build.gradle

@@ -5,10 +5,6 @@ group 'org.radarcns.restructurehdfs'
 version '0.4.0'
 mainClassName = 'org.radarcns.RestructureAvroRecords'
 
-run {
-    args = ['webhdfs://localhost:50070', '/topicAndroidNew/android_phone_sensor_acceleration', "${projectDir}/data"]
-}
-
 sourceCompatibility = '1.8'
 targetCompatibility = '1.8'
 
@@ -37,26 +33,37 @@ dependencies {
     testImplementation group: 'junit', name: 'junit', version: '4.12'
 }
 
-//create a single Jar with all dependencies
-task fatJar(type: Jar) {
+jar {
     manifest {
-        attributes 'Implementation-Title': 'radar-restructure-hdfs',
-                'Implementation-Version': version,
-                'Main-Class': mainClassName
+        attributes 'Implementation-Title': 'RADAR-base HDFS data restructuring',
+                'Implementation-Version': version
     }
-    classifier = 'all'
-    from {
-        configurations.runtimeClasspath.collect {
-            it.isDirectory() ? it : zipTree(it)
+}
+
+distributions {
+    main {
+        contents {
+            into ("share/${project.name}") {
+                from 'README.md', 'LICENSE'
+            }
         }
-    } {
-        exclude 'META-INF/*'
     }
-    with jar
 }
 
-artifacts {
-    archives fatJar
+tasks.withType(Tar){
+    compression = Compression.GZIP
+    extension = 'tar.gz'
+}
+
+task downloadDependencies {
+    description "Pre-downloads dependencies"
+    configurations.compileClasspath.files
+    configurations.runtimeClasspath.files
+}
+
+task copyDependencies(type: Copy) {
+    from configurations.runtimeClasspath.files
+    into "${buildDir}/third-party/"
 }
 
 wrapper {

gradle.properties

@@ -1,2 +1 @@
-rootProject.name = 'restructurehdfs'
-
+rootProject.name = 'radar-hdfs-restructure'

settings.gradle

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import org.apache.commons.collections.MapIterator;
 import org.apache.commons.collections.keyvalue.MultiKey;

src/main/java/org/radarcns/Frequency.java renamed to src/main/java/org/radarcns/hdfs/Frequency.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;

src/main/java/org/radarcns/OffsetRange.java renamed to src/main/java/org/radarcns/hdfs/OffsetRange.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import com.fasterxml.jackson.databind.MappingIterator;
 import com.fasterxml.jackson.databind.ObjectReader;

src/main/java/org/radarcns/OffsetRangeFile.java renamed to src/main/java/org/radarcns/hdfs/OffsetRangeFile.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.radarcns;
+package org.radarcns.hdfs;
 
 import javax.annotation.Nonnull;
 import java.util.Iterator;