Merge pull request #32 from RADAR-base/release-0.5.0

Release 0.5.0

Author: blootsvoets

.travis.yml

@@ -12,12 +12,21 @@ cache:
     - $HOME/.gradle/native
     - $HOME/.gradle/wrapper
 
+before_install:
+  - ./gradlew downloadDependencies
+
 deploy:
-  provider: releases
-  api_key: ${GH_TOKEN}
-  file_glob: true
-  file: "build/libs/*.jar"
-  skip_cleanup: true
-  on:
-    tags: true
+  - provider: releases
+    api_key: ${GH_TOKEN}
+    file_glob: true
+    file:
+      - "build/libs/*.jar"
+    skip_cleanup: true
+    on:
+      tags: true
+  - provider: script
+    script: ./gradlew bintrayUpload
+    skip_cleanup: true
+    on:
+      tags: true
 

Dockerfile

@@ -29,12 +29,14 @@ COPY ./src /code/src
 
 RUN ./gradlew jar
 
-FROM openjdk:8-jre-alpine
+FROM smizy/hadoop-base:3.0.3-alpine
 
 MAINTAINER Joris Borgdorff <[email protected]>
 
 LABEL description="RADAR-base HDFS data restructuring"
 
+ENV JAVA_OPTS -Djava.library.path=${HADOOP_HOME}/lib/native
+
 COPY --from=builder /code/build/third-party/* /usr/lib/
 COPY --from=builder /code/build/scripts/* /usr/bin/
 COPY --from=builder /code/build/libs/* /usr/lib/

README.md

@@ -8,7 +8,7 @@ Data streamed to HDFS using the [RADAR HDFS sink connector](https://github.com/R
 
 This package is available as docker image [`radarbase/radar-hdfs-restructure`](https://hub.docker.com/r/radarbase/radar-hdfs-restructure). The entrypoint of the image is the current application. So in all of the commands listed in usage, replace `radar-hdfs-restructure` with for example:
 ```shell
-docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.4.0 -u hdfs://hdfs -o /output /myTopic
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-hdfs-restructure:0.5.0 -u hdfs://hdfs -o /output /myTopic
 ```
 if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
 
@@ -24,7 +24,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-hdfs-restructure-0.4.0.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-hdfs-restructure-0.5.0.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-hdfs-restructure` command should be available.
@@ -59,4 +59,16 @@ radar-hdfs-restructure --compression gzip  --hdfs-uri <webhdfs_url> --output-dir
 
 By default, files records are not deduplicated after writing. To enable this behaviour, specify the option `--deduplicate` or `-d`. This set to false by default because of an issue with Biovotion data. Please see - [issue #16](https://github.com/RADAR-base/Restructure-HDFS-topic/issues/16) before enabling it.
 
-Finally, while processing, files are staged to a temporary directory and moved to the output directory afterwards. This has the advantage of less chance of data corruption, but it may result in slower performance. Disable staging using the `--no-stage` option.
+## Extending the connector
+
+To implement alternative storage paths, storage drivers or storage formats, put your custom JAR in
+`$APP_DIR/lib/radar-hdfs-plugins`. To load them, use the following options:
+
+| Option                  | Class                                       | Behaviour                                  | Default                   |
+| ----------------------- | ------------------------------------------- | ------------------------------------------ | ------------------------- |
+| `--path-factory`        | `org.radarcns.hdfs.RecordPathFactory`       | Factory to create output path names with.  | ObservationKeyPathFactory |
+| `--storage-driver`      | `org.radarcns.hdfs.data.StorageDriver`      | Storage driver to use for storing data.    | LocalStorageDriver        |
+| `--format-factory`      | `org.radarcns.hdfs.data.FormatFactory`      | Factory for output formats.                | FormatFactory             |
+| `--compression-factory` | `org.radarcns.hdfs.data.CompressionFactory` | Factory class to use for data compression. | CompressionFactory        |
+
+To pass arguments to self-assigned plugins, use `-p arg1=value1 -p arg2=value2` command-line flags and read those arguments in the `Plugin#init(Map<String, String>)` method.

build.gradle

@@ -1,42 +1,61 @@
-apply plugin: 'java'
-apply plugin: 'application'
+plugins {
+    id 'java'
+    id 'java-library'
+    id 'application'
+    id 'com.jfrog.bintray' version '1.8.1'
+    id 'maven-publish'
+}
 
-group 'org.radarcns.restructurehdfs'
-version '0.4.0'
-mainClassName = 'org.radarcns.hdfs.RestructureAvroRecords'
+group 'org.radarcns'
+version '0.5.0'
+mainClassName = 'org.radarcns.hdfs.Application'
 
 sourceCompatibility = '1.8'
 targetCompatibility = '1.8'
 
 ext {
+    moduleDescription = 'RADAR HDFS restructuring'
+    website = 'https://radar-base.org'
+    githubRepoName = 'RADAR-base/Restructure-HDFS-topic'
+    githubUrl = "https://github.com/${githubRepoName}"
+    issueUrl = "${githubUrl}/issues"
+
     avroVersion = '1.8.2'
     jacksonVersion = '2.9.6'
-    hadoopVersion = '2.7.6'
+    hadoopVersion = '3.0.3'
     jCommanderVersion = '1.72'
+    almworksVersion = '1.1.1'
 }
 
 repositories {
     jcenter()
 }
 
 dependencies {
-    implementation group: 'org.apache.avro', name: 'avro', version: avroVersion
+    api group: 'org.apache.avro', name: 'avro', version: avroVersion
     implementation group: 'com.fasterxml.jackson.core' , name: 'jackson-databind', version: jacksonVersion
     implementation group: 'com.fasterxml.jackson.dataformat' , name: 'jackson-dataformat-csv', version: jacksonVersion
     implementation group: 'com.beust', name: 'jcommander', version: jCommanderVersion
+    implementation group: 'com.almworks.integers', name: 'integers', version: almworksVersion
 
     implementation group: 'org.apache.avro', name: 'avro-mapred', version: avroVersion
     implementation group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion
 
-    runtimeOnly group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion
+    runtimeOnly group: 'org.apache.hadoop', name: 'hadoop-hdfs-client', version: hadoopVersion
 
     testImplementation group: 'junit', name: 'junit', version: '4.12'
 }
 
+ext.sharedManifest = manifest {
+    attributes(
+            "Implementation-Title": rootProject.name,
+            "Implementation-Version": version)
+}
+
 jar {
     manifest {
-        attributes 'Implementation-Title': 'RADAR-base HDFS data restructuring',
-                'Implementation-Version': version
+        from sharedManifest
+        attributes 'Main-Class': mainClassName
     }
 }
 
@@ -50,6 +69,17 @@ distributions {
     }
 }
 
+startScripts {
+    classpath += files('lib/PlaceHolderForPluginPath')
+
+    doLast {
+        def windowsScriptFile = file getWindowsScript()
+        def unixScriptFile    = file getUnixScript()
+        windowsScriptFile.text = windowsScriptFile.text.replace('PlaceHolderForPluginPath', 'radar-hdfs-plugins\\*')
+        unixScriptFile.text = unixScriptFile.text.replace('PlaceHolderForPluginPath', 'radar-hdfs-plugins/*')
+    }
+}
+
 tasks.withType(Tar){
     compression = Compression.GZIP
     extension = 'tar.gz'
@@ -66,6 +96,88 @@ task copyDependencies(type: Copy) {
     into "${buildDir}/third-party/"
 }
 
+// custom tasks for creating source/javadoc jars
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+    manifest.from sharedManifest
+}
+
+task javadocJar(type: Jar, dependsOn: javadoc) {
+    classifier = 'javadoc'
+    from javadoc.destinationDir
+    manifest.from sharedManifest
+}
+
+publishing {
+    publications {
+        mavenJar(MavenPublication) {
+            from components.java
+            artifact sourcesJar
+            artifact javadocJar
+            pom {
+                url = githubUrl as String
+                description = moduleDescription
+
+                licenses {
+                    license {
+                        name = 'The Apache Software License, Version 2.0'
+                        url = 'http://www.apache.org/licenses/LICENSE-2.0.txt'
+                        distribution = 'repo'
+                    }
+                }
+                developers {
+                    developer {
+                        id = 'blootsvoets'
+                        name = 'Joris Borgdorff'
+                        email = '[email protected]'
+                        organization = 'The Hyve'
+                    }
+
+                }
+                issueManagement {
+                    system = 'GitHub'
+                    url = issueUrl as String
+                }
+                organization {
+                    name = 'RADAR-base'
+                    url = website
+                }
+                scm {
+                    connection = "scm:git:${githubUrl}" as String
+                    url = githubUrl as String
+                }
+            }
+        }
+    }
+}
+
+
+bintray {
+    user project.hasProperty('bintrayUser') ? project.property('bintrayUser') : System.getenv('BINTRAY_USER')
+    key project.hasProperty('bintrayApiKey') ? project.property('bintrayApiKey') : System.getenv('BINTRAY_API_KEY')
+    override false
+    publications 'mavenJar'
+    pkg {
+        repo = project.group
+        name = rootProject.name
+        userOrg = 'radar-cns'
+        desc = moduleDescription
+        licenses = ['Apache-2.0']
+        websiteUrl = website
+        issueTrackerUrl = issueUrl
+        vcsUrl = githubUrl
+        githubRepo = githubRepoName
+        githubReleaseNotesFile = 'README.md'
+        version {
+            name = project.version
+            desc = description
+            vcsTag = System.getenv('TRAVIS_TAG')
+            released = new Date()
+        }
+    }
+}
+
 wrapper {
-    gradleVersion '4.8'
+    gradleVersion '4.9'
 }

gradle/wrapper/gradle-wrapper.jar

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.8-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradle/wrapper/gradle-wrapper.properties

@@ -1 +1,2 @@
 rootProject.name = 'radar-hdfs-restructure'
+enableFeaturePreview('STABLE_PUBLISHING')