Merge pull request #88 from RADAR-base/release-1.1.6

Release 1.1.6

Author: blootsvoets

.travis.yml

@@ -16,10 +16,10 @@ cache:
     - $HOME/.gradle/wrapper/
 
 env:
-  - DOCKER_COMPOSE_VERSION=1.25.4
+  - DOCKER_COMPOSE_VERSION=1.27.4
 
 before_install:
-  - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
+  - curl -L "https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-$(uname -s)-$(uname -m)" > docker-compose
   - chmod +x docker-compose
   - sudo mv docker-compose /usr/local/bin
   - ./gradlew downloadDependencies

Dockerfile

@@ -10,7 +10,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM gradle:6.6.1-jdk11 AS builder
+FROM gradle:6.7.1-jdk11 AS builder
 
 RUN mkdir /code
 WORKDIR /code
@@ -31,10 +31,18 @@ MAINTAINER Joris Borgdorff <[email protected]>, Yatharth Ranjan<yatharth.ranjan@k
 
 LABEL description="RADAR-base output data restructuring"
 
+ENV RADAR_OUTPUT_RESTRUCTURE_OPTS=""
 ENV JAVA_OPTS="-Djava.security.egd=file:/dev/./urandom -XX:+UseG1GC -XX:MaxHeapFreeRatio=10 -XX:MinHeapFreeRatio=10"
 
 COPY --from=builder /code/build/third-party/* /usr/lib/
 COPY --from=builder /code/build/scripts/* /usr/bin/
 COPY --from=builder /code/build/libs/* /usr/lib/
 
+RUN mkdir /output \
+  && chown 101:101 /output
+
+VOLUME ["/output"]
+
+USER 101:101
+
 ENTRYPOINT ["radar-output-restructure"]

Dockerfile.hdfs

@@ -7,6 +7,12 @@ It supports data written by [RADAR HDFS sink connector](https://github.com/RADAR
 
 ## Upgrade instructions
 
+When upgrading to version 1.2.0, please follow the following instructions:
+
+  - When using local target storage, ensure that:
+    1. it is writable by the user 101, or change the runtime user using the docker command-line flag `--user` to a user that can write to the target storage and
+    2. local storage properties `userId` and `groupId` are set to values that can write to the target storage.
+
 When upgrading to version 1.0.0 or later from version 0.6.0 please follow the following instructions:
 
   - This package now relies on Redis for locking and offset management. Please install Redis or use
@@ -36,9 +42,11 @@ When upgrading to version 1.0.0 or later from version 0.6.0 please follow the fo
       target:
         type: local
         local:
-          # User ID to write data as
+          # User ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           userId: 123
-          # Group ID to write data as
+          # Group ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           groupId: 123
       ```
 
@@ -60,12 +68,10 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 ## Docker usage
 
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
+
 ```shell
-docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.1.5-hdfs -n hdfs-namenode -o /output /myTopic
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.1.6 -n hdfs-namenode -o /output /myTopic
 ```
-if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
-
-Docker image tags that are optimized for HDFS are suffixed with `-hdfs`. Otherwise, please use the image without that suffix.
 
 ## Command line usage
 
@@ -171,7 +177,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-output-restructure-1.1.5.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-output-restructure-1.1.6.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-output-restructure` command should be available.

README.md

@@ -1,37 +1,33 @@
 plugins {
-    id 'java'
-    id 'java-library'
+    id 'org.jetbrains.kotlin.jvm'
     id 'application'
     id 'com.jfrog.bintray' version '1.8.5'
     id 'maven-publish'
-    id 'org.jetbrains.kotlin.jvm'
-    id 'com.avast.gradle.docker-compose' version "0.13.4"
+    id "org.jetbrains.dokka"
+    id 'com.avast.gradle.docker-compose' version "0.14.0"
 }
 
 group 'org.radarbase'
-version '1.1.5'
+version '1.1.6'
 mainClassName = 'org.radarbase.output.Application'
 
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
 ext {
     moduleDescription = 'RADAR-base output restructuring'
     website = 'https://radar-base.org'
     githubRepoName = 'RADAR-base/Restructure-HDFS-topic'
     githubUrl = "https://github.com/${githubRepoName}"
     issueUrl = "${githubUrl}/issues"
 
-    avroVersion = '1.10.0'
-    jacksonVersion = '2.11.3'
+    avroVersion = '1.10.1'
+    jacksonVersion = '2.12.0'
     hadoopVersion = '3.3.0'
     jCommanderVersion = '1.78'
     almworksVersion = '1.1.2'
-    junitVersion = '5.6.2'
-    minioVersion = '7.1.4'
-    jedisVersion = '3.3.0'
+    junitVersion = '5.7.0'
+    minioVersion = '8.0.3'
+    jedisVersion = '3.4.0'
     slf4jVersion = '1.7.30'
-    azureStorageVersion = '12.8.0'
+    azureStorageVersion = '12.9.0'
 }
 
 repositories {
@@ -80,21 +76,15 @@ dependencies {
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: junitVersion
     testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
     testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
-}
 
-compileKotlin {
-    kotlinOptions {
-        jvmTarget = "1.8"
-    }
+    dokkaHtmlPlugin("org.jetbrains.dokka:kotlin-as-java-plugin:$kotlinVersion")
 }
-compileTestKotlin {
-    kotlinOptions {
-        jvmTarget = "1.8"
-    }
-}
-compileIntegrationTestKotlin {
+
+tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
     kotlinOptions {
-        jvmTarget = "1.8"
+        jvmTarget = "11"
+        apiVersion = "1.4"
+        languageVersion = "1.4"
     }
 }
 
@@ -152,6 +142,7 @@ task integrationTest(type: Test) {
         showStandardStreams = true
         setExceptionFormat("full")
     }
+    outputs.upToDateWhen { false }
     shouldRunAfter test
 }
 
@@ -186,18 +177,17 @@ task sourcesJar(type: Jar, dependsOn: classes) {
     manifest.from sharedManifest
 }
 
-task javadocJar(type: Jar, dependsOn: javadoc) {
-    archiveClassifier.set('javadoc')
-    from javadoc.destinationDir
-    manifest.from sharedManifest
+task dokkaJar(type: Jar, dependsOn: dokkaJavadoc) {
+    archiveClassifier.set("javadoc")
+    from("$buildDir/dokka/javadoc/")
 }
 
 publishing {
     publications {
         mavenJar(MavenPublication) {
             from components.java
             artifact sourcesJar
-            artifact javadocJar
+            artifact dokkaJar
             pom {
                 url = githubUrl as String
                 description = moduleDescription
@@ -261,5 +251,5 @@ bintray {
 }
 
 wrapper {
-    gradleVersion '6.6.1'
+    gradleVersion '6.7.1'
 }

build.gradle

@@ -10,7 +10,7 @@ services:
       - /data
 
   redis:
-    image: bitnami/redis
+    image: bitnami/redis:6.0
     ports:
       - "6379:6379"
     environment:

docker-compose.yml

@@ -1 +1 @@
-kotlinVersion=1.4.0
+kotlinVersion=1.4.20

gradle.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.6.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.7.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

gradle/wrapper/gradle-wrapper.properties

@@ -58,8 +58,8 @@ target:
     container: MyTargetContainer
   # only actually needed if target type is local
   local:
-    userId: 1000  # write as regular user, use -1 to use current user (default).
-    groupId: 100  # write as regular group, use -1 to use current user (default).
+    userId: 1000  # write as user ID 1000, use -1 to use current user (default).
+    groupId: 100  # write as group ID 100, use -1 to use current user (default).
 
 # Redis configuration
 # @since: 0.7.0

restructure.yml

@@ -2,7 +2,10 @@ pluginManagement {
     resolutionStrategy {
         eachPlugin {
             if ( requested.id.id == 'org.jetbrains.kotlin.jvm' ) {
-                useModule( "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" )
+                useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
+            }
+            if (requested.id.id == "org.jetbrains.dokka") {
+                useModule("org.jetbrains.dokka:dokka-gradle-plugin:$kotlinVersion")
             }
         }
     }