Merge pull request #86 from RADAR-base/phaseOutHdfs

Phase out hdfs

Author: blootsvoets

Dockerfile

@@ -31,10 +31,18 @@ MAINTAINER Joris Borgdorff <[email protected]>, Yatharth Ranjan<yatharth.ranjan@k
 
 LABEL description="RADAR-base output data restructuring"
 
+ENV RADAR_OUTPUT_RESTRUCTURE_OPTS=""
 ENV JAVA_OPTS="-Djava.security.egd=file:/dev/./urandom -XX:+UseG1GC -XX:MaxHeapFreeRatio=10 -XX:MinHeapFreeRatio=10"
 
 COPY --from=builder /code/build/third-party/* /usr/lib/
 COPY --from=builder /code/build/scripts/* /usr/bin/
 COPY --from=builder /code/build/libs/* /usr/lib/
 
+RUN mkdir /output \
+  && chown 101:101 /output
+
+VOLUME ["/output"]
+
+USER 101:101
+
 ENTRYPOINT ["radar-output-restructure"]

Dockerfile.hdfs

@@ -7,6 +7,12 @@ It supports data written by [RADAR HDFS sink connector](https://github.com/RADAR
 
 ## Upgrade instructions
 
+When upgrading to version 1.2.0, please follow the following instructions:
+
+  - When using local target storage, ensure that:
+    1. it is writable by the user 101, or change the runtime user using the docker command-line flag `--user` to a user that can write to the target storage and
+    2. local storage properties `userId` and `groupId` are set to values that can write to the target storage.
+
 When upgrading to version 1.0.0 or later from version 0.6.0 please follow the following instructions:
 
   - This package now relies on Redis for locking and offset management. Please install Redis or use
@@ -36,9 +42,11 @@ When upgrading to version 1.0.0 or later from version 0.6.0 please follow the fo
       target:
         type: local
         local:
-          # User ID to write data as
+          # User ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           userId: 123
-          # Group ID to write data as
+          # Group ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           groupId: 123
       ```
 
@@ -60,12 +68,10 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 ## Docker usage
 
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
+
 ```shell
-docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.1.5-hdfs -n hdfs-namenode -o /output /myTopic
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.1.5 -n hdfs-namenode -o /output /myTopic
 ```
-if your docker cluster is running in the `hadoop` network and your output directory should be `./output`.
-
-Docker image tags that are optimized for HDFS are suffixed with `-hdfs`. Otherwise, please use the image without that suffix.
 
 ## Command line usage
 

README.md

@@ -1,20 +1,16 @@
 plugins {
-    id 'java'
-    id 'java-library'
+    id 'org.jetbrains.kotlin.jvm'
     id 'application'
     id 'com.jfrog.bintray' version '1.8.5'
     id 'maven-publish'
-    id 'org.jetbrains.kotlin.jvm'
+    id "org.jetbrains.dokka"
     id 'com.avast.gradle.docker-compose' version "0.13.4"
 }
 
 group 'org.radarbase'
-version '1.1.6-SNAPSHOT'
+version '1.2.0-SNAPSHOT'
 mainClassName = 'org.radarbase.output.Application'
 
-sourceCompatibility = '1.8'
-targetCompatibility = '1.8'
-
 ext {
     moduleDescription = 'RADAR-base output restructuring'
     website = 'https://radar-base.org'
@@ -80,21 +76,15 @@ dependencies {
     testRuntimeOnly group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: junitVersion
     testImplementation group: 'org.hamcrest', name: 'hamcrest-all', version: '1.3'
     testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
-}
 
-compileKotlin {
-    kotlinOptions {
-        jvmTarget = "1.8"
-    }
+    dokkaHtmlPlugin("org.jetbrains.dokka:kotlin-as-java-plugin:$kotlinVersion")
 }
-compileTestKotlin {
-    kotlinOptions {
-        jvmTarget = "1.8"
-    }
-}
-compileIntegrationTestKotlin {
+
+tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
     kotlinOptions {
-        jvmTarget = "1.8"
+        jvmTarget = "11"
+        apiVersion = "1.4"
+        languageVersion = "1.4"
     }
 }
 
@@ -186,18 +176,17 @@ task sourcesJar(type: Jar, dependsOn: classes) {
     manifest.from sharedManifest
 }
 
-task javadocJar(type: Jar, dependsOn: javadoc) {
-    archiveClassifier.set('javadoc')
-    from javadoc.destinationDir
-    manifest.from sharedManifest
+task dokkaJar(type: Jar, dependsOn: dokkaJavadoc) {
+    archiveClassifier.set("javadoc")
+    from("$buildDir/dokka/javadoc/")
 }
 
 publishing {
     publications {
         mavenJar(MavenPublication) {
             from components.java
             artifact sourcesJar
-            artifact javadocJar
+            artifact dokkaJar
             pom {
                 url = githubUrl as String
                 description = moduleDescription

build.gradle

@@ -58,8 +58,8 @@ target:
     container: MyTargetContainer
   # only actually needed if target type is local
   local:
-    userId: 1000  # write as regular user, use -1 to use current user (default).
-    groupId: 100  # write as regular group, use -1 to use current user (default).
+    userId: 1000  # write as user ID 1000, use -1 to use current user (default).
+    groupId: 100  # write as group ID 100, use -1 to use current user (default).
 
 # Redis configuration
 # @since: 0.7.0

restructure.yml

@@ -2,7 +2,10 @@ pluginManagement {
     resolutionStrategy {
         eachPlugin {
             if ( requested.id.id == 'org.jetbrains.kotlin.jvm' ) {
-                useModule( "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion" )
+                useModule("org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion")
+            }
+            if (requested.id.id == "org.jetbrains.dokka") {
+                useModule("org.jetbrains.dokka:dokka-gradle-plugin:$kotlinVersion")
             }
         }
     }

settings.gradle

@@ -198,7 +198,7 @@ class Application(
                 logger.error("Failed to initialize plugins", ex)
                 exitProcess(1)
             } catch (e: IllegalStateException) {
-                logger.error("Cannot process configuration", e)
+                logger.error("Invalid configuration: {}", e.message)
                 exitProcess(1)
             }
 

src/main/java/org/radarbase/output/Application.kt

@@ -31,7 +31,7 @@ data class RestructureConfig(
         /** Topic exceptional handling. */
         val topics: Map<String, TopicConfig> = emptyMap(),
         /** Source data resource configuration. */
-        val source: ResourceConfig = ResourceConfig("hdfs", hdfs = HdfsConfig()),
+        val source: ResourceConfig = ResourceConfig("s3"),
         /** Target data resource configration. */
         val target: ResourceConfig = ResourceConfig("local", local = LocalConfig()),
         /** Redis configuration for synchronization and storing offsets. */
@@ -284,10 +284,10 @@ data class ResourceConfig(
         sourceType = type.toResourceType()
 
         when(sourceType) {
-            ResourceType.S3 -> checkNotNull(s3)
-            ResourceType.HDFS -> checkNotNull(hdfs).also { it.validate() }
-            ResourceType.LOCAL -> checkNotNull(local)
-            ResourceType.AZURE -> checkNotNull(azure)
+            ResourceType.S3 -> checkNotNull(s3) { "No S3 configuration provided." }
+            ResourceType.HDFS -> checkNotNull(hdfs) { "No HDFS configuration provided." }.also { it.validate() }
+            ResourceType.LOCAL -> checkNotNull(local) { "No local configuration provided." }
+            ResourceType.AZURE -> checkNotNull(azure) { "No Azure configuration provided." }
         }
     }
 }
@@ -301,7 +301,7 @@ fun String.toResourceType() = when(toLowerCase()) {
     "hdfs" -> ResourceType.HDFS
     "local" -> ResourceType.LOCAL
     "azure" -> ResourceType.AZURE
-    else -> throw IllegalArgumentException("Unknown resource type $this, choose s3, hdfs or local")
+    else -> throw IllegalStateException("Unknown resource type $this, choose s3, hdfs or local")
 }
 
 data class LocalConfig(

src/main/java/org/radarbase/output/config/RestructureConfig.kt

@@ -8,6 +8,6 @@ class TargetStorageFactory(private val config: ResourceConfig) {
         ResourceType.S3 -> S3TargetStorage(config.s3!!)
         ResourceType.LOCAL -> LocalTargetStorage(config.local!!)
         ResourceType.AZURE -> AzureTargetStorage(config.azure!!)
-        else -> throw IllegalArgumentException("Cannot create storage driver for ${config.sourceType}")
+        else -> throw IllegalStateException("Cannot create storage driver for ${config.sourceType}")
     }
 }

src/main/java/org/radarbase/output/target/TargetStorageFactory.kt

@@ -0,0 +1,24 @@
+package org.radarbase.output.util
+
+import org.hamcrest.MatcherAssert.assertThat
+import org.hamcrest.Matchers.`is`
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.io.TempDir
+import java.nio.file.Files
+import java.nio.file.Path
+
+internal class TemporaryDirectoryTest {
+    @Test
+    fun createAndDelete(@TempDir root: Path) {
+        TemporaryDirectory(root, "worker-").use {
+            assertThat(Files.list(root).count(), `is`(1L))
+            Files.createTempFile(it.path, "test", "txt")
+            Files.createTempFile(it.path, "test", "txt")
+
+            assertThat(Files.list(it.path).count(), `is`(2L))
+            assertThat(Files.list(root).count(), `is`(1L))
+        }
+
+        assertThat(Files.list(root).count(), `is`(0L))
+    }
+}