More emphasis on local user; improved error reporting

Author: blootsvoets

Dockerfile

@@ -38,4 +38,11 @@ COPY --from=builder /code/build/third-party/* /usr/lib/
 COPY --from=builder /code/build/scripts/* /usr/bin/
 COPY --from=builder /code/build/libs/* /usr/lib/
 
+RUN mkdir /output \
+  && chown 101:101 /output
+
+VOLUME ["/output"]
+
+USER 101:101
+
 ENTRYPOINT ["radar-output-restructure"]

README.md

@@ -7,6 +7,12 @@ It supports data written by [RADAR HDFS sink connector](https://github.com/RADAR
 
 ## Upgrade instructions
 
+When upgrading to version 1.2.0, please follow the following instructions:
+
+  - When using local target storage, ensure that:
+    1. it is writable by the user 101, or change the runtime user using the docker command-line flag `--user` to a user that can write to the target storage and
+    2. local storage properties `userId` and `groupId` are set to values that can write to the target storage.
+
 When upgrading to version 1.0.0 or later from version 0.6.0 please follow the following instructions:
 
   - This package now relies on Redis for locking and offset management. Please install Redis or use
@@ -36,9 +42,11 @@ When upgrading to version 1.0.0 or later from version 0.6.0 please follow the fo
       target:
         type: local
         local:
-          # User ID to write data as
+          # User ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           userId: 123
-          # Group ID to write data as
+          # Group ID to write data as. This only works when explicitly setting
+          # the runtime user to root.
           groupId: 123
       ```
 
@@ -65,11 +73,6 @@ This package is available as docker image [`radarbase/radar-output-restructure`]
 docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.1.5 -n hdfs-namenode -o /output /myTopic
 ```
 
-if your docker cluster is running in the `hadoop` network and your output directory should be `./output`. Note that to run this in production this should be run as a non-root user. If local output is used, specify the user that the data should be written as both in the local storage configuration `target: {local: {userId: <MyUid>, groupId: <MyGroupId>}}` and also in the docker command line or docker-compose configuration:
-```shell
-docker run --rm -t --network hadoop -v "$PWD/output:/output" --user <MyUserId> --group-add <MyGroupId> radarbase/radar-output-restructure:1.1.5 -n hdfs-namenode -o /output /myTopic
-```
-
 ## Command line usage
 
 When the application is installed, it can be used as follows:

restructure.yml

@@ -58,8 +58,8 @@ target:
     container: MyTargetContainer
   # only actually needed if target type is local
   local:
-    userId: 1000  # write as regular user, use -1 to use current user (default).
-    groupId: 100  # write as regular group, use -1 to use current user (default).
+    userId: 1000  # write as user ID 1000, use -1 to use current user (default).
+    groupId: 100  # write as group ID 100, use -1 to use current user (default).
 
 # Redis configuration
 # @since: 0.7.0

src/main/java/org/radarbase/output/Application.kt

@@ -198,7 +198,7 @@ class Application(
                 logger.error("Failed to initialize plugins", ex)
                 exitProcess(1)
             } catch (e: IllegalStateException) {
-                logger.error("Cannot process configuration", e)
+                logger.error("Invalid configuration: {}", e.message)
                 exitProcess(1)
             }
 

src/main/java/org/radarbase/output/config/RestructureConfig.kt

@@ -31,7 +31,7 @@ data class RestructureConfig(
         /** Topic exceptional handling. */
         val topics: Map<String, TopicConfig> = emptyMap(),
         /** Source data resource configuration. */
-        val source: ResourceConfig = ResourceConfig("hdfs", hdfs = HdfsConfig()),
+        val source: ResourceConfig = ResourceConfig("s3"),
         /** Target data resource configration. */
         val target: ResourceConfig = ResourceConfig("local", local = LocalConfig()),
         /** Redis configuration for synchronization and storing offsets. */
@@ -284,10 +284,10 @@ data class ResourceConfig(
         sourceType = type.toResourceType()
 
         when(sourceType) {
-            ResourceType.S3 -> checkNotNull(s3)
-            ResourceType.HDFS -> checkNotNull(hdfs).also { it.validate() }
-            ResourceType.LOCAL -> checkNotNull(local)
-            ResourceType.AZURE -> checkNotNull(azure)
+            ResourceType.S3 -> checkNotNull(s3) { "No S3 configuration provided." }
+            ResourceType.HDFS -> checkNotNull(hdfs) { "No HDFS configuration provided." }.also { it.validate() }
+            ResourceType.LOCAL -> checkNotNull(local) { "No local configuration provided." }
+            ResourceType.AZURE -> checkNotNull(azure) { "No Azure configuration provided." }
         }
     }
 }
@@ -301,7 +301,7 @@ fun String.toResourceType() = when(toLowerCase()) {
     "hdfs" -> ResourceType.HDFS
     "local" -> ResourceType.LOCAL
     "azure" -> ResourceType.AZURE
-    else -> throw IllegalArgumentException("Unknown resource type $this, choose s3, hdfs or local")
+    else -> throw IllegalStateException("Unknown resource type $this, choose s3, hdfs or local")
 }
 
 data class LocalConfig(

src/main/java/org/radarbase/output/target/TargetStorageFactory.kt

@@ -8,6 +8,6 @@ class TargetStorageFactory(private val config: ResourceConfig) {
         ResourceType.S3 -> S3TargetStorage(config.s3!!)
         ResourceType.LOCAL -> LocalTargetStorage(config.local!!)
         ResourceType.AZURE -> AzureTargetStorage(config.azure!!)
-        else -> throw IllegalArgumentException("Cannot create storage driver for ${config.sourceType}")
+        else -> throw IllegalStateException("Cannot create storage driver for ${config.sourceType}")
     }
 }