Merge pull request #94 from RADAR-base/release-1.2.1

Release 1.2.1

Author: blootsvoets

README.md

@@ -70,7 +70,7 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
 
 ```shell
-docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.2.0 -n hdfs-namenode -o /output /myTopic
+docker run --rm -t --network hadoop -v "$PWD/output:/output" radarbase/radar-output-restructure:1.2.1 -n hdfs-namenode -o /output /myTopic
 ```
 
 ## Command line usage
@@ -192,7 +192,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-output-restructure-1.2.0.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-output-restructure-1.2.1.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-output-restructure` command should be available.

build.gradle.kts

@@ -16,7 +16,7 @@ plugins {
 }
 
 group = "org.radarbase"
-version = "1.2.0"
+version = "1.2.1"
 
 description = "RADAR-base output restructuring"
 val website = "https://radar-base.org"

src/main/java/org/radarbase/output/config/RestructureConfig.kt

@@ -6,6 +6,7 @@ import com.azure.storage.blob.BlobServiceClientBuilder
 import com.azure.storage.common.StorageSharedKeyCredential
 import com.fasterxml.jackson.annotation.JsonIgnore
 import io.minio.MinioClient
+import io.minio.credentials.IamAwsProvider
 import org.apache.hadoop.conf.Configuration
 import org.radarbase.output.Application.Companion.CACHE_SIZE_DEFAULT
 import org.radarbase.output.Plugin
@@ -305,12 +306,12 @@ data class ResourceConfig(
     val local: LocalConfig? = null,
     val azure: AzureConfig? = null,
 ) {
-    @JsonIgnore
-    lateinit var sourceType: ResourceType
+    @get:JsonIgnore
+    val sourceType: ResourceType by lazy {
+        requireNotNull(type.toResourceType()) { "Unknown resource type $type, choose s3, hdfs or local" }
+    }
 
     fun validate() {
-        sourceType = type.toResourceType()
-
         when (sourceType) {
             ResourceType.S3 -> checkNotNull(s3) { "No S3 configuration provided." }
             ResourceType.HDFS -> checkNotNull(hdfs) { "No HDFS configuration provided." }.also { it.validate() }
@@ -336,7 +337,7 @@ fun String.toResourceType() = when(toLowerCase()) {
     "hdfs" -> ResourceType.HDFS
     "local" -> ResourceType.LOCAL
     "azure" -> ResourceType.AZURE
-    else -> throw IllegalStateException("Unknown resource type $this, choose s3, hdfs or local")
+    else -> null
 }
 
 data class LocalConfig(
@@ -350,18 +351,23 @@ data class S3Config(
     /** URL to reach object store at. */
     val endpoint: String,
     /** Access token for writing data with. */
-    val accessToken: String,
+    val accessToken: String?,
     /** Secret key belonging to access token. */
-    val secretKey: String,
+    val secretKey: String?,
     /** Bucket name. */
     val bucket: String,
     /** If no endOffset is in the filename, read it from object tags. */
     val endOffsetFromTags: Boolean = false,
 ) {
-    fun createS3Client(): MinioClient = MinioClient.Builder()
-        .endpoint(endpoint)
-        .credentials(accessToken, secretKey)
-        .build()
+
+    fun createS3Client(): MinioClient = MinioClient.Builder().apply {
+        endpoint(endpoint)
+        if (accessToken.isNullOrBlank() || secretKey.isNullOrBlank()) {
+            credentialsProvider(IamAwsProvider(null, null))
+        } else {
+            credentials(accessToken, secretKey)
+        }
+    }.build()
 
     fun withEnv(prefix: String): S3Config = this
         .copyEnv("${prefix}S3_ACCESS_TOKEN") { copy(accessToken = it) }