Release 0.9.3

build.gradle.kts

@@ -8,6 +8,17 @@ plugins {
 
 description = "RADAR Gateway to handle secured data flow to backend."
 
+allprojects {
+    repositories {
+        mavenCentral()
+        maven(url = "https://central.sonatype.com/repository/maven-snapshots/") {
+            mavenContent {
+                snapshotsOnly()
+            }
+        }
+    }
+}
+
 radarRootProject {
     projectVersion.set(Versions.project)
 }

buildSrc/src/main/kotlin/Versions.kt

@@ -1,22 +1,26 @@
 @Suppress("ConstPropertyName")
 object Versions {
-    const val project = "0.9.2"
+    const val project = "0.9.3"
 
     const val java = 17
     const val kotlin = "1.9.22"
     const val dockerCompose = "0.17.6"
 
     const val ktor = "2.3.10"
-    const val radarJersey = "0.12.4"
+    const val radarJersey = "0.12.6"
     const val radarCommons = "1.2.4"
     const val radarSchemas = "0.8.14"
     const val jackson = "2.17.2"
+    const val assertJ = "3.27.3"
+    const val mockk = "1.13.16"
     const val log4j2 = "2.23.1"
     const val lzfse = "0.1.1"
     const val radarAuth = "2.1.12"
     const val avro = "1.12.0"
     const val confluent = "7.6.0"
     const val kafka = "$confluent-ce"
+    const val minio = "8.5.10"
+    const val multipart = "3.1.10"
 
     const val mockitoKotlin = "5.3.1"
     const val grizzly = "4.0.2"

gateway.yml

@@ -55,3 +55,29 @@ auth:
         #rsa: []
     # jwks URLs to fetch public keys from
     #publicKeyUrls: []
+# Storage settings for the application
+storageCondition:
+  # Enables or disables file upload functionality in the application
+  fileUploadEnabled: true
+  # Specifies the type of storage used for storing uploaded files.
+  radarStorageType: s3
+# Amazon S3 storage configurations
+s3:
+  # The endpoint URL for the S3-compatible storage service.
+  # This can be a local MinIO server (e.g., http://localhost:9000) or AWS S3.
+  url: http://localhost:9000
+  # The access key used for authentication with the S3 storage.
+  accessKey: access-key
+  # The secret key used for authentication with the S3 storage.
+  secretKey: secret-key
+  # The name of the S3 bucket where files will be uploaded.
+  bucketName: radar
+  # specifies the region for the bucket
+  region: eu-west-2
+  # Path-related settings for organizing stored files.
+  path:
+    # A prefix to be added to the stored file paths (optional).
+    prefix:
+    # Determines whether files should be organized into daily subdirectories.
+    # If true, files will be stored in a structure like "bucketName/yyyy-MM-dd/".
+    collectPerDay: true

radar-gateway/build.gradle.kts

@@ -61,12 +61,15 @@ dependencies {
     implementation(kotlin("stdlib-jdk8"))
     implementation(kotlin("reflect"))
 
+    implementation("org.glassfish.jersey.media:jersey-media-multipart:${Versions.multipart}")
+
     implementation("org.radarbase:radar-commons:${Versions.radarCommons}")
     implementation("org.radarbase:radar-commons-kotlin:${Versions.radarCommons}")
     implementation("org.radarbase:radar-jersey:${Versions.radarJersey}")
     implementation("org.radarbase:managementportal-client:${Versions.radarAuth}")
     implementation("org.radarbase:lzfse-decode:${Versions.lzfse}")
     implementation("org.radarbase:radar-auth:${Versions.radarAuth}")
+    implementation("io.minio:minio:${Versions.minio}")
 
     implementation("org.apache.kafka:kafka-clients:${Versions.kafka}")
     implementation("io.confluent:kafka-avro-serializer:${Versions.confluent}")
@@ -93,6 +96,8 @@ dependencies {
     integrationTestImplementation("io.ktor:ktor-serialization-kotlinx-json")
 
     testImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}")
+    testImplementation("org.assertj:assertj-core:${Versions.assertJ}")
+    testImplementation("io.mockk:mockk:${Versions.mockk}")
     integrationTestImplementation("org.radarbase:radar-schemas-commons:${Versions.radarSchemas}")
     integrationTestImplementation("org.radarbase:radar-commons-testing:${Versions.radarCommons}")
 }

radar-gateway/src/integrationTest/docker/etc/gateway.yml

@@ -9,3 +9,29 @@ kafka:
 
 auth:
   managementPortalUrl: http://managementportal-app:8080/managementportal
+# Storage settings for the application
+storageCondition:
+  # Enables or disables file upload functionality in the application
+  fileUploadEnabled: false
+  # Specifies the type of storage used for storing uploaded files.
+  radarStorageType: s3
+# Amazon S3 storage configurations
+s3:
+  # The endpoint URL for the S3-compatible storage service.
+  # This can be a local MinIO server (e.g., http://localhost:9000) or AWS S3.
+  url: http://localhost:9000
+  # The access key used for authentication with the S3 storage.
+  #  accessKey: access-key
+  # The secret key used for authentication with the S3 storage.
+  #  secretKey: secret-key
+  # The name of the S3 bucket where files will be uploaded.
+  #  bucketName: radar
+  # Path-related settings for organizing stored files.
+  # specifies the region for the bucket
+  # region: eu-west-2
+  path:
+    # A prefix to be added to the stored file paths (optional).
+    #    prefix:
+    # Determines whether files should be organized into daily subdirectories.
+    # If true, files will be stored in a structure like "bucketName/yyyy-MM-dd/".
+    collectPerDay: true

radar-gateway/src/main/kotlin/org/radarbase/gateway/Main.kt

@@ -19,6 +19,7 @@ fun main(args: Array<String>) {
             ),
             args,
         ).withDefaults()
+            .withEnv()
     } catch (ex: IllegalArgumentException) {
         logger.error("No configuration file was found.")
         logger.error("Usage: radar-gateway <config-file>")
@@ -33,6 +34,13 @@ fun main(args: Array<String>) {
     }
 
     val resources = ConfigLoader.loadResources(config.resourceConfig, config)
-    val server = GrizzlyServer(config.server.baseUri, resources, config.server.isJmxEnabled)
+    val server = GrizzlyServer(
+        config.server.baseUri,
+        resources,
+        config.server.isJmxEnabled,
+        config.server.workerCorePoolSize,
+        config.server.workerMaxPoolSize,
+    )
+
     server.listen()
 }

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/GatewayConfig.kt

@@ -1,6 +1,7 @@
 package org.radarbase.gateway.config
 
 import org.radarbase.gateway.inject.ManagementPortalEnhancerFactory
+import org.radarbase.jersey.config.ConfigLoader.copyOnChange
 import org.radarbase.jersey.enhancer.EnhancerFactory
 
 data class GatewayConfig(
@@ -12,6 +13,10 @@ data class GatewayConfig(
     val kafka: KafkaConfig = KafkaConfig(),
     /** Server configurations. */
     val server: GatewayServerConfig = GatewayServerConfig(),
+    /** AWS s3 storage configuration */
+    val s3: S3StorageConfig = S3StorageConfig(),
+    /** Whether to enable or disable the configurations based on the storage conditions */
+    val storageCondition: StorageConditionConfig = StorageConditionConfig(),
 ) {
     /** Fill in some default values for the configuration. */
     fun withDefaults(): GatewayConfig = copy(kafka = kafka.withDefaults())
@@ -24,4 +29,14 @@ data class GatewayConfig(
         kafka.validate()
         auth.validate()
     }
+
+    fun withEnv(): GatewayConfig = this.copyOnChange(
+        s3,
+        {
+            it.withEnv()
+        },
+        {
+            copy(s3 = it)
+        },
+    )
 }

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/GatewayServerConfig.kt

@@ -21,4 +21,10 @@ data class GatewayServerConfig(
      * Whether JMX should be enabled. Disable if not needed, for higher performance.
      */
     val isJmxEnabled: Boolean = true,
+
+    /** Number of core worker threads in main thread pool */
+    val workerCorePoolSize: Int? = 32,
+
+    /** Max number of worker threads */
+    val workerMaxPoolSize: Int? = 128,
 )

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/S3StorageConfig.kt

@@ -0,0 +1,32 @@
+package org.radarbase.gateway.config
+
+import org.radarbase.gateway.utils.Env.AWS_ACCESS_KEY_ID
+import org.radarbase.gateway.utils.Env.AWS_DEFAULT_REGION
+import org.radarbase.gateway.utils.Env.AWS_ENDPOINT_URL_S3
+import org.radarbase.gateway.utils.Env.AWS_S3_BUCKET_NAME
+import org.radarbase.gateway.utils.Env.AWS_SECRET_ACCESS_KEY
+import org.radarbase.jersey.config.ConfigLoader.copyEnv
+import org.radarbase.jersey.config.ConfigLoader.copyOnChange
+
+data class S3StorageConfig(
+    val url: String? = null,
+    val accessKey: String? = null,
+    val secretKey: String? = null,
+    val bucketName: String? = null,
+    val region: String? = null,
+    val path: S3StoragePathConfig = S3StoragePathConfig(),
+) {
+    fun withEnv(): S3StorageConfig = this.copyEnv(AWS_ENDPOINT_URL_S3) {
+        copy(url = it)
+    }.copyEnv(AWS_ACCESS_KEY_ID) {
+        copy(accessKey = it)
+    }.copyEnv(AWS_SECRET_ACCESS_KEY) {
+        copy(secretKey = it)
+    }.copyEnv(AWS_S3_BUCKET_NAME) {
+        copy(bucketName = it)
+    }.copyEnv(AWS_DEFAULT_REGION) {
+        copy(region = it)
+    }.copyOnChange(path, S3StoragePathConfig::withEnv) {
+        copy(path = it)
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/S3StoragePathConfig.kt

@@ -0,0 +1,14 @@
+package org.radarbase.gateway.config
+
+import org.radarbase.gateway.utils.Env.AWS_S3_PATH_PREFIX
+import org.radarbase.jersey.config.ConfigLoader.copyEnv
+
+data class S3StoragePathConfig(
+    val prefix: String? = null,
+    val collectPerDay: Boolean = true,
+) {
+    fun withEnv(): S3StoragePathConfig = this
+        .copyEnv(AWS_S3_PATH_PREFIX) {
+            copy(prefix = it)
+        }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/config/StorageConditionConfig.kt

@@ -0,0 +1,6 @@
+package org.radarbase.gateway.config
+
+data class StorageConditionConfig(
+    val fileUploadEnabled: Boolean = true,
+    val radarStorageType: String = "s3",
+)

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/FileStorageException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class FileStorageException(message: String) :
+    HttpApplicationException(
+        Response.Status.INTERNAL_SERVER_ERROR,
+        Response.Status.INTERNAL_SERVER_ERROR.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/InvalidFileDetailsException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class InvalidFileDetailsException(message: String) :
+    HttpApplicationException(
+        Response.Status.EXPECTATION_FAILED,
+        Response.Status.EXPECTATION_FAILED.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/exception/InvalidPathDetailsException.kt

@@ -0,0 +1,11 @@
+package org.radarbase.gateway.exception
+
+import jakarta.ws.rs.core.Response
+import org.radarbase.jersey.exception.HttpApplicationException
+
+class InvalidPathDetailsException(message: String) :
+    HttpApplicationException(
+        Response.Status.EXPECTATION_FAILED,
+        Response.Status.EXPECTATION_FAILED.name.lowercase(),
+        message,
+    )

radar-gateway/src/main/kotlin/org/radarbase/gateway/filter/FileUploadFilter.kt

@@ -0,0 +1,31 @@
+package org.radarbase.gateway.filter
+
+import jakarta.annotation.Priority
+import jakarta.inject.Singleton
+import jakarta.ws.rs.Priorities
+import jakarta.ws.rs.container.ContainerRequestContext
+import jakarta.ws.rs.container.ContainerRequestFilter
+import jakarta.ws.rs.core.Context
+import jakarta.ws.rs.core.Response
+import jakarta.ws.rs.ext.Provider
+import org.radarbase.gateway.config.GatewayConfig
+import org.radarbase.gateway.inject.ProcessFileUpload
+
+@Provider
+@Singleton
+@Priority(Priorities.USER)
+@ProcessFileUpload
+class FileUploadFilter(
+    @Context private val config: GatewayConfig,
+) : ContainerRequestFilter {
+
+    override fun filter(requestContext: ContainerRequestContext?) {
+        if (!config.storageCondition.fileUploadEnabled) {
+            requestContext?.abortWith(
+                Response.status(Response.Status.FORBIDDEN)
+                    .entity("File uploading is not configured")
+                    .build(),
+            )
+        }
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/FileStorageEnhancer.kt

@@ -0,0 +1,37 @@
+package org.radarbase.gateway.inject
+
+import jakarta.inject.Singleton
+import org.glassfish.jersey.internal.inject.AbstractBinder
+import org.glassfish.jersey.media.multipart.MultiPartFeature
+import org.radarbase.gateway.config.GatewayConfig
+import org.radarbase.gateway.config.S3StorageConfig
+import org.radarbase.gateway.service.storage.RadarMinioClient
+import org.radarbase.gateway.service.storage.RadarMinioClientFactory
+import org.radarbase.gateway.service.storage.S3StorageService
+import org.radarbase.gateway.service.storage.StorageService
+import org.radarbase.jersey.enhancer.JerseyResourceEnhancer
+
+class FileStorageEnhancer(
+    private val config: GatewayConfig,
+) : JerseyResourceEnhancer {
+
+    override val classes: Array<Class<*>> = buildList(1) {
+        add(MultiPartFeature::class.java)
+    }.toTypedArray()
+
+    override fun AbstractBinder.enhance() {
+        bind(config.s3)
+            .to(S3StorageConfig::class.java)
+            .`in`(Singleton::class.java)
+
+        if (config.storageCondition.radarStorageType == "s3") {
+            bindFactory(RadarMinioClientFactory::class.java)
+                .to(RadarMinioClient::class.java)
+                .`in`(Singleton::class.java)
+
+            bind(S3StorageService::class.java)
+                .to(StorageService::class.java)
+                .`in`(Singleton::class.java)
+        }
+    }
+}

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/ManagementPortalEnhancerFactory.kt

@@ -18,12 +18,15 @@ class ManagementPortalEnhancerFactory(private val config: GatewayConfig) : Enhan
             jwtIssuer = config.auth.issuer,
             jwksUrls = config.auth.publicKeyUrls ?: emptyList(),
         )
-        return listOf(
-            GatewayResourceEnhancer(config),
-            Enhancers.radar(authConfig),
-            Enhancers.managementPortal(authConfig),
-            Enhancers.health,
-            Enhancers.exception,
-        )
+        return buildList {
+            add(GatewayResourceEnhancer(config))
+            add(Enhancers.radar(authConfig))
+            add(Enhancers.managementPortal(authConfig))
+            add(Enhancers.health)
+            add(Enhancers.exception)
+            if (config.storageCondition.fileUploadEnabled) {
+                add(FileStorageEnhancer(config))
+            }
+        }
     }
 }

radar-gateway/src/main/kotlin/org/radarbase/gateway/inject/ProcessFileUpload.kt

@@ -0,0 +1,13 @@
+package org.radarbase.gateway.inject
+
+import jakarta.ws.rs.NameBinding
+
+@NameBinding
+@Target(
+    AnnotationTarget.CLASS,
+    AnnotationTarget.FUNCTION,
+    AnnotationTarget.PROPERTY_GETTER,
+    AnnotationTarget.PROPERTY_SETTER,
+)
+@Retention(AnnotationRetention.RUNTIME)
+annotation class ProcessFileUpload