Merge pull request #537 from RADAR-base/release-2.3.2

blootsvoets · web-flow · commit 459d7e244686 · 2022-12-05T10:19:39.000+01:00
Release 2.3.2
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -89,7 +89,7 @@ jobs:
           context: .
           file: ./Dockerfile
           cache-from: type=local,src=/tmp/.buildx-cache
-          cache-to: type=local,dest=/tmp/.buildx-cache,mode=max
+          cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max
           load: true
           tags: ${{ steps.docker_meta.outputs.tags }}
           # Use runtime labels from docker_meta as well as fixed labels
@@ -109,3 +109,8 @@ jobs:
       - name: Push image
         if: ${{ github.event_name != 'pull_request' }}
         run: docker push ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
diff --git a/.github/workflows/scheduled_snyk.yaml b/.github/workflows/scheduled_snyk.yaml
@@ -2,6 +2,10 @@ name: Snyk scheduled test
 on:
   schedule:
     - cron: '0 2 * * 1'
+  push:
+    branches:
+      - master
+
 jobs:
   security:
     runs-on: ubuntu-latest
@@ -27,13 +31,15 @@ jobs:
         run: >
           snyk test
           --configuration-matching='^runtimeClasspath$'
+          --fail-on=upgradable
           --json-file-output=${{ env.REPORT_FILE }}
           --org=radar-base
+          --policy-path=$PWD/.snyk
 
       - name: Report new vulnerabilities
         uses: thehyve/report-vulnerability@master
+        if: success() || failure()
         with:
           report-file: ${{ env.REPORT_FILE }}
         env:
           TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        if: ${{ failure() }}
diff --git a/.github/workflows/snyk.yaml b/.github/workflows/snyk.yaml
@@ -3,6 +3,7 @@ on:
   pull_request:
     branches:
       - main
+
 jobs:
   security:
     runs-on: ubuntu-latest
@@ -26,6 +27,5 @@ jobs:
         run: >
           snyk test
           --configuration-matching='^runtimeClasspath$'
-          --fail-on=upgradable
           --org=radar-base
-          --severity-threshold=high
+          --policy-path=$PWD/.snyk
diff --git a/README.md b/README.md
@@ -70,7 +70,7 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
 
 ```shell
-docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.3.1 -o /output /myTopic
+docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.3.2 -o /output /myTopic
 ```
 
 ## Command line usage
@@ -228,7 +228,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-output-restructure-2.3.1.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-output-restructure-2.3.2.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-output-restructure` command should be available.
diff --git a/build.gradle.kts b/build.gradle.kts
@@ -16,7 +16,7 @@ plugins {
 }
 
 group = "org.radarbase"
-version = "2.3.1"
+version = "2.3.2"
 
 repositories {
     mavenCentral()
@@ -86,7 +86,9 @@ dependencies {
     val azureStorageVersion: String by project
     implementation("com.azure:azure-storage-blob:$azureStorageVersion") {
         val nettyVersion: String by project
-        implementation(platform("io.netty:netty-bom:$nettyVersion"))
+        runtimeOnly(platform("io.netty:netty-bom:$nettyVersion"))
+        val projectReactorNettyVersion: String by project
+        runtimeOnly("io.projectreactor.netty:reactor-netty-http:$projectReactorNettyVersion")
     }
     val opencsvVersion: String by project
     implementation("com.opencsv:opencsv:$opencsvVersion") {
@@ -321,5 +323,5 @@ ktlint {
 }
 
 tasks.wrapper {
-    gradleVersion = "7.5.1"
+    gradleVersion = "7.6"
 }
diff --git a/gradle.properties b/gradle.properties
@@ -1,30 +1,31 @@
 kotlin.code.style=official
 
-kotlinVersion=1.7.20
+kotlinVersion=1.7.22
 dokkaVersion=1.7.20
-dockerComposeVersion=0.16.9
-dependencyUpdateVersion=0.43.0
+dockerComposeVersion=0.16.11
+dependencyUpdateVersion=0.44.0
 nexusPublishVersion=1.1.0
 jsoupVersion=1.15.3
 
 coroutinesVersion=1.6.4
 avroVersion=1.11.1
 snappyVersion=1.1.8.4
-jacksonVersion=2.13.4.20221013
+jacksonVersion=2.14.1
 jCommanderVersion=1.82
 almworksVersion=1.1.2
-minioVersion=8.4.5
+minioVersion=8.4.6
 guavaVersion=31.1-jre
-opencsvVersion=5.7.0
+opencsvVersion=5.7.1
 okhttpVersion=4.10.0
-jedisVersion=4.3.0
-slf4jVersion=2.0.3
+jedisVersion=4.3.1
+slf4jVersion=2.0.5
 log4jVersion=2.19.0
-azureStorageVersion=12.20.0
-nettyVersion=4.1.84.Final
-snakeYamlVersion=1.32
+azureStorageVersion=12.20.1
+nettyVersion=4.1.85.Final
+snakeYamlVersion=1.33
 apacheCommonsTextVersion=1.10.0
+projectReactorNettyVersion=1.0.24
 
 junitVersion=5.9.1
-mockitoKotlinVersion=4.0.0
-radarSchemasVersion=0.7.9
+mockitoKotlinVersion=4.1.0
+radarSchemasVersion=0.8.2
diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,5 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip
+networkTimeout=10000
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
diff --git a/gradlew b/gradlew
@@ -55,7 +55,7 @@
 #       Darwin, MinGW, and NonStop.
 #
 #   (3) This script is generated from the Groovy template
-#       https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
 #       within the Gradle project.
 #
 #       You can find Gradle at https://github.com/gradle/gradle/.
@@ -80,10 +80,10 @@ do
     esac
 done
 
-APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
-
-APP_NAME="Gradle"
+# This is normally unused
+# shellcheck disable=SC2034
 APP_BASE_NAME=${0##*/}
+APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
 DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
@@ -143,12 +143,16 @@ fi
 if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
     case $MAX_FD in #(
       max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045 
         MAX_FD=$( ulimit -H -n ) ||
             warn "Could not query maximum file descriptor limit"
     esac
     case $MAX_FD in  #(
       '' | soft) :;; #(
       *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC3045 
         ulimit -n "$MAX_FD" ||
             warn "Could not set maximum file descriptor limit to $MAX_FD"
     esac
diff --git a/gradlew.bat b/gradlew.bat
@@ -26,6 +26,7 @@ if "%OS%"=="Windows_NT" setlocal
 
 set DIRNAME=%~dp0
 if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
diff --git a/src/main/java/org/radarbase/output/Application.kt b/src/main/java/org/radarbase/output/Application.kt
@@ -18,8 +18,7 @@ package org.radarbase.output
 
 import com.beust.jcommander.JCommander
 import com.beust.jcommander.ParameterException
-import kotlinx.coroutines.launch
-import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.*
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.Semaphore
 import org.radarbase.output.accounting.*
@@ -200,9 +199,17 @@ class Application(
             } catch (e: IllegalStateException) {
                 logger.error("Invalid configuration: {}", e.message)
                 exitProcess(1)
+            } catch (ex: Throwable) {
+                logger.error("Failed to initialize application: {}", ex.message)
+                exitProcess(1)
             }
 
-            application.start()
+            try {
+                application.start()
+            } catch (ex: Throwable) {
+                logger.error("Application failed: {}", ex.message)
+                exitProcess(1)
+            }
         }
     }
 }
diff --git a/src/main/java/org/radarbase/output/accounting/OffsetIntervals.kt b/src/main/java/org/radarbase/output/accounting/OffsetIntervals.kt
@@ -73,7 +73,7 @@ class OffsetIntervals {
 
     fun add(range: OffsetRangeSet.Range) {
         val (from, to, lastModified) = range
-        checkNotNull(to)
+        checkNotNull(to) { "Cannot add OffsetRange if the `to` value is missing." }
         var index = offsetsFrom.binarySearch(from)
         if (index < 0) {
             // index where this range would be entered
@@ -143,7 +143,7 @@ class OffsetIntervals {
 
     fun remove(range: OffsetRangeSet.Range) {
         val (from, to, _) = range
-        checkNotNull(to)
+        checkNotNull(to) { "Cannot add OffsetRange if the `to` value is missing." }
 
         var index = offsetsFrom.binarySearch(from)
 
diff --git a/src/main/java/org/radarbase/output/config/RestructureConfig.kt b/src/main/java/org/radarbase/output/config/RestructureConfig.kt
@@ -81,8 +81,11 @@ data class RestructureConfig(
         inline fun <T> T.copyEnv(key: String, doCopy: T.(String) -> T): T =
             copyOnChange<T, String?>(
                 null,
-                modification = { System.getenv(key) },
-                doCopy = { doCopy(requireNotNull(it)) }
+                modification = {
+                    System.getenv(key)
+                        ?.takeIf { it.isNotEmpty() }
+                },
+                doCopy = { doCopy(requireNotNull(it) { "Environment variable $key is empty" }) },
             )
 
         inline fun <T, V> T.copyOnChange(
diff --git a/src/main/java/org/radarbase/output/source/SourceStorageFactory.kt b/src/main/java/org/radarbase/output/source/SourceStorageFactory.kt
@@ -11,18 +11,20 @@ class SourceStorageFactory(
     private val tempPath: Path,
 ) {
     private val s3SourceClient: MinioClient? = if (resourceConfig.sourceType == ResourceType.S3) {
-        requireNotNull(resourceConfig.s3).createS3Client()
+        requireNotNull(resourceConfig.s3) { "Missing S3 configuration" }
+            .createS3Client()
     } else null
 
     private val azureSourceClient: BlobServiceClient? =
         if (resourceConfig.sourceType == ResourceType.AZURE) {
-            requireNotNull(resourceConfig.azure).createAzureClient()
+            requireNotNull(resourceConfig.azure) { "Missing Azure configuration" }
+                .createAzureClient()
         } else null
 
     fun createSourceStorage() = when (resourceConfig.sourceType) {
         ResourceType.S3 -> {
-            val s3Config = requireNotNull(resourceConfig.s3)
-            val minioClient = requireNotNull(s3SourceClient)
+            val s3Config = requireNotNull(resourceConfig.s3) { "Missing S3 configuration for source storage" }
+            val minioClient = requireNotNull(s3SourceClient) { "Missing S3 client configuration for source storage" }
             S3SourceStorage(minioClient, s3Config, tempPath)
         }
         ResourceType.HDFS -> {
@@ -34,8 +36,8 @@ class SourceStorageFactory(
             createSourceStorage.invoke(factory) as SourceStorage
         }
         ResourceType.AZURE -> {
-            val azureClient = requireNotNull(azureSourceClient)
-            val azureConfig = requireNotNull(resourceConfig.azure)
+            val azureClient = requireNotNull(azureSourceClient) { "Missing Azure client configuration for source storage" }
+            val azureConfig = requireNotNull(resourceConfig.azure) { "Missing Azure configuration for source storage" }
             AzureSourceStorage(azureClient, azureConfig, tempPath)
         }
         else -> throw IllegalStateException("Cannot create kafka storage for type ${resourceConfig.sourceType}")
diff --git a/src/test/java/org/radarbase/output/source/S3SourceStorageTest.kt b/src/test/java/org/radarbase/output/source/S3SourceStorageTest.kt
@@ -0,0 +1,27 @@
+package org.radarbase.output.source
+
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.radarbase.output.source.S3SourceStorage.Companion.faultTolerant
+import java.io.IOException
+
+class S3SourceStorageTest {
+    @Test
+    fun testSuspend() {
+        assertThrows<IOException> {
+            runBlocking {
+                launch { throw IOException("ex") }
+            }
+        }
+    }
+    @Test
+    fun testFaultTolerant() {
+        assertThrows<IOException> {
+            runBlocking {
+                faultTolerant { throw IOException("test") }
+            }
+        }
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@ plugins {`
`16`	`16`	`}`
`17`	`17`
`18`	`18`	`group = "org.radarbase"`
`19`		`-version = "2.3.1"`
	`19`	`+version = "2.3.2"`
`20`	`20`
`21`	`21`	`repositories {`
`22`	`22`	`mavenCentral()`
`@@ -86,7 +86,9 @@ dependencies {`
`86`	`86`	`val azureStorageVersion: String by project`
`87`	`87`	`implementation("com.azure:azure-storage-blob:$azureStorageVersion") {`
`88`	`88`	`val nettyVersion: String by project`
`89`		`- implementation(platform("io.netty:netty-bom:$nettyVersion"))`
	`89`	`+ runtimeOnly(platform("io.netty:netty-bom:$nettyVersion"))`
	`90`	`+ val projectReactorNettyVersion: String by project`
	`91`	`+ runtimeOnly("io.projectreactor.netty:reactor-netty-http:$projectReactorNettyVersion")`
`90`	`92`	`}`
`91`	`93`	`val opencsvVersion: String by project`
`92`	`94`	`implementation("com.opencsv:opencsv:$opencsvVersion") {`
`@@ -321,5 +323,5 @@ ktlint {`
`321`	`323`	`}`
`322`	`324`
`323`	`325`	`tasks.wrapper {`
`324`		`- gradleVersion = "7.5.1"`
	`326`	`+ gradleVersion = "7.6"`
`325`	`327`	`}`