Merge pull request #525 from RADAR-base/release-2.2.0

Release 2.2.0

Author: blootsvoets

.github/workflows/main.yml

@@ -20,9 +20,9 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v2
+      - uses: actions/setup-java@v3
         with:
           distribution: zulu
           java-version: 17
@@ -38,7 +38,7 @@ jobs:
       - name: Check
         run: ./gradlew check
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v3
         if: always()
         with:
           name: integration-test-logs
@@ -53,10 +53,10 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Cache Docker layers
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: /tmp/.buildx-cache
           key: ${{ runner.os }}-buildx-${{ hashFiles('Dockerfile', '**/*.gradle.kts', 'gradle.properties', 'src/main/**') }}
@@ -66,27 +66,27 @@ jobs:
       # Add Docker labels and tags
       - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.DOCKER_IMAGE }}
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       # Setup docker build environment
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
-          context: ./
+          context: .
           file: ./Dockerfile
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,dest=/tmp/.buildx-cache,mode=max

.github/workflows/publish_snapshots.yml

@@ -15,13 +15,13 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Has SNAPSHOT version
         id: is-snapshot
         run: grep 'version = ".*-SNAPSHOT"' build.gradle.kts
 
-      - uses: actions/setup-java@v2
+      - uses: actions/setup-java@v3
         with:
           distribution: zulu
           java-version: 17

.github/workflows/release.yml

@@ -16,15 +16,15 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      - uses: actions/setup-java@v2
+      - uses: actions/setup-java@v3
         with:
           distribution: zulu
           java-version: 17
 
       - name: Setup Gradle
-        uses: gradle/gradle-build-action@v2
+        uses: gradle/gradle-build-action@v3
 
       # Compile code
       - name: Compile code
@@ -55,39 +55,37 @@ jobs:
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       # Add Docker labels and tags
       - name: Docker meta
         id: docker_meta
-        uses: crazy-max/ghaction-docker-meta@v2
+        uses: docker/metadata-action@v4
         with:
           images: ${{ env.DOCKER_IMAGE }}
           # output 2.1.2, 2.1 and 2
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}},enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }}
 
       # Setup docker build environment
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Build and push
         id: docker_build
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
-          context: ./
+          context: .
           file: ./Dockerfile
-          # Allow running the image on the architectures supported by openjdk:11-jre-slim
           platforms: linux/amd64,linux/arm64
           push: true
           tags: ${{ steps.docker_meta.outputs.tags }}
@@ -99,7 +97,17 @@ jobs:
             org.opencontainers.image.vendor=RADAR-base
             org.opencontainers.image.licenses=Apache-2.0
 
+      - name: Build locally
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile
+          platforms: linux/amd64
+          load: true
+          tags: ${{ steps.docker_meta.outputs.tags }}
+
       - name: Inspect image
         run: |
-          docker pull ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
           docker image inspect ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }}
+          docker run --rm ${{ env.DOCKER_IMAGE }}:${{ steps.docker_meta.outputs.version }} --help

.github/workflows/scheduled_snyk.yaml

@@ -8,12 +8,14 @@ jobs:
     env:
       REPORT_FILE: test.json
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v3
       - uses: snyk/actions/setup@master
+        with:
+          snyk-version: v1.931.0
 
-      - uses: actions/setup-java@v2
+      - uses: actions/setup-java@v3
         with:
-          distribution: zulu
+          distribution: temurin
           java-version: 17
 
       - name: Setup Gradle
@@ -27,3 +29,11 @@ jobs:
           --configuration-matching='^runtimeClasspath$'
           --json-file-output=${{ env.REPORT_FILE }}
           --org=radar-base
+
+      - name: Report new vulnerabilities
+        uses: thehyve/report-vulnerability@master
+        with:
+          report-file: ${{ env.REPORT_FILE }}
+        env:
+          TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        if: ${{ failure() }}

.github/workflows/snyk.yaml

@@ -7,12 +7,14 @@ jobs:
   security:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
+      - uses: actions/checkout@v3
       - uses: snyk/actions/setup@master
+        with:
+          snyk-version: v1.931.0
 
-      - uses: actions/setup-java@v2
+      - uses: actions/setup-java@v3
         with:
-          distribution: zulu
+          distribution: temurin
           java-version: 17
 
       - name: Setup Gradle
@@ -23,7 +25,7 @@ jobs:
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
         run: >
           snyk test
-          --severity-threshold=high
           --configuration-matching='^runtimeClasspath$'
           --fail-on=upgradable
           --org=radar-base
+          --severity-threshold=high

Dockerfile

@@ -25,14 +25,14 @@ COPY ./src /code/src
 
 RUN gradle jar --no-watch-fs
 
-FROM azul/zulu-openjdk-alpine:17-jre-headless
+FROM eclipse-temurin:17-jre
 
 MAINTAINER Joris Borgdorff <[email protected]>, Yatharth Ranjan<[email protected]>
 
 LABEL description="RADAR-base output data restructuring"
 
-ENV RADAR_OUTPUT_RESTRUCTURE_OPTS=""
-ENV JAVA_OPTS="-Djava.security.egd=file:/dev/./urandom -XX:+UseG1GC -XX:MaxHeapFreeRatio=10 -XX:MinHeapFreeRatio=10"
+ENV RADAR_OUTPUT_RESTRUCTURE_OPTS="" \
+  JAVA_OPTS="-XX:+UseG1GC -XX:MaxHeapFreeRatio=10 -XX:MinHeapFreeRatio=10"
 
 COPY --from=builder /code/build/third-party/* /usr/lib/
 COPY --from=builder /code/build/scripts/* /usr/bin/

README.md

@@ -70,7 +70,7 @@ When upgrading to version 0.6.0 from version 0.5.x or earlier, please follow the
 This package is available as docker image [`radarbase/radar-output-restructure`](https://hub.docker.com/r/radarbase/radar-output-restructure). The entrypoint of the image is the current application. So in all the commands listed in usage, replace `radar-output-restructure` with for example:
 
 ```shell
-docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.1.0 -o /output /myTopic
+docker run --rm -t --network s3 -v "$PWD/output:/output" radarbase/radar-output-restructure:2.2.0 -o /output /myTopic
 ```
 
 ## Command line usage
@@ -182,7 +182,7 @@ This package requires at least Java JDK 8. Build the distribution with
 and install the package into `/usr/local` with for example
 ```shell
 sudo mkdir -p /usr/local
-sudo tar -xzf build/distributions/radar-output-restructure-2.1.0.tar.gz -C /usr/local --strip-components=1
+sudo tar -xzf build/distributions/radar-output-restructure-2.2.0.tar.gz -C /usr/local --strip-components=1
 ```
 
 Now the `radar-output-restructure` command should be available.

build.gradle.kts

@@ -12,10 +12,11 @@ plugins {
     id("com.avast.gradle.docker-compose")
     id("com.github.ben-manes.versions")
     id("io.github.gradle-nexus.publish-plugin")
+    id("org.jlleitschuh.gradle.ktlint") version "10.3.0"
 }
 
 group = "org.radarbase"
-version = "2.1.0.1"
+version = "2.2.0"
 
 repositories {
     mavenCentral()
@@ -24,8 +25,8 @@ repositories {
 description = "RADAR-base output restructuring"
 val website = "https://radar-base.org"
 val githubRepoName = "RADAR-base/radar-output-restructure"
-val githubUrl = "https://github.com/${githubRepoName}"
-val issueUrl = "${githubUrl}/issues"
+val githubUrl = "https://github.com/$githubRepoName"
+val issueUrl = "$githubUrl/issues"
 
 sourceSets {
     create("integrationTest") {
@@ -50,6 +51,8 @@ dependencies {
     runtimeOnly("org.xerial.snappy:snappy-java:$snappyVersion")
 
     implementation(kotlin("reflect"))
+    val coroutinesVersion: String by project
+    implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion")
 
     val jacksonVersion: String by project
     api(platform("com.fasterxml.jackson:jackson-bom:$jacksonVersion"))
@@ -90,9 +93,12 @@ dependencies {
 
     val log4jVersion: String by project
     runtimeOnly("org.apache.logging.log4j:log4j-slf4j-impl:$log4jVersion")
+    runtimeOnly("org.apache.logging.log4j:log4j-api:$log4jVersion")
+    runtimeOnly("org.apache.logging.log4j:log4j-jul:$log4jVersion")
 
     val radarSchemasVersion: String by project
     testImplementation("org.radarbase:radar-schemas-commons:$radarSchemasVersion")
+    testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion")
 
     val junitVersion: String by project
     testImplementation("org.junit.jupiter:junit-jupiter-api:$junitVersion")
@@ -114,6 +120,10 @@ dependencies {
 
 application {
     mainClass.set("org.radarbase.output.Application")
+    applicationDefaultJvmArgs = listOf(
+        "-Djava.security.egd=file:/dev/./urandom",
+        "-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager",
+    )
 }
 
 distributions {
@@ -195,7 +205,6 @@ publishing {
                         email.set("[email protected]")
                         organization.set("The Hyve")
                     }
-
                 }
                 issueManagement {
                     system.set("GitHub")
@@ -206,7 +215,7 @@ publishing {
                     url.set(website)
                 }
                 scm {
-                    connection.set("scm:git:${githubUrl}")
+                    connection.set("scm:git:$githubUrl")
                     url.set(githubUrl)
                 }
             }
@@ -259,8 +268,9 @@ dockerCompose {
     isRequiredBy(integrationTest)
 }
 
-val check by tasks
-check.dependsOn(integrationTest)
+tasks["composeUp"].dependsOn("composePull")
+
+tasks["check"].dependsOn(integrationTest)
 
 tasks.withType<Test> {
     useJUnitPlatform()
@@ -298,6 +308,11 @@ tasks.named<DependencyUpdatesTask>("dependencyUpdates").configure {
     }
 }
 
+ktlint {
+    version.set("0.45.2")
+    disabledRules.set(setOf("no-wildcard-imports"))
+}
+
 tasks.wrapper {
-    gradleVersion = "7.4.1"
+    gradleVersion = "7.4.2"
 }

gradle.properties

@@ -1,27 +1,28 @@
 kotlin.code.style=official
 
-kotlinVersion=1.6.10
-dokkaVersion=1.6.10
-dockerComposeVersion=0.15.2
+kotlinVersion=1.6.21
+dokkaVersion=1.6.21
+dockerComposeVersion=0.16.4
 dependencyUpdateVersion=0.42.0
 nexusPublishVersion=1.1.0
-jsoupVersion=1.14.3
+jsoupVersion=1.15.1
 
+coroutinesVersion=1.6.1
 avroVersion=1.11.0
 snappyVersion=1.1.8.4
-jacksonVersion=2.13.2.20220324
+jacksonVersion=2.13.3
 jCommanderVersion=1.82
 almworksVersion=1.1.2
-minioVersion=8.3.7
+minioVersion=8.4.1
 guavaVersion=31.1-jre
 opencsvVersion=5.6
 okhttpVersion=4.9.3
-jedisVersion=4.1.1
+jedisVersion=4.2.3
 slf4jVersion=1.7.36
 log4jVersion=2.17.2
-azureStorageVersion=12.15.0
-nettyVersion=4.1.75.Final
+azureStorageVersion=12.16.1
+nettyVersion=4.1.77.Final
 
 junitVersion=5.8.2
 mockitoKotlinVersion=4.0.0
-radarSchemasVersion=0.7.7
+radarSchemasVersion=0.7.9