diff --git a/README.md b/README.md index 7e56876..85f96c9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,22 @@ # Restructure Kafka connector output files + +* [Restructure Kafka connector output files](#restructure-kafka-connector-output-files) + * [Upgrade instructions](#upgrade-instructions) + * [Docker usage](#docker-usage) + * [Command line usage](#command-line-usage) + * [File Format](#file-format) + * [Compression](#compression) + * [Redis](#redis) + * [Source and target](#source-and-target) + * [Path format](#path-format) + * [Cleaner](#cleaner) + * [Service](#service) + * [Sentry monitoring](#sentry-monitoring) + * [Local build](#local-build) + * [Extending the connector](#extending-the-connector) + + Data streamed by a Kafka Connector will be converted to a RADAR-base oriented output directory, by organizing it by project, user and collection date. It supports data written by [RADAR S3 sink connector](https://github.com/RADAR-base/RADAR-S3-Connector) is streamed to files based on topic name only. This package transforms that output to a local directory structure as follows: `projectId/userId/topic/date_hour.csv`. The date and hour are extracted from the `time` field of each record, and is formatted in UTC time. @@ -225,6 +242,23 @@ The cleaner can also be enabled with the `--cleaner` command-line flag. To run t To run the output generator as a service that will regularly poll the source directory, add the `--service` flag and optionally the `--interval` flag to adjust the polling interval or use the corresponding configuration file parameters. +## Sentry monitoring + +To enable Sentry monitoring: + +1. Set a `SENTRY_DSN` environment variable that points to the desired Sentry DSN. +2. (Optional) Set the `SENTRY_LOG_LEVEL` environment variable to control the minimum log level of events sent to Sentry. + The default log level for Sentry is `ERROR`. Possible values are `TRACE`, `DEBUG`, `INFO`, `WARN`, and `ERROR`. + +For further configuration of Sentry via environmental variables see [here](https://docs.sentry.io/platforms/java/configuration/#configuration-via-the-runtime-environment). For instance: + +``` +SENTRY_LOG_LEVEL: 'ERROR' +SENTRY_DSN: 'https://000000000000.ingest.de.sentry.io/000000000000' +SENTRY_ATTACHSTACKTRACE: true +SENTRY_STACKTRACE_APP_PACKAGES: io.confluent.connect,org.radarbase.connect.rest +``` + ## Local build This package requires at least Java JDK 8. Build the distribution with diff --git a/build.gradle.kts b/build.gradle.kts index 9f40c6b..20c806d 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -19,11 +19,9 @@ radarRootProject { } radarKotlin { - kotlinVersion.set(Versions.kotlin) javaVersion.set(Versions.java) log4j2Version.set(Versions.log4j2) - slf4jVersion.set(Versions.slf4j) - junitVersion.set(Versions.junit) + sentryEnabled.set(true) } radarPublishing { @@ -31,9 +29,9 @@ radarPublishing { githubUrl.set("https://github.com/$githubRepoName.git") developers { developer { - id.set("bdegraaf1234") - name.set("Bastiaan de Graaf") - email.set("bastiaan@thehyve.nl") + id.set("pvannierop") + name.set("Pim Van Nierop") + email.set("pim@thehyve.nl") organization.set("The Hyve") } } diff --git a/buildSrc/src/main/kotlin/Versions.kt b/buildSrc/src/main/kotlin/Versions.kt index ef38812..0dfd05e 100644 --- a/buildSrc/src/main/kotlin/Versions.kt +++ b/buildSrc/src/main/kotlin/Versions.kt @@ -3,10 +3,9 @@ object Versions { const val project = "3.0.2" const val java = 17 - const val kotlin = "1.9.22" const val dockerCompose = "0.17.5" - const val radarCommons = "1.1.2" + const val radarCommons = "1.1.3" const val radarSchemas = "0.8.11" const val jackson = "2.16.1" const val slf4j = "2.0.9" diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml index 725abb3..a0b4c28 100644 --- a/src/main/resources/log4j2.xml +++ b/src/main/resources/log4j2.xml @@ -1,13 +1,38 @@ - - - - + + + + + + - + + + - - - - - - + + + + + + + +