feat: add strimzi chart for s3-connector

Author: pvannierop

charts/radar-s3-connector-strimzi/.helmignore

@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

charts/radar-s3-connector-strimzi/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: common
+  repository: https://radar-base.github.io/radar-helm-charts
+  version: 2.27.0
+digest: sha256:397fd12e07f1aa1a5280579406a75b169f8262ac4214584ea366460277288546
+generated: "2024-12-27T18:49:31.278906468+01:00"

charts/radar-s3-connector-strimzi/Chart.yaml

@@ -0,0 +1,27 @@
+apiVersion: v2
+appVersion: 3.9.0
+description: A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options
+name: radar-s3-connector-strimzi
+version: 0.0.1
+icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png"
+sources:
+- https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-s3-connector
+- https://github.com/RADAR-base/kafka-connect-transform-keyvalue
+- https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options
+keywords:
+  - radar-base
+  - remote-trial
+annotations:
+  artifacthub.io/license: Apache-2.0
+type: application
+home: "https://radar-base.org"
+maintainers:
+  - email: pim@thehyve.nl
+    name: Pim van Nierop
+    url: https://www.thehyve.nl/experts/pim-van-nierop
+dependencies:
+- name: common
+  repository: https://radar-base.github.io/radar-helm-charts
+  tags:
+  - bitnami-common
+  version: 2.x.x

charts/radar-s3-connector-strimzi/DOCS.md.gotmpl

@@ -0,0 +1,104 @@
+
+
+# radar-s3-connector-strimzi
+[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-s3-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-s3-connector-strimzi)
+
+![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.9.0](https://img.shields.io/badge/AppVersion-3.9.0-informational?style=flat-square)
+
+A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options
+
+**Homepage:** <https://radar-base.org>
+
+## Maintainers
+
+| Name | Email | Url |
+| ---- | ------ | --- |
+| Pim van Nierop | <pim@thehyve.nl> | <https://www.thehyve.nl/experts/pim-van-nierop> |
+
+## Source Code
+
+* <https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-s3-connector>
+* <https://github.com/RADAR-base/kafka-connect-transform-keyvalue>
+* <https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options>
+
+## Prerequisites
+* Kubernetes 1.28+
+* Kubectl 1.28+
+* Helm 3.1.0+
+
+## Requirements
+
+| Repository | Name | Version |
+|------------|------|---------|
+| https://radar-base.github.io/radar-helm-charts | common | 2.x.x |
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| replicaCount | int | `1` | Number of radar-s3-connector replicas to deploy |
+| image.registry | string | `"docker.io"` | Image registry |
+| image.repository | string | `"radarbase/kafka-connect-transform-s3"` | Image repository |
+| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. |
+| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag |
+| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy |
+| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets:   - myRegistryKeySecretName  |
+| nameOverride | string | `""` | String to partially override common.names.fullname template with a string (will prepend the release name) |
+| fullnameOverride | string | `""` | String to fully override common.names.fullname template with a string |
+| podSecurityContext | object | `{}` | Configure radar-s3-connector pods' Security Context |
+| securityContext | object | `{}` | Configure radar-s3-connector containers' Security Context |
+| service.type | string | `"ClusterIP"` | Kubernetes Service type |
+| service.port | int | `8083` | radar-s3-connector port |
+| resources.requests | object | `{"cpu":"100m","memory":"3Gi"}` | CPU/Memory resource requests |
+| jvmOptions.xmx | string | `"4g"` |  |
+| jvmOptions.xms | string | `"3g"` |  |
+| nodeSelector | object | `{}` | Node labels for pod assignment |
+| tolerations | list | `[]` | Toleration labels for pod assignment |
+| affinity | object | `{}` | Affinity labels for pod assignment |
+| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret for the Kafka SASL JAAS configuration |
+| extraEnvVars | list | `[]` | Extra environment variables |
+| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one |
+| livenessProbe.enabled | bool | `true` | Enable livenessProbe |
+| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe |
+| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe |
+| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe |
+| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe |
+| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe |
+| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one |
+| readinessProbe.enabled | bool | `true` | Enable readinessProbe |
+| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe |
+| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe |
+| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe |
+| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe |
+| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe |
+| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to |
+| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry |
+| radar_rest_sources_backend_url | string | `"http://radar-rest-sources-backend:8080/rest-sources/backend/"` | Base URL of the rest-sources-authorizer-backend service |
+| connector_num_tasks | string | `"5"` | Number of connector tasks to be used in kafka-connector spec properties |
+| catalogServer.url | string | `"http://catalog-server:9010"` | Catalog server URL |
+| topics | string | `""` | List of topics to be consumed by the sink connector separated by comma. Topics defined in the catalog server will automatically be loaded if `initTopics.enabled` is true. |
+| s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint url |
+| s3Tagging | bool | `false` | set to true, if S3 objects should be tagged with start and end offsets, as well as record count. |
+| s3PartSize | int | `5242880` | The Part Size in S3 Multi-part Uploads. |
+| s3Region | string | `nil` | The AWS region to be used the connector. Some compatibility layers require this. |
+| flushSize | int | `10000` | Number of records written to store before invoking file commits. |
+| rotateInterval | int | `900000` | The time interval in milliseconds to invoke file commits. |
+| maxTasks | int | `4` | Number of tasks in the connector |
+| bucketAccessKey | string | `"access_key"` | Access key of the target S3 bucket |
+| bucketSecretKey | string | `"secret"` | Secret key of the target S3 bucket |
+| bucketName | string | `"radar_intermediate_storage"` | Bucket name of the target S3 bucket |
+| cc.enabled | bool | `false` | Set to true, if Confluent Cloud is used |
+| cc.bootstrapServerurl | string | `""` | Confluent cloud based Kafka broker URL (if Confluent Cloud based Kafka cluster is used) |
+| cc.schemaRegistryUrl | string | `""` | Confluent cloud based Schema registry URL (if Confluent Cloud based Schema registry is used) |
+| cc.apiKey | string | `"ccApikey"` | API Key of the Confluent Cloud cluster |
+| cc.apiSecret | string | `"ccApiSecret"` | API secret of the Confluent Cloud cluster |
+| cc.schemaRegistryApiKey | string | `"srApiKey"` | API Key of the Confluent Cloud Schema registry |
+| cc.schemaRegistryApiSecret | string | `"srApiSecret"` | API Key of the Confluent Cloud Schema registry |
+| initTopics.enabled | bool | `true` | If true, fetch list of topics from catalog server |
+| initTopics.image.repository | string | `"linuxserver/yq"` | Image repository to fetch topics with |
+| initTopics.image.tag | string | `"3.2.2"` | Image tag to fetch topics with |
+| initTopics.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy to fetch topics with |
+| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration |
+| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance |
+| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server |
+| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) |

charts/radar-s3-connector-strimzi/README.md

@@ -0,0 +1,18 @@
+{{ template "common.header" . }}
+{{ template "chart.deprecationWarning" . }}
+
+{{ template "chart.badgesSection" . }}
+
+{{ template "chart.description" . }}
+
+{{ template "chart.homepageLine" . }}
+
+{{ template "chart.maintainersSection" . }}
+
+{{ template "chart.sourcesSection" . }}
+
+{{ template "common.prerequisites" . }}
+
+{{ template "chart.requirementsSection" . }}
+
+{{ template "chart.valuesSection" . }}

charts/radar-s3-connector-strimzi/README.md.gotmpl

@@ -0,0 +1,6 @@
+{{/*
+Return the proper image name
+*/}}
+{{- define "radar-s3-connector.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }}
+{{- end -}}

charts/radar-s3-connector-strimzi/charts/common-2.27.0.tgz

@@ -0,0 +1,28 @@
+# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification.
+# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "common.names.fullname" . }}-log4j
+  namespace: {{ include "common.names.namespace" . | quote }}
+  labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  log4j.properties: |-
+    # Root logger configuration
+    # The sentryAppender is added to the rootLogger if SENTRY_DSN is set
+    log4j.rootLogger=${env:CONNECT_LOG4J_ROOT_LOGLEVEL:-INFO}, stdout${env:SENTRY_DSN:+, sentryAppender}
+
+    log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+    log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+    log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n
+
+    # Appender for Sentry monitoring, only active if SENTRY_DSN is set
+    log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender
+    log4j.appender.sentryAppender.threshold=${env:SENTRY_LOG_LEVEL:-ERROR}
+
+    # Default log levels
+    log4j.logger.org.reflections=ERROR
+    log4j.logger.org.I0Itec.zkclient=ERROR