diff --git a/charts/management-portal/Chart.yaml b/charts/management-portal/Chart.yaml index edbe0170..48d4d360 100644 --- a/charts/management-portal/Chart.yaml +++ b/charts/management-portal/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: "2.1.8" description: A Helm chart for RADAR-Base Management Portal to manage projects and participants throughout RADAR-base. name: management-portal -version: 1.6.0 +version: 1.6.1 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/management-portal diff --git a/charts/management-portal/README.md b/charts/management-portal/README.md index 225700fb..277d199f 100644 --- a/charts/management-portal/README.md +++ b/charts/management-portal/README.md @@ -3,7 +3,7 @@ # management-portal [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/management-portal)](https://artifacthub.io/packages/helm/radar-base/management-portal) -![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.1.8](https://img.shields.io/badge/AppVersion-2.1.8-informational?style=flat-square) +![Version: 1.6.1](https://img.shields.io/badge/Version-1.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.1.8](https://img.shields.io/badge/AppVersion-2.1.8-informational?style=flat-square) A Helm chart for RADAR-Base Management Portal to manage projects and participants throughout RADAR-base. diff --git a/charts/management-portal/values.yaml b/charts/management-portal/values.yaml index 285e8b1d..9e5ed26b 100644 --- a/charts/management-portal/values.yaml +++ b/charts/management-portal/values.yaml @@ -548,6 +548,18 @@ oauth_clients: - client_credentials access_token_validity: 900 + radar_oura_connector: + enable: false + resource_ids: + - res_restAuthorizer + client_secret: "" + scope: + - SUBJECT.READ + - MEASUREMENT.CREATE + authorized_grant_types: + - client_credentials + access_token_validity: 900 + radar_appconfig: enable: false resource_ids: diff --git a/charts/radar-fitbit-connector-strimzi/.helmignore b/charts/radar-fitbit-connector-strimzi/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/radar-fitbit-connector-strimzi/Chart.lock b/charts/radar-fitbit-connector-strimzi/Chart.lock new file mode 100644 index 00000000..88085987 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + version: 2.27.0 +digest: sha256:397fd12e07f1aa1a5280579406a75b169f8262ac4214584ea366460277288546 +generated: "2025-04-08T16:44:09.158978088+02:00" diff --git a/charts/radar-fitbit-connector-strimzi/Chart.yaml b/charts/radar-fitbit-connector-strimzi/Chart.yaml new file mode 100644 index 00000000..17f0ff38 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +appVersion: 0.6.2 +description: A Helm chart for RADAR-base fitbit Kafka connector using the Strimzi Operator. This application collects data from participants via the Fitbit Web API. +name: radar-fitbit-connector-strimzi +version: 0.0.2 +icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" +sources: +- https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-fitbit-connector-strimzi +- https://github.com/RADAR-base/RADAR-REST-Connector +keywords: + - radar-base + - remote-trial +annotations: + artifacthub.io/license: Apache-2.0 +deprecated: false +type: application +home: "https://radar-base.org" +maintainers: + - email: pim@thehyve.nl + name: Pim van Nierop + url: https://www.thehyve.nl/experts/pim-van-nierop +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + tags: + - bitnami-common + version: 2.x.x diff --git a/charts/radar-fitbit-connector-strimzi/DOCS.md.gotmpl b/charts/radar-fitbit-connector-strimzi/DOCS.md.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-fitbit-connector-strimzi/README.md b/charts/radar-fitbit-connector-strimzi/README.md new file mode 100644 index 00000000..b255745f --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/README.md @@ -0,0 +1,96 @@ + + +# radar-fitbit-connector-strimzi +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-fitbit-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-fitbit-connector-strimzi) + +![Version: 0.0.2](https://img.shields.io/badge/Version-0.0.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) + +A Helm chart for RADAR-base fitbit Kafka connector using the Strimzi Operator. This application collects data from participants via the Fitbit Web API. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Pim van Nierop | | | + +## Source Code + +* +* + +## Prerequisites +* Kubernetes 1.28+ +* Kubectl 1.28+ +* Helm 3.1.0+ +* PV provisioner support in the underlying infrastructure + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://radar-base.github.io/radar-helm-charts | common | 2.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | Number of radar-fitbit-connector-strimzi replicas to deploy | +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"radarbase/kafka-connect-rest-fitbit-source"` | Image repository | +| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. | +| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets: - myRegistryKeySecretName | +| nameOverride | string | `""` | String to partially override radar-fitbit-connector-strimzi.fullname template with a string (will prepend the release name) | +| fullnameOverride | string | `""` | String to fully override radar-fitbit-connector-strimzi.fullname template with a string | +| podSecurityContext | object | `{}` | Configure radar-fitbit-connector-strimzi pods' Security Context | +| securityContext | object | `{}` | Configure radar-fitbit-connector-strimzi containers' Security Context | +| service.type | string | `"ClusterIP"` | Kubernetes Service type | +| service.port | int | `8083` | radar-fitbit-connector-strimzi port | +| resources.requests | object | `{"cpu":"100m","memory":"1Gi"}` | CPU/Memory resource requests | +| jvmOptions.xmx | string | `"768m"` | | +| jvmOptions.xms | string | `"256m"` | | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| tolerations | list | `[]` | Toleration labels for pod assignment | +| affinity | object | `{}` | Affinity labels for pod assignment | +| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret for the Kafka SASL JAAS configuration | +| extraEnvVars | list | `[]` | Extra environment variables | +| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one | +| livenessProbe.enabled | bool | `true` | Enable livenessProbe | +| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe | +| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe | +| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe | +| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe | +| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe | +| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one | +| readinessProbe.enabled | bool | `true` | Enable readinessProbe | +| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe | +| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe | +| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe | +| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe | +| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe | +| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to | +| maxTasks | int | `5` | Maximum number of worker threads inside a connector pod. | +| kafkaVersion | string | `"3.9.0"` | The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. | +| kafka | string | `"SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094"` | URI of Kafka brokers of the cluster | +| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry | +| radar_rest_sources_backend_url | string | `"http://radar-rest-sources-backend:8080/rest-sources/backend/"` | Base URL of the rest-sources-authorizer-backend service | +| fitbit_api_url | string | `"https://api.fitbit.com"` | Fitbit API URL. | +| fitbit_api_client | string | `""` | Fitbit API client id. | +| fitbit_api_secret | string | `""` | Fitbit API client secret. | +| oauthClientId | string | `"radar_fitbit_connector"` | OAuth2 client id from Management Portal | +| oauthClientSecret | string | `"secret"` | OAuth2 client secret from Management Portal | +| auth_url | string | `"http://management-portal:8080/managementportal/oauth/token"` | OAuth2 Auth URL for connector client to get access tokens | +| managementportal_url | string | `"http://management-portal:8080/managementportal"` | URL of Management Portal. This will be used to create URLs to access Management Portal | +| includeIntradayData | bool | `true` | Set to true, if intraday access data should be collected by the connector. This will be set in connector.properties. | +| user_repository_class | string | `"org.radarbase.connect.rest.fitbit.user.ServiceUserRepositoryLegacy"` | Class of the user repository to use. This should be a class that implements the UserRepository interface. Use of the YamlUserRepository is deprecated, use ServiceUserRepositoryLegacy instead. | +| rest_source_poll_interval_ms | int | `60000` | How often to poll the source URL. Only use to speed up processing times during e2e testing. | +| fitbit_user_poll_interval | int | `5000` | Polling interval per Fitbit user per request route in seconds. Only use to speed up processing times during e2e testing. | +| application_loop_interval_ms | int | `300000` | How often to perform the main application loop (only controls how often to poll for new user registrations). Only use to speed up processing times during e2e testing. | +| user_cache_refresh_interval_ms | int | `3600000` | How often to invalidate the cache and poll for new user registrations. Only use to speed up processing times during e2e testing. | +| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration | +| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance | +| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server | +| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) | diff --git a/charts/radar-fitbit-connector-strimzi/README.md.gotmpl b/charts/radar-fitbit-connector-strimzi/README.md.gotmpl new file mode 100644 index 00000000..929d551e --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/README.md.gotmpl @@ -0,0 +1,18 @@ +{{ template "common.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "common.prerequisiteswithpv" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/radar-fitbit-connector-strimzi/charts/common-2.27.0.tgz b/charts/radar-fitbit-connector-strimzi/charts/common-2.27.0.tgz new file mode 100644 index 00000000..5c539c35 Binary files /dev/null and b/charts/radar-fitbit-connector-strimzi/charts/common-2.27.0.tgz differ diff --git a/charts/radar-fitbit-connector-strimzi/templates/NOTES.txt b/charts/radar-fitbit-connector-strimzi/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-fitbit-connector-strimzi/templates/_helpers.tpl b/charts/radar-fitbit-connector-strimzi/templates/_helpers.tpl new file mode 100644 index 00000000..ba7ccf49 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Return the proper image name +*/}} +{{- define "radar-fitbit-connector.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} diff --git a/charts/radar-fitbit-connector-strimzi/templates/configmap-log4j.yaml b/charts/radar-fitbit-connector-strimzi/templates/configmap-log4j.yaml new file mode 100644 index 00000000..2a0aa4b3 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/configmap-log4j.yaml @@ -0,0 +1,29 @@ +# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification. +# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }}-log4j + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + # Root logger configuration + # The sentryAppender is added to the rootLogger if SENTRY_DSN is set + log4j.rootLogger={{ .Values.log4j.rootLogLevel | default "INFO" }}, stdout{{- if .Values.sentry.dsn }}, sentryAppender{{- end }} + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Appender for Sentry monitoring + {{- if .Values.sentry.dsn }} + log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender + log4j.appender.sentryAppender.threshold={{ .Values.sentry.level | default "INFO" }} + {{- end }} + + # Default log levels + log4j.logger.org.reflections=ERROR + log4j.logger.org.I0Itec.zkclient=ERROR diff --git a/charts/radar-fitbit-connector-strimzi/templates/kafka-connect.yaml b/charts/radar-fitbit-connector-strimzi/templates/kafka-connect.yaml new file mode 100644 index 00000000..ebdb940b --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/kafka-connect.yaml @@ -0,0 +1,133 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + version: {{ .Values.kafkaVersion }} + replicas: {{ .Values.replicaCount }} + bootstrapServers: {{ .Values.kafka }} + image: {{ template "radar-fitbit-connector.image" . }} + {{- if .Values.secret.jaas.name }} + # Authentication-related props are handled by the template builder in https://github.com/strimzi/strimzi-kafka-operator/blob/main/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaConnectConfigurationBuilder.java + authentication: + type: scram-sha-512 + username: shared-service-user + passwordSecret: + secretName: {{ .Values.secret.jaas.name }} + password: password # The name of the key in the secret that contains the password + {{- end }} + autoRestart: + enabled: true # automatic restarts of failed connectors or their tasks + jvmOptions: + "-Xmx": {{ .Values.jvmOptions.xmx | quote }} + "-Xms": {{ .Values.jvmOptions.xms | quote }} + config: # Properties that configure the Kafka Connect cluster itself + group.id: {{ include "common.names.fullname" . }} + offset.storage.topic: {{ template "common.names.fullname" . }}.offset + config.storage.topic: {{ template "common.names.fullname" . }}.config + status.storage.topic: {{ template "common.names.fullname" . }}.status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + key.converter: "io.confluent.connect.avro.AvroConverter" + value.converter: "io.confluent.connect.avro.AvroConverter" + key.converter.schema.registry.url: {{ .Values.schema_registry }} + value.converter.schema.registry.url: {{ .Values.schema_registry }} + internal.key.converter: "org.apache.kafka.connect.json.JsonConverter" + internal.value.converter: "org.apache.kafka.connect.json.JsonConverter" + template: + pod: + metadata: + annotations: + checksum/configmap-log4j: {{ include (print $.Template.BasePath "/configmap-log4j.yaml") . | sha256sum }} + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + affinity: + {{- if .Values.affinity }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - {{ template "common.names.name" . }} + - key: "app.kubernetes.io/instance" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + connectContainer: + env: + - name: CONNECT_REST_ADVERTISED_HOST_NAME # Possibly replaced by ADVERTISED_HOSTNAME in kafka_connect_run.sh + value: {{ include "common.names.fullname" . }} + - name: KAFKA_LOG4J_OPTS + value: "-Dlog4j.configuration=file:/opt/kafka/custom-config/log4j.properties" + {{- if .Values.sentry.dsn }} + - name: SENTRY_DSN + value: {{ .Values.sentry.dsn | quote }} + - name: SENTRY_LOG_LEVEL + value: {{ .Values.sentry.level | quote }} + {{- end }} + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 8 }} + {{- end }} + logging: + type: external + valueFrom: + configMapKeyRef: + name: {{ template "common.names.fullname" . }}-log4j + key: log4j.properties + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- .Values.customLivenessProbe | toYaml | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- .Values.customReadinessProbe | toYaml | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 4 }} diff --git a/charts/radar-fitbit-connector-strimzi/templates/kafka-connector.yaml b/charts/radar-fitbit-connector-strimzi/templates/kafka-connector.yaml new file mode 100644 index 00000000..3d6e6096 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/kafka-connector.yaml @@ -0,0 +1,25 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: + strimzi.io/cluster: {{ template "common.names.fullname" . }} +spec: + class: org.radarbase.connect.rest.fitbit.FitbitSourceConnector + tasksMax: {{ .Values.maxTasks }} + config: + fitbit.api.client: {{ .Values.fitbit_api_client }} + fitbit.api.secret: {{ .Values.fitbit_api_secret }} + fitbit.api.intraday: {{ .Values.includeIntradayData }} + fitbit.user.repository.class: {{ .Values.user_repository_class }} + fitbit.user.repository.url: {{ .Values.radar_rest_sources_backend_url }} + fitbit.user.repository.client.id: {{ .Values.oauthClientId }} + fitbit.user.repository.client.secret: {{ .Values.oauthClientSecret }} + fitbit.user.repository.oauth2.token.url: {{ .Values.managementportal_url }}/oauth/token + fitbit.user.poll.interval: {{ .Values.fitbit_user_poll_interval | int }} + application.loop.interval.ms: {{ .Values.application_loop_interval_ms | int }} + user.cache.refresh.interval.ms: {{ .Values.user_cache_refresh_interval_ms | int }} + rest.source.base.url: {{ .Values.fitbit_api_url }} + rest.source.poll.interval.ms: {{ .Values.rest_source_poll_interval_ms | int }} + rest.source.request.generator.class: org.radarbase.connect.rest.fitbit.request.FitbitRequestGenerator diff --git a/charts/radar-fitbit-connector-strimzi/templates/networkpolicy.yaml b/charts/radar-fitbit-connector-strimzi/templates/networkpolicy.yaml new file mode 100644 index 00000000..7c4edf91 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkpolicy }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: +{{ include "common.labels.standard" . | indent 6 }} + {{- tpl (toYaml .Values.networkpolicy) . | nindent 2 }} +{{- end -}} diff --git a/charts/radar-fitbit-connector-strimzi/templates/service.yaml b/charts/radar-fitbit-connector-strimzi/templates/service.yaml new file mode 100644 index 00000000..d8c4f364 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} diff --git a/charts/radar-fitbit-connector-strimzi/templates/tests/test-connection.yaml b/charts/radar-fitbit-connector-strimzi/templates/tests/test-connection.yaml new file mode 100644 index 00000000..f4959a02 --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + helm.sh/chart: {{ include "common.names.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/radar-fitbit-connector-strimzi/values.yaml b/charts/radar-fitbit-connector-strimzi/values.yaml new file mode 100644 index 00000000..c25374ee --- /dev/null +++ b/charts/radar-fitbit-connector-strimzi/values.yaml @@ -0,0 +1,231 @@ +# Default values for radar-fitbit-connector-strimzi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of radar-fitbit-connector-strimzi replicas to deploy +replicaCount: 1 + +image: + # -- Image registry + registry: docker.io + # -- Image repository + repository: radarbase/kafka-connect-rest-fitbit-source + # -- Image tag (immutable tags are recommended) + # Overrides the image tag whose default is the chart appVersion. + tag: + # -- Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + digest: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # e.g: + # pullSecrets: + # - myRegistryKeySecretName + # + pullSecrets: [] + +# -- String to partially override radar-fitbit-connector-strimzi.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override radar-fitbit-connector-strimzi.fullname template with a string +fullnameOverride: "" + +# -- Configure radar-fitbit-connector-strimzi pods' Security Context +podSecurityContext: {} + # fsGroup: 2000 + +# -- Configure radar-fitbit-connector-strimzi containers' Security Context +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # -- Kubernetes Service type + type: ClusterIP + # -- radar-fitbit-connector-strimzi port + port: 8083 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + + # -- CPU/Memory resource requests + requests: + cpu: 100m + memory: 1Gi + +jvmOptions: + xmx: "768m" + xms: "256m" + +# -- Node labels for pod assignment +nodeSelector: {} + +# -- Toleration labels for pod assignment +tolerations: [] + +# -- Affinity labels for pod assignment +affinity: {} + +secret: + # -- Secret for the Kafka SASL JAAS configuration + jaas: + name: shared-service-user + key: sasl.jaas.config + +# -- Extra environment variables +extraEnvVars: [] + +# -- Custom livenessProbe that overrides the default one +customLivenessProbe: {} + +livenessProbe: + # -- Enable livenessProbe + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 5 + # -- Success threshold for livenessProbe + successThreshold: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + +# -- Custom readinessProbe that overrides the default one +customReadinessProbe: {} + +readinessProbe: + # -- Enable readinessProbe + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 5 + # -- Success threshold for readinessProbe + successThreshold: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + +# -- Network policy defines who can access this application and who this applications has access to +# @default -- check `values.yaml` +networkpolicy: + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 192.168.0.0/16 + - 172.16.0.0/12 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-kafka-bootstrap' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-schema-registry' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-rest-sources-backend' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'management-portal' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + +# -- Maximum number of worker threads inside a connector pod. +maxTasks: 5 + +# -- The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. +kafkaVersion: 3.9.0 + +# -- URI of Kafka brokers of the cluster +kafka: SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094 +# -- URL of the Kafka schema registry +schema_registry: http://radar-kafka-schema-registry:8081 +# -- Base URL of the rest-sources-authorizer-backend service +radar_rest_sources_backend_url: http://radar-rest-sources-backend:8080/rest-sources/backend/ + +# -- Fitbit API URL. +fitbit_api_url: https://api.fitbit.com +# -- Fitbit API client id. +fitbit_api_client: "" +# -- Fitbit API client secret. +fitbit_api_secret: "" + +# -- OAuth2 client id from Management Portal +oauthClientId: radar_fitbit_connector +# -- OAuth2 client secret from Management Portal +oauthClientSecret: secret +# -- OAuth2 Auth URL for connector client to get access tokens +auth_url: http://management-portal:8080/managementportal/oauth/token +# -- URL of Management Portal. This will be used to create URLs to access Management Portal +managementportal_url: http://management-portal:8080/managementportal +# -- Set to true, if intraday access data should be collected by the connector. This will be set in connector.properties. +includeIntradayData: true + +# -- Class of the user repository to use. This should be a class that implements the UserRepository interface. +# Use of the YamlUserRepository is deprecated, use ServiceUserRepositoryLegacy instead. +user_repository_class: org.radarbase.connect.rest.fitbit.user.ServiceUserRepositoryLegacy + +# -- How often to poll the source URL. +# Only use to speed up processing times during e2e testing. +rest_source_poll_interval_ms: 60000 +# -- Polling interval per Fitbit user per request route in seconds. +# Only use to speed up processing times during e2e testing. +fitbit_user_poll_interval: 5000 +# -- How often to perform the main application loop (only controls how often to poll for new user registrations). +# Only use to speed up processing times during e2e testing. +application_loop_interval_ms: 300000 +# -- How often to invalidate the cache and poll for new user registrations. +# Only use to speed up processing times during e2e testing. +user_cache_refresh_interval_ms: 3600000 + +# -- Log4j configuration +log4j: + # -- Root log level for the Kafka Connect instance + rootLogLevel: INFO + +# Sentry monitoring configuration +sentry: + # -- DSN (Data Source Name) of the sentry server + dsn: + # -- Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) + level: ERROR diff --git a/charts/radar-fitbit-connector/Chart.yaml b/charts/radar-fitbit-connector/Chart.yaml index 7c57e640..119e3c01 100644 --- a/charts/radar-fitbit-connector/Chart.yaml +++ b/charts/radar-fitbit-connector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.6.2 description: A Helm chart for RADAR-base fitbit connector. This application collects data from participants via the Fitbit Web API. name: radar-fitbit-connector -version: 0.9.1 +version: 0.10.0 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-fitbit-connector @@ -12,7 +12,7 @@ keywords: - remote-trial annotations: artifacthub.io/license: Apache-2.0 -deprecated: false +deprecated: true type: application home: "https://radar-base.org" maintainers: diff --git a/charts/radar-fitbit-connector/README.md b/charts/radar-fitbit-connector/README.md index 82eeb95e..704d75be 100644 --- a/charts/radar-fitbit-connector/README.md +++ b/charts/radar-fitbit-connector/README.md @@ -2,8 +2,9 @@ # radar-fitbit-connector [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-fitbit-connector)](https://artifacthub.io/packages/helm/radar-base/radar-fitbit-connector) +> **:exclamation: This Helm Chart is deprecated!** -![Version: 0.9.1](https://img.shields.io/badge/Version-0.9.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) +![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) A Helm chart for RADAR-base fitbit connector. This application collects data from participants via the Fitbit Web API. diff --git a/charts/radar-jdbc-connector-strimzi/.helmignore b/charts/radar-jdbc-connector-strimzi/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/radar-jdbc-connector-strimzi/Chart.lock b/charts/radar-jdbc-connector-strimzi/Chart.lock new file mode 100644 index 00000000..f91876e6 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/Chart.lock @@ -0,0 +1,9 @@ +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + version: 2.27.0 +- name: radar-cloudnative-timescaledb + repository: file://../radar-cloudnative-timescaledb + version: 0.2.0 +digest: sha256:981cd25bb9d292f14c113b34c6dcef00e92843f35372e6818dc235acd5743ba2 +generated: "2025-07-08T09:10:01.02024042+02:00" diff --git a/charts/radar-jdbc-connector-strimzi/Chart.yaml b/charts/radar-jdbc-connector-strimzi/Chart.yaml new file mode 100644 index 00000000..d3cbcd5b --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/Chart.yaml @@ -0,0 +1,37 @@ +apiVersion: v2 +appVersion: 3.9.0 +name: radar-jdbc-connector-strimzi +description: A Helm chart for RADAR-base JDBC Kafka connector for Strimzi Connect. This is a f rk of the Kafka JDBC connector which allows data from topics to be imported into JDBC databases (including TimescaleDB databases which is used in the dashboard pipeline). +version: 0.10.1 +icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" +sources: +- https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-jdbc-connector +- https://github.com/RADAR-base/RADAR-JDBC-Connector +keywords: + - radar-base + - remote-trial +annotations: + artifacthub.io/license: Apache-2.0 +deprecated: false +type: application +home: "https://radar-base.org" +maintainers: + - email: pauline.conde@kcl.ac.uk + name: Pauline Conde + url: https://www.kcl.ac.uk/people/pauline-conde + - email: pim@thehyve.nl + name: Pim van Nierop + url: https://www.thehyve.nl/experts/pim-van-nierop +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + tags: + - bitnami-common + version: 2.x.x +- alias: timescaledb + name: radar-cloudnative-timescaledb + condition: radar-cloudnative-timescaledb.enabled + repository: file://../radar-cloudnative-timescaledb + tags: + - timescaledb + version: 0.2.0 diff --git a/charts/radar-jdbc-connector-strimzi/DOCS.md.gotmpl b/charts/radar-jdbc-connector-strimzi/DOCS.md.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-jdbc-connector-strimzi/README.md b/charts/radar-jdbc-connector-strimzi/README.md new file mode 100644 index 00000000..869b61bc --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/README.md @@ -0,0 +1,113 @@ + + +# radar-jdbc-connector-strimzi +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-jdbc-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-jdbc-connector-strimzi) + +![Version: 0.10.1](https://img.shields.io/badge/Version-0.10.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.9.0](https://img.shields.io/badge/AppVersion-3.9.0-informational?style=flat-square) + +A Helm chart for RADAR-base JDBC Kafka connector for Strimzi Connect. This is a f rk of the Kafka JDBC connector which allows data from topics to be imported into JDBC databases (including TimescaleDB databases which is used in the dashboard pipeline). + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Pauline Conde | | | +| Pim van Nierop | | | + +## Source Code + +* +* + +## Prerequisites +* Kubernetes 1.28+ +* Kubectl 1.28+ +* Helm 3.1.0+ +* PV provisioner support in the underlying infrastructure + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| file://../radar-cloudnative-timescaledb | timescaledb(radar-cloudnative-timescaledb) | 0.2.0 | +| https://radar-base.github.io/radar-helm-charts | common | 2.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"radarbase/radar-jdbc-connector"` | Image repository | +| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. | +| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets: - myRegistryKeySecretName | +| nameOverride | string | `""` | String to partially override common.names.fullname template with a string (will prepend the release name) | +| fullnameOverride | string | `""` | String to fully override common.names.fullname template with a string | +| podSecurityContext | object | `{}` | Configure radar-jdbc-connector pods' Security Context | +| securityContext | object | `{}` | Configure radar-jdbc-connector containers' Security Context | +| service.type | string | `"ClusterIP"` | Kubernetes Service type | +| service.port | int | `8083` | radar-jdbc-connector port | +| resources.requests | object | `{"cpu":"100m","memory":"1Gi"}` | CPU/Memory resource requests | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| tolerations | list | `[]` | Toleration labels for pod assignment | +| affinity | object | `{}` | Affinity labels for pod assignment | +| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret name for the JAAS configuration | +| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one | +| livenessProbe.enabled | bool | `true` | Enable livenessProbe | +| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe | +| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe | +| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe | +| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe | +| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe | +| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one | +| readinessProbe.enabled | bool | `true` | Enable readinessProbe | +| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe | +| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe | +| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe | +| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe | +| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe | +| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to | +| maxTasks | int | `5` | Maximum number of worker threads inside a connector pod. | +| kafkaVersion | string | `"3.9.0"` | The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. | +| kafka | string | `"SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094"` | URI of Kafka brokers of the cluster | +| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry | +| mode | string | `"sink"` | Either source or sink | +| jvmOptions | object | `{"xms":"1500m","xmx":"2500m"}` | Java heap options | +| source.name | string | `"radar-jdbc-source"` | Name of the connector Kafka producer group | +| source.schema | string | `"public"` | Database schema (if any) | +| source.tableWhitelist | string | `""` | Comma-separated list of tables to read | +| source.topicPrefix | string | `""` | Prefix to prepend to table names to generate the name of the Kafka topic to publish data to. | +| source.mode | string | `"incrementing"` | How to detect new values in a table. | +| source.incrementingColumnName | string | `""` | When using mode incrementing, which column to use as incrementing. If empty, autodetection will be used. | +| source.keyField | string | `""` | Field to use as key for the records. If empty, no key is used. | +| source.persistence.enabled | bool | `true` | Whether to enable persistence for storing offsets | +| source.persistence.existingClaim | string | `nil` | Existing persistent volume claim to use | +| source.persistence.accessMode | string | `"ReadWriteOnce"` | PVC access mode | +| source.persistence.size | string | `"20Mi"` | PVC storage size request | +| sink.name | string | `"radar-jdbc-sink"` | Name of the connector Kafka consumer group | +| sink.autoCreate | bool | `true` | create table if it does not exist | +| sink.insertMode | string | `"upsert"` | How to insert new values into the database | +| sink.transforms.enabled | bool | `true` | Whether to merge the key fields into the inserted values. | +| sink.transforms.mergeKeyType | string | `"org.radarbase.kafka.connect.transforms.MergeKey"` | Class to handle merging the key into the record | +| sink.transforms.timestampType | string | `"org.radarbase.kafka.connect.transforms.TimestampConverter"` | Class to convert time-based fields into SQL timestamps | +| sink.transforms.timestampFields | list | `["time","timeReceived","timeCompleted","timestamp"]` | Field names to consider for converting to timestamp | +| sink.primaryKeys.mode | string | `"record_value"` | where to read the primary keys from when creating the table | +| sink.primaryKeys.fields | list | `["time","userId","projectId"]` | fields to include as primary keys when creating the table | +| sink.topics | string | `"android_phone_relative_location, android_phone_battery_level, connect_upload_altoida_summary, connect_fitbit_intraday_heart_rate, connect_fitbit_intraday_steps"` | Comma-separated list of topics the connector will read from and ingest into the database | +| sink.tableNameFormat | string | `"${topic}"` | How to format a table name based on the inserted topic | +| jdbc.url | string | `nil` | Host of the TimescaleDB database | +| jdbc.urlSecret | object | `{"key":null,"name":null}` | Kubernetes secret name for the JDBC connection URL (disables the use of 'url' value) | +| jdbc.user | string | `nil` | TimescaleDB database username | +| jdbc.userSecret | object | `{"key":null,"name":null}` | Kubernetes secret name for the username (disables the use of 'user' value) | +| jdbc.password | string | `nil` | TimescaleDB database password (using a secret is recommended) | +| jdbc.passwordSecret | object | `{"key":null,"name":null}` | Kubernetes secret name for the password | +| jdbc.dialect | string | `"TimescaleDBDatabaseDialect"` | JDBC connect dialect that the database uses | +| timescaledb.enabled | bool | `true` | Use the local cloudnativepg timescaledb cluster | +| timescaledb.cluster | object | check `values.yaml` | CloudNativePG TimescaleDB configuration | +| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration | +| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance | +| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server | +| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) | diff --git a/charts/radar-jdbc-connector-strimzi/README.md.gotmpl b/charts/radar-jdbc-connector-strimzi/README.md.gotmpl new file mode 100644 index 00000000..929d551e --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/README.md.gotmpl @@ -0,0 +1,18 @@ +{{ template "common.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "common.prerequisiteswithpv" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/radar-jdbc-connector-strimzi/charts/common-2.27.0.tgz b/charts/radar-jdbc-connector-strimzi/charts/common-2.27.0.tgz new file mode 100644 index 00000000..5c539c35 Binary files /dev/null and b/charts/radar-jdbc-connector-strimzi/charts/common-2.27.0.tgz differ diff --git a/charts/radar-jdbc-connector-strimzi/charts/radar-cloudnative-timescaledb-0.2.0.tgz b/charts/radar-jdbc-connector-strimzi/charts/radar-cloudnative-timescaledb-0.2.0.tgz new file mode 100644 index 00000000..3b068332 Binary files /dev/null and b/charts/radar-jdbc-connector-strimzi/charts/radar-cloudnative-timescaledb-0.2.0.tgz differ diff --git a/charts/radar-jdbc-connector-strimzi/templates/NOTES.txt b/charts/radar-jdbc-connector-strimzi/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-jdbc-connector-strimzi/templates/_helpers.tpl b/charts/radar-jdbc-connector-strimzi/templates/_helpers.tpl new file mode 100644 index 00000000..bafdc1b0 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/_helpers.tpl @@ -0,0 +1,76 @@ +{{/* +Return the proper image name +*/}} +{{- define "radar-jdbc-connector.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} + +{{/* +Get the name of the secret object. +*/}} +{{- define "radar-jdbc-connector.secretName" -}} +{{- $useCloudnative := .context.Values.timescaledb.enabled }} +{{- $suffix := ternary "-app" "" $useCloudnative }} +{{- $fullName := ternary (.context.Values.timescaledb.cluster.fullnameOverride) (include "common.names.fullname" .context) $useCloudnative }} +{{- $fullSecretName := print $fullName $suffix }} +{{- if (eq .type "user") }} + {{- .context.Values.jdbc.userSecret.name | default $fullSecretName }} +{{- else if (eq .type "password") }} + {{- .context.Values.jdbc.passwordSecret.name | default $fullSecretName }} +{{- else if (eq .type "url") }} + {{- .context.Values.jdbc.urlSecret.name | default $fullSecretName }} +{{- end -}} +{{- end -}} + +{{/* +Get the key for the secret object. +*/}} +{{- define "radar-jdbc-connector.secretKey" -}} +{{- $useCloudnative := .context.Values.timescaledb.enabled }} +{{- if (eq .type "user") }} + {{- if $useCloudnative }} + {{- "username" }} + {{- else }} + {{ .context.Values.jdbc.userSecret.key | default "databaseUser" }} + {{- end }} +{{- else if (eq .type "password") }} + {{- if $useCloudnative }} + {{- "password" }} + {{- else }} + {{ .context.Values.jdbc.passwordSecret.key | default "databasePassword" }} + {{- end }} +{{- else if (eq .type "url") }} + {{- if $useCloudnative }} + {{- "jdbc-uri" }} + {{- else }} + {{ .context.Values.jdbc.urlSecret.key | default "databaseUrl" }} + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Get the database url. Has to be created before the secret is created. +*/}} +{{- define "radar-jdbc-connector.databaseUrl" -}} +{{- $useCloudnative := .Values.timescaledb.enabled }} +{{- $port := .Values.jdbc.port | default 5432 }} +{{- if $useCloudnative -}} + {{- include "radar-jdbc-connector.cloudnativeDatabaseUrl" . -}} +{{- else if .Values.jdbc.urlSecret.name -}} + {{- print include "common.secrets.lookup" (dict "secret" .Values.jdbc.urlSecret.name "key" .Values.jdbc.urlSecret.key "default") -}} +{{- else -}} + {{- .Values.jdbc.url -}} +{{- end -}} +{{- end -}} + +{{/* +Construct the Cloudnative-PG database URL +*/}} +{{- define "radar-jdbc-connector.cloudnativeDatabaseUrl" -}} +{{- $port := .Values.jdbc.port | default 5432 }} +{{- $dbConfig := .Values.timescaledb.cluster }} +{{- $dbname := $dbConfig.cluster.initdb.database | default "postgres" }} +{{- $override := $dbConfig.nameOverride | default "cluster" }} +{{- $dbServiceName := $dbConfig.fullnameOverride | default (printf "%s-%s" .Release.Name $override) -}} +{{- printf "jdbc:postgresql://%s-rw.default:%d/%s" $dbServiceName $port $dbname -}} +{{- end -}} diff --git a/charts/radar-jdbc-connector-strimzi/templates/configmap-log4j.yaml b/charts/radar-jdbc-connector-strimzi/templates/configmap-log4j.yaml new file mode 100644 index 00000000..2a0aa4b3 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/configmap-log4j.yaml @@ -0,0 +1,29 @@ +# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification. +# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }}-log4j + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + # Root logger configuration + # The sentryAppender is added to the rootLogger if SENTRY_DSN is set + log4j.rootLogger={{ .Values.log4j.rootLogLevel | default "INFO" }}, stdout{{- if .Values.sentry.dsn }}, sentryAppender{{- end }} + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Appender for Sentry monitoring + {{- if .Values.sentry.dsn }} + log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender + log4j.appender.sentryAppender.threshold={{ .Values.sentry.level | default "INFO" }} + {{- end }} + + # Default log levels + log4j.logger.org.reflections=ERROR + log4j.logger.org.I0Itec.zkclient=ERROR diff --git a/charts/radar-jdbc-connector-strimzi/templates/kafka-connect.yaml b/charts/radar-jdbc-connector-strimzi/templates/kafka-connect.yaml new file mode 100644 index 00000000..6f8ebb4f --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/kafka-connect.yaml @@ -0,0 +1,141 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + version: {{ .Values.kafkaVersion }} + replicas: {{ .Values.replicaCount }} + bootstrapServers: {{ .Values.kafka }} + image: {{ template "radar-jdbc-connector.image" . }} + {{- if .Values.secret.jaas.name }} + # Authentication-related props are handled by the template builder in https://github.com/strimzi/strimzi-kafka-operator/blob/main/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaConnectConfigurationBuilder.java + authentication: + type: scram-sha-512 + username: shared-service-user + passwordSecret: + secretName: {{ .Values.secret.jaas.name }} + password: password # The name of the key in the secret that contains the password + {{- end }} + autoRestart: + enabled: true # automatic restarts of failed connectors or their tasks + jvmOptions: + "-Xmx": {{ .Values.jvmOptions.xmx | quote }} + "-Xms": {{ .Values.jvmOptions.xms | quote }} + config: # Properties that configure the Kafka Connect cluster itself + group.id: {{ include "common.names.fullname" . }} + offset.storage.topic: {{ template "common.names.fullname" . }}.offset + config.storage.topic: {{ template "common.names.fullname" . }}.config + status.storage.topic: {{ template "common.names.fullname" . }}.status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + key.converter: "io.confluent.connect.avro.AvroConverter" + value.converter: "io.confluent.connect.avro.AvroConverter" + key.converter.schema.registry.url: {{ .Values.schema_registry }} + value.converter.schema.registry.url: {{ .Values.schema_registry }} + template: + pod: + metadata: + annotations: + checksum/configmap-log4j: {{ include (print $.Template.BasePath "/configmap-log4j.yaml") . | sha256sum }} + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + affinity: + {{- if .Values.affinity }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - {{ template "common.names.name" . }} + - key: "app.kubernetes.io/instance" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + connectContainer: + env: + - name: CONNECT_JDBC_CONNECTION_USER + valueFrom: + secretKeyRef: + name: {{ include "radar-jdbc-connector.secretName" (dict "type" "user" "context" .) }} + key: {{ include "radar-jdbc-connector.secretKey" (dict "type" "user" "context" .) }} + - name: CONNECT_JDBC_CONNECTION_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "radar-jdbc-connector.secretName" (dict "type" "password" "context" .) }} + key: {{ include "radar-jdbc-connector.secretKey" (dict "type" "password" "context" .) }} + - name: CONNECT_REST_ADVERTISED_HOST_NAME # Possibly replaced by ADVERTISED_HOSTNAME in kafka_connect_run.sh + value: {{ include "common.names.fullname" . }} + - name: KAFKA_LOG4J_OPTS + value: "-Dlog4j.configuration=file:/opt/kafka/custom-config/log4j.properties" + {{- if .Values.sentry.dsn }} + - name: SENTRY_DSN + value: {{ .Values.sentry.dsn | quote }} + - name: SENTRY_LOG_LEVEL + value: {{ .Values.sentry.level | quote }} + {{- end }} + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 8 }} + {{- end }} + logging: + type: external + valueFrom: + configMapKeyRef: + name: {{ template "common.names.fullname" . }}-log4j + key: log4j.properties + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- .Values.customLivenessProbe | toYaml | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- .Values.customReadinessProbe | toYaml | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 4 }} diff --git a/charts/radar-jdbc-connector-strimzi/templates/kafka-connector.yaml b/charts/radar-jdbc-connector-strimzi/templates/kafka-connector.yaml new file mode 100644 index 00000000..c45e2a7c --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/kafka-connector.yaml @@ -0,0 +1,64 @@ +{{- $isSource := eq .Values.mode "source" -}} +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: + strimzi.io/cluster: {{ template "common.names.fullname" . }} +spec: + class: io.confluent.connect.jdbc.JdbcSinkConnector + tasksMax: {{ .Values.maxTasks }} + config: # Properties that configure Fitbit source connector + {{/*At moment of this writing, 'connection.url' cannot be set from environment variable with JDBC-connector. + As such, it has to be added in config file and is created when any URL secret has been created*/}} + connection.url: {{ include "radar-jdbc-connector.databaseUrl" . }} + jdbc.credentials.provider.class: io.confluent.connect.jdbc.util.EnvVarsJdbcCredentialsProvider + dialect.name: {{ .Values.jdbc.dialect }} + max.retries: 15 + retry.backoff.ms: 600000 + connection.attempts: 15 + connection.backoff.ms: 600000 + + {{- if not $isSource }} + connector.class: io.confluent.connect.jdbc.JdbcSinkConnector + + {{- with .Values.sink }} + insert.mode: {{ .insertMode }} + topics: {{ .topics }} + table.name.format: {{ .tableNameFormat }} + auto.create: {{ .autoCreate }} + {{- with .primaryKeys }} + pk.mode: {{ .mode }} + pk.fields: {{ join ", " .fields }} + {{- end }} + + {{- if or (dig "mergeKey" false .) (dig "transforms" "enabled" false .) }} + {{- with .transforms }} + transforms: mergeKey,timestamp + transforms.mergeKey.type: {{ .mergeKeyType }} + transforms.timestamp.type: {{ .timestampType }} + transforms.timestamp.fields: {{ join ", " .timestampFields }} + {{- end }} + {{- end }} + {{- end }} + {{- end }}}} + + {{- if $isSource }} + {{- with .Values.source }} + {{- if .schema }} + schema.pattern: {{ .schema }} + {{- end }} + table.whitelist: {{ .tableWhitelist }} + topic.prefix: {{ .topicPrefix }} + mode: {{ .mode }} + incrementing.column.name: {{ .incrementingColumnName }} + {{- if .keyField }} + transforms: createKey,extractField + transforms.createKey.type: org.apache.kafka.connect.transforms.ValueToKey + transforms.createKey.fields: {{ .keyField }} + transforms.extractField.type: org.apache.kafka.connect.transforms.ExtractField$Key + transforms.extractField.field: {{ .keyField }} + {{- end }} + {{- end }} + {{- end }} diff --git a/charts/radar-jdbc-connector-strimzi/templates/networkpolicy.yaml b/charts/radar-jdbc-connector-strimzi/templates/networkpolicy.yaml new file mode 100644 index 00000000..7c4edf91 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkpolicy }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: +{{ include "common.labels.standard" . | indent 6 }} + {{- tpl (toYaml .Values.networkpolicy) . | nindent 2 }} +{{- end -}} diff --git a/charts/radar-jdbc-connector-strimzi/templates/service.yaml b/charts/radar-jdbc-connector-strimzi/templates/service.yaml new file mode 100644 index 00000000..d8c4f364 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} diff --git a/charts/radar-jdbc-connector-strimzi/templates/tests/test-connection.yaml b/charts/radar-jdbc-connector-strimzi/templates/tests/test-connection.yaml new file mode 100644 index 00000000..f4959a02 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + helm.sh/chart: {{ include "common.names.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/radar-jdbc-connector-strimzi/values.yaml b/charts/radar-jdbc-connector-strimzi/values.yaml new file mode 100644 index 00000000..029811b1 --- /dev/null +++ b/charts/radar-jdbc-connector-strimzi/values.yaml @@ -0,0 +1,277 @@ +# Default values for radar-jdbc-connector. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +image: + # -- Image registry + registry: docker.io + # -- Image repository + repository: radarbase/radar-jdbc-connector + # -- Image tag (immutable tags are recommended) + # Overrides the image tag whose default is the chart appVersion. + tag: + # -- Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + digest: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # e.g: + # pullSecrets: + # - myRegistryKeySecretName + # + pullSecrets: [] + +# -- String to partially override common.names.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override common.names.fullname template with a string +fullnameOverride: "" + +# -- Configure radar-jdbc-connector pods' Security Context +podSecurityContext: {} + # fsGroup: 2000 + +# -- Configure radar-jdbc-connector containers' Security Context +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # -- Kubernetes Service type + type: ClusterIP + # -- radar-jdbc-connector port + port: 8083 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + + # -- CPU/Memory resource requests + requests: + cpu: 100m + memory: 1Gi + +# -- Node labels for pod assignment +nodeSelector: {} + +# -- Toleration labels for pod assignment +tolerations: [] + +# -- Affinity labels for pod assignment +affinity: {} + +secret: + # -- Secret name for the JAAS configuration + jaas: + name: shared-service-user + key: sasl.jaas.config + +# -- Custom livenessProbe that overrides the default one +customLivenessProbe: {} + +livenessProbe: + # -- Enable livenessProbe + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 5 + # -- Success threshold for livenessProbe + successThreshold: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + +# -- Custom readinessProbe that overrides the default one +customReadinessProbe: {} + +readinessProbe: + # -- Enable readinessProbe + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 5 + # -- Success threshold for readinessProbe + successThreshold: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + +# -- Network policy defines who can access this application and who this applications has access to +# @default -- check `values.yaml` +networkpolicy: + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-kafka-bootstrap' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-schema-registry' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + cnpg.io/cluster: '{{ .Values.timescaledb.cluster.fullnameOverride }}' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + + +# -- Maximum number of worker threads inside a connector pod. +maxTasks: 5 + +# -- The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. +kafkaVersion: 3.9.0 + +# -- URI of Kafka brokers of the cluster +kafka: SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094 +# -- URL of the Kafka schema registry +schema_registry: http://radar-kafka-schema-registry:8081 + +# -- Either source or sink +mode: sink + +# -- Java heap options +jvmOptions: + xms: "1500m" + xmx: "2500m" + +source: + # -- Name of the connector Kafka producer group + name: radar-jdbc-source + # -- Database schema (if any) + schema: public + # -- Comma-separated list of tables to read + tableWhitelist: "" + # -- Prefix to prepend to table names to generate the name of the Kafka topic to publish data to. + topicPrefix: "" + # -- How to detect new values in a table. + mode: incrementing + # -- When using mode incrementing, which column to use as incrementing. If empty, autodetection will be used. + incrementingColumnName: "" + # -- Field to use as key for the records. If empty, no key is used. + keyField: "" + persistence: + # -- Whether to enable persistence for storing offsets + enabled: true + # -- Existing persistent volume claim to use + existingClaim: + # -- PVC access mode + accessMode: ReadWriteOnce + # -- PVC storage size request + size: 20Mi + +sink: + # -- Name of the connector Kafka consumer group + name: radar-jdbc-sink + # -- create table if it does not exist + autoCreate: true + # -- How to insert new values into the database + insertMode: upsert + transforms: + # -- Whether to merge the key fields into the inserted values. + enabled: true + # -- Class to handle merging the key into the record + mergeKeyType: org.radarbase.kafka.connect.transforms.MergeKey + # -- Class to convert time-based fields into SQL timestamps + timestampType: org.radarbase.kafka.connect.transforms.TimestampConverter + # -- Field names to consider for converting to timestamp + timestampFields: + - time + - timeReceived + - timeCompleted + - timestamp + + primaryKeys: + # -- where to read the primary keys from when creating the table + mode: record_value + # -- fields to include as primary keys when creating the table + fields: + - time + - userId + - projectId + # -- Comma-separated list of topics the connector will read from and ingest into the database + topics: android_phone_relative_location, android_phone_battery_level, connect_upload_altoida_summary, connect_fitbit_intraday_heart_rate, connect_fitbit_intraday_steps + # -- How to format a table name based on the inserted topic + tableNameFormat: "${topic}" + +jdbc: + # -- Host of the TimescaleDB database + url: + # -- Kubernetes secret name for the JDBC connection URL (disables the use of 'url' value) + urlSecret: + name: + key: + # -- TimescaleDB database username + user: + # -- Kubernetes secret name for the username (disables the use of 'user' value) + userSecret: + name: + key: + # -- TimescaleDB database password (using a secret is recommended) + password: + # -- Kubernetes secret name for the password + passwordSecret: + name: + key: + # -- JDBC connect dialect that the database uses + dialect: TimescaleDBDatabaseDialect + +timescaledb: + # -- Use the local cloudnativepg timescaledb cluster + enabled: true + # -- CloudNativePG TimescaleDB configuration + # @default -- check `values.yaml` + cluster: + fullnameOverride: jdbc-connector-timescaledb + mode: standalone + cluster: + instances: 1 + initdb: + database: database_name + owner: database_name + +# -- Log4j configuration +log4j: + # -- Root log level for the Kafka Connect instance + rootLogLevel: INFO + +# Sentry monitoring configuration +sentry: + # -- DSN (Data Source Name) of the sentry server + dsn: + # -- Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) + level: ERROR diff --git a/charts/radar-jdbc-connector/Chart.yaml b/charts/radar-jdbc-connector/Chart.yaml index d3d770fa..88f5a9b2 100644 --- a/charts/radar-jdbc-connector/Chart.yaml +++ b/charts/radar-jdbc-connector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 10.8.0 name: radar-jdbc-connector description: A Helm chart for RADAR-base JDBC Kafka connector. This is a fork of the Kafka JDBC connector which allows data from topics to be imported into JDBC databases (including TimescaleDB databases which is used in the dashboard pipeline). -version: 0.10.0 +version: 0.10.1 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-jdbc-connector @@ -12,7 +12,7 @@ keywords: - remote-trial annotations: artifacthub.io/license: Apache-2.0 -deprecated: false +deprecated: true type: application home: "https://radar-base.org" maintainers: diff --git a/charts/radar-jdbc-connector/README.md b/charts/radar-jdbc-connector/README.md index 954647cf..500a40c0 100644 --- a/charts/radar-jdbc-connector/README.md +++ b/charts/radar-jdbc-connector/README.md @@ -2,8 +2,9 @@ # radar-jdbc-connector [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-jdbc-connector)](https://artifacthub.io/packages/helm/radar-base/radar-jdbc-connector) +> **:exclamation: This Helm Chart is deprecated!** -![Version: 0.10.0](https://img.shields.io/badge/Version-0.10.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 10.8.0](https://img.shields.io/badge/AppVersion-10.8.0-informational?style=flat-square) +![Version: 0.10.1](https://img.shields.io/badge/Version-0.10.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 10.8.0](https://img.shields.io/badge/AppVersion-10.8.0-informational?style=flat-square) A Helm chart for RADAR-base JDBC Kafka connector. This is a fork of the Kafka JDBC connector which allows data from topics to be imported into JDBC databases (including TimescaleDB databases which is used in the dashboard pipeline). diff --git a/charts/radar-kafka/Chart.yaml b/charts/radar-kafka/Chart.yaml index b589c7e4..61f97852 100644 --- a/charts/radar-kafka/Chart.yaml +++ b/charts/radar-kafka/Chart.yaml @@ -4,7 +4,7 @@ apiVersion: v2 appVersion: 3.9.0 description: "Apache Kafka for RADAR-base using the Strimzi Operator" name: radar-kafka -version: 0.0.2 +version: 0.0.3 keywords: - kafka - queue diff --git a/charts/radar-kafka/README.md b/charts/radar-kafka/README.md index 54245b44..6616dd37 100644 --- a/charts/radar-kafka/README.md +++ b/charts/radar-kafka/README.md @@ -3,7 +3,7 @@ # radar-kafka [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-kafka)](https://artifacthub.io/packages/helm/radar-base/radar-kafka) -![Version: 0.0.2](https://img.shields.io/badge/Version-0.0.2-informational?style=flat-square) ![AppVersion: 3.9.0](https://img.shields.io/badge/AppVersion-3.9.0-informational?style=flat-square) +![Version: 0.0.3](https://img.shields.io/badge/Version-0.0.3-informational?style=flat-square) ![AppVersion: 3.9.0](https://img.shields.io/badge/AppVersion-3.9.0-informational?style=flat-square) Apache Kafka for RADAR-base using the Strimzi Operator diff --git a/charts/radar-kafka/templates/topics-schema-registry.yaml b/charts/radar-kafka/templates/topics-schema-registry.yaml index 15fd1197..ef823bb0 100644 --- a/charts/radar-kafka/templates/topics-schema-registry.yaml +++ b/charts/radar-kafka/templates/topics-schema-registry.yaml @@ -9,7 +9,8 @@ metadata: labels: strimzi.io/cluster: {{ template "common.names.fullname" . }} spec: - partitions: {{ $context.partitions }} + # IMPORTANT: do not change, According to https://stackoverflow.com/a/74777178 schema topic should not have more than 1 partition + partitions: 1 replicas: {{ $topicReplicationFactor }} config: {{- toYaml $context.config | nindent 4 }} diff --git a/charts/radar-kafka/values.yaml b/charts/radar-kafka/values.yaml index 8855b234..718a4f8d 100644 --- a/charts/radar-kafka/values.yaml +++ b/charts/radar-kafka/values.yaml @@ -91,7 +91,6 @@ schema-registry: - '{{ $.Values.server_name }}' topic: - partitions: 1 replicationFactor: 3 config: cleanup.policy: compact diff --git a/charts/radar-oura-connector-strimzi/.helmignore b/charts/radar-oura-connector-strimzi/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/radar-oura-connector-strimzi/Chart.lock b/charts/radar-oura-connector-strimzi/Chart.lock new file mode 100644 index 00000000..5d06dada --- /dev/null +++ b/charts/radar-oura-connector-strimzi/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + version: 2.27.0 +digest: sha256:397fd12e07f1aa1a5280579406a75b169f8262ac4214584ea366460277288546 +generated: "2024-12-27T17:15:02.992863091+01:00" diff --git a/charts/radar-oura-connector-strimzi/Chart.yaml b/charts/radar-oura-connector-strimzi/Chart.yaml new file mode 100644 index 00000000..7d0d01fb --- /dev/null +++ b/charts/radar-oura-connector-strimzi/Chart.yaml @@ -0,0 +1,33 @@ +apiVersion: v2 +appVersion: 0.6.2 +description: A Helm chart for RADAR-base oura connector using the Strimzi Operator. This application collects data from participants via the Oura Web API. +name: radar-oura-connector-strimzi +version: 0.0.1 +icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" +sources: +- https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-oura-connector-strimzi +- https://github.com/RADAR-base/RADAR-REST-Connector +keywords: + - radar-base + - remote-trial +annotations: + artifacthub.io/license: Apache-2.0 +deprecated: false +type: application +home: "https://radar-base.org" +maintainers: + - email: pauline.conde@kcl.ac.uk + name: Pauline Conde + url: https://www.kcl.ac.uk/people/pauline-conde + - email: yatharth.ranjan@kcl.ac.uk + name: Yatharth Ranjan + url: https://www.kcl.ac.uk/people/yatharth-ranjan + - email: pim@thehyve.nl + name: Pim van Nierop + url: https://www.thehyve.nl/experts/pim-van-nierop +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + tags: + - bitnami-common + version: 2.x.x diff --git a/charts/radar-oura-connector-strimzi/DOCS.md.gotmpl b/charts/radar-oura-connector-strimzi/DOCS.md.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-oura-connector-strimzi/README.md b/charts/radar-oura-connector-strimzi/README.md new file mode 100644 index 00000000..ac22eecb --- /dev/null +++ b/charts/radar-oura-connector-strimzi/README.md @@ -0,0 +1,102 @@ + + +# radar-oura-connector-strimzi +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-oura-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-oura-connector-strimzi) + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) + +A Helm chart for RADAR-base oura connector using the Strimzi Operator. This application collects data from participants via the Oura Web API. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Pauline Conde | | | +| Yatharth Ranjan | | | +| Pim van Nierop | | | + +## Source Code + +* +* + +## Prerequisites +* Kubernetes 1.28+ +* Kubectl 1.28+ +* Helm 3.1.0+ +* PV provisioner support in the underlying infrastructure + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://radar-base.github.io/radar-helm-charts | common | 2.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | Number of radar-oura-connector-strimzi replicas to deploy | +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"radarbase/kafka-connect-rest-oura-source"` | Image repository | +| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. | +| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets: - myRegistryKeySecretName | +| nameOverride | string | `""` | String to partially override radar-oura-connector-strimzi.fullname template with a string (will prepend the release name) | +| fullnameOverride | string | `""` | String to fully override radar-oura-connector-strimzi.fullname template with a string | +| podSecurityContext | object | `{}` | Configure radar-oura-connector-strimzi pods' Security Context | +| securityContext | object | `{}` | Configure radar-oura-connector-strimzi containers' Security Context | +| service.type | string | `"ClusterIP"` | Kubernetes Service type | +| service.port | int | `8083` | radar-oura-connector-strimzi port | +| resources.requests | object | `{"cpu":"100m","memory":"1Gi"}` | CPU/Memory resource requests | +| persistence.enabled | bool | `true` | Enable persistence using PVC | +| persistence.accessMode | string | `"ReadWriteOnce"` | PVC Access Mode for radar-oura-connector-strimzi volume | +| persistence.size | string | `"5Gi"` | PVC Storage Request for radar-oura-connector-strimzi volume | +| persistence.fsUserOverride | string | `nil` | Overrides the user of the oura connector logs, for example, `"1000:1000"`. | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| tolerations | list | `[]` | Toleration labels for pod assignment | +| affinity | object | `{}` | Affinity labels for pod assignment | +| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret for the Kafka SASL JAAS configuration | +| extraEnvVars | list | `[]` | Extra environment variables | +| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one | +| livenessProbe.enabled | bool | `true` | Enable livenessProbe | +| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe | +| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe | +| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe | +| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe | +| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe | +| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one | +| readinessProbe.enabled | bool | `true` | Enable readinessProbe | +| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe | +| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe | +| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe | +| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe | +| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe | +| customStartupProbe | object | `{}` | Custom startupProbe that overrides the default one | +| startupProbe.enabled | bool | `true` | Enable startupProbe | +| startupProbe.initialDelaySeconds | int | `5` | Initial delay seconds for startupProbe | +| startupProbe.periodSeconds | int | `10` | Period seconds for startupProbe | +| startupProbe.timeoutSeconds | int | `10` | Timeout seconds for startupProbe | +| startupProbe.successThreshold | int | `1` | Success threshold for startupProbe | +| startupProbe.failureThreshold | int | `30` | Failure threshold for startupProbe | +| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to | +| maxTasks | int | `5` | Maximum number of worker threads inside a connector pod. | +| kafkaVersion | string | `"3.9.0"` | The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. | +| kafka | string | `"SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094"` | URI of Kafka brokers of the cluster | +| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry | +| radar_rest_sources_backend_url | string | `"http://radar-rest-sources-backend:8080/rest-sources/backend/"` | Base URL of the rest-sources-authorizer-backend service | +| oura_api_client | string | `""` | Oura API client id. | +| oura_api_secret | string | `""` | Oura API client secret. | +| oauthClientId | string | `"radar_oura_connector"` | OAuth2 client id from Management Portal | +| oauthClientSecret | string | `"secret"` | OAuth2 client secret from Management Portal | +| auth_url | string | `"http://management-portal:8080/managementportal/oauth/token"` | OAuth2 Auth URL for connector client to get access tokens | +| managementportal_url | string | `"http://management-portal:8080/managementportal"` | URL of Management Portal. This will be used to create URLs to access Management Portal | +| includeIntradayData | bool | `true` | Set to true, if intraday access data should be collected by the connector. This will be set in connector.properties. | +| user_repository_class | string | `"OuraServiceUserRepositoryLegacy"` | Class name of the user repository. This should be the same as the one used in the connector. | +| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration | +| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance | +| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server | +| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) | diff --git a/charts/radar-oura-connector-strimzi/README.md.gotmpl b/charts/radar-oura-connector-strimzi/README.md.gotmpl new file mode 100644 index 00000000..929d551e --- /dev/null +++ b/charts/radar-oura-connector-strimzi/README.md.gotmpl @@ -0,0 +1,18 @@ +{{ template "common.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "common.prerequisiteswithpv" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/radar-oura-connector-strimzi/charts/common-2.27.0.tgz b/charts/radar-oura-connector-strimzi/charts/common-2.27.0.tgz new file mode 100644 index 00000000..5c539c35 Binary files /dev/null and b/charts/radar-oura-connector-strimzi/charts/common-2.27.0.tgz differ diff --git a/charts/radar-oura-connector-strimzi/templates/NOTES.txt b/charts/radar-oura-connector-strimzi/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-oura-connector-strimzi/templates/_helpers.tpl b/charts/radar-oura-connector-strimzi/templates/_helpers.tpl new file mode 100644 index 00000000..2fb21811 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Return the proper image name +*/}} +{{- define "radar-oura-connector.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} diff --git a/charts/radar-oura-connector-strimzi/templates/configmap-log4j.yaml b/charts/radar-oura-connector-strimzi/templates/configmap-log4j.yaml new file mode 100644 index 00000000..2a0aa4b3 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/configmap-log4j.yaml @@ -0,0 +1,29 @@ +# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification. +# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }}-log4j + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + # Root logger configuration + # The sentryAppender is added to the rootLogger if SENTRY_DSN is set + log4j.rootLogger={{ .Values.log4j.rootLogLevel | default "INFO" }}, stdout{{- if .Values.sentry.dsn }}, sentryAppender{{- end }} + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Appender for Sentry monitoring + {{- if .Values.sentry.dsn }} + log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender + log4j.appender.sentryAppender.threshold={{ .Values.sentry.level | default "INFO" }} + {{- end }} + + # Default log levels + log4j.logger.org.reflections=ERROR + log4j.logger.org.I0Itec.zkclient=ERROR diff --git a/charts/radar-oura-connector-strimzi/templates/kafka-connect.yaml b/charts/radar-oura-connector-strimzi/templates/kafka-connect.yaml new file mode 100644 index 00000000..7f10e36a --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/kafka-connect.yaml @@ -0,0 +1,131 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + version: {{ .Values.kafkaVersion }} + replicas: {{ .Values.replicaCount }} + bootstrapServers: {{ .Values.kafka }} + image: {{ template "radar-oura-connector.image" . }} + {{- if .Values.secret.jaas.name }} + # Authentication-related props are handled by the template builder in https://github.com/strimzi/strimzi-kafka-operator/blob/main/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaConnectConfigurationBuilder.java + authentication: + type: scram-sha-512 + username: shared-service-user + passwordSecret: + secretName: {{ .Values.secret.jaas.name }} + password: password # The name of the key in the secret that contains the password + {{- end }} + autoRestart: + enabled: true # automatic restarts of failed connectors or their tasks + config: # Properties that configure the Kafka Connect cluster itself + group.id: {{ include "common.names.fullname" . }} + offset.storage.topic: {{ template "common.names.fullname" . }}.offset + config.storage.topic: {{ template "common.names.fullname" . }}.config + status.storage.topic: {{ template "common.names.fullname" . }}.status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + key.converter: "io.confluent.connect.avro.AvroConverter" + value.converter: "io.confluent.connect.avro.AvroConverter" + key.converter.schema.registry.url: {{ .Values.schema_registry }} + value.converter.schema.registry.url: {{ .Values.schema_registry }} + internal.key.converter: "org.apache.kafka.connect.json.JsonConverter" + internal.value.converter: "org.apache.kafka.connect.json.JsonConverter" + template: + pod: + metadata: + annotations: + checksum/configmap-log4j: {{ include (print $.Template.BasePath "/configmap-log4j.yaml") . | sha256sum }} + backup.velero.io/backup-volumes: logs + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + affinity: + {{- if .Values.affinity }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - {{ template "common.names.name" . }} + - key: "app.kubernetes.io/instance" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + connectContainer: + env: + - name: CONNECT_REST_ADVERTISED_HOST_NAME # Possibly replaced by ADVERTISED_HOSTNAME in kafka_connect_run.sh + value: {{ include "common.names.fullname" . }} + - name: KAFKA_LOG4J_OPTS + value: "-Dlog4j.configuration=file:/opt/kafka/custom-config/log4j.properties" + {{- if .Values.sentry.dsn }} + - name: SENTRY_DSN + value: {{ .Values.sentry.dsn | quote }} + - name: SENTRY_LOG_LEVEL + value: {{ .Values.sentry.level | quote }} + {{- end }} + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 8 }} + {{- end }} + logging: + type: external + valueFrom: + configMapKeyRef: + name: {{ template "common.names.fullname" . }}-log4j + key: log4j.properties + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- .Values.customLivenessProbe | toYaml | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- .Values.customReadinessProbe | toYaml | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 4 }} diff --git a/charts/radar-oura-connector-strimzi/templates/kafka-connector.yaml b/charts/radar-oura-connector-strimzi/templates/kafka-connector.yaml new file mode 100644 index 00000000..6bf4eca3 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/kafka-connector.yaml @@ -0,0 +1,23 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: radar-oura-source + namespace: {{ include "common.names.namespace" . | quote }} + labels: + strimzi.io/cluster: {{ template "common.names.fullname" . }} +spec: + class: org.radarbase.connect.rest.oura.OuraSourceConnector + tasksMax: {{ .Values.maxTasks }} + config: # Properties that configure Oura source connector + name: radar-oura-source + connector.class: org.radarbase.connect.rest.oura.OuraSourceConnector + rest.source.base.url: https://api.ouraring.com + rest.source.poll.interval.ms: 5000 + oura.api.client: {{ .Values.oura_api_client }} + oura.api.secret: {{ .Values.oura_api_secret }} + oura.api.intraday: {{ .Values.includeIntradayData }} + oura.user.repository.class: org.radarbase.connect.rest.oura.user.{{ .Values.user_repository_class }} + oura.user.repository.url: {{ .Values.radar_rest_sources_backend_url }} + oura.user.repository.client.id: {{ .Values.oauthClientId }} + oura.user.repository.client.secret: {{ .Values.oauthClientSecret }} + oura.user.repository.oauth2.token.url: {{ .Values.auth_url }} diff --git a/charts/radar-oura-connector-strimzi/templates/networkpolicy.yaml b/charts/radar-oura-connector-strimzi/templates/networkpolicy.yaml new file mode 100644 index 00000000..7c4edf91 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkpolicy }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: +{{ include "common.labels.standard" . | indent 6 }} + {{- tpl (toYaml .Values.networkpolicy) . | nindent 2 }} +{{- end -}} diff --git a/charts/radar-oura-connector-strimzi/templates/service.yaml b/charts/radar-oura-connector-strimzi/templates/service.yaml new file mode 100644 index 00000000..d8c4f364 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} diff --git a/charts/radar-oura-connector-strimzi/templates/tests/test-connection.yaml b/charts/radar-oura-connector-strimzi/templates/tests/test-connection.yaml new file mode 100644 index 00000000..f4959a02 --- /dev/null +++ b/charts/radar-oura-connector-strimzi/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + helm.sh/chart: {{ include "common.names.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/radar-oura-connector-strimzi/values.yaml b/charts/radar-oura-connector-strimzi/values.yaml new file mode 100644 index 00000000..ede709db --- /dev/null +++ b/charts/radar-oura-connector-strimzi/values.yaml @@ -0,0 +1,250 @@ +# Default values for radar-oura-connector-strimzi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of radar-oura-connector-strimzi replicas to deploy +replicaCount: 1 + +image: + # -- Image registry + registry: docker.io + # -- Image repository + repository: radarbase/kafka-connect-rest-oura-source + # -- Image tag (immutable tags are recommended) + # Overrides the image tag whose default is the chart appVersion. + tag: + # -- Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + digest: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # e.g: + # pullSecrets: + # - myRegistryKeySecretName + # + pullSecrets: [] + +# -- String to partially override radar-oura-connector-strimzi.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override radar-oura-connector-strimzi.fullname template with a string +fullnameOverride: "" + +# -- Configure radar-oura-connector-strimzi pods' Security Context +podSecurityContext: {} + # fsGroup: 2000 + +# -- Configure radar-oura-connector-strimzi containers' Security Context +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # -- Kubernetes Service type + type: ClusterIP + # -- radar-oura-connector-strimzi port + port: 8083 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + + # -- CPU/Memory resource requests + requests: + cpu: 100m + memory: 1Gi + +persistence: + # -- Enable persistence using PVC + enabled: true + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + # storageClass: "-" + ## + ## If you want to reuse an existing claim, you can pass the name of the PVC using + ## the existingClaim variable + # existingClaim: your-claim + + # -- PVC Access Mode for radar-oura-connector-strimzi volume + accessMode: ReadWriteOnce + # -- PVC Storage Request for radar-oura-connector-strimzi volume + size: 5Gi + # -- Overrides the user of the oura connector logs, for example, `"1000:1000"`. + fsUserOverride: null + +# -- Node labels for pod assignment +nodeSelector: {} + +# -- Toleration labels for pod assignment +tolerations: [] + +# -- Affinity labels for pod assignment +affinity: {} + +secret: + # -- Secret for the Kafka SASL JAAS configuration + jaas: + name: shared-service-user + key: sasl.jaas.config + +# -- Extra environment variables +extraEnvVars: [] + +# -- Custom livenessProbe that overrides the default one +customLivenessProbe: {} + +livenessProbe: + # -- Enable livenessProbe + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 5 + # -- Success threshold for livenessProbe + successThreshold: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + +# -- Custom readinessProbe that overrides the default one +customReadinessProbe: {} + +readinessProbe: + # -- Enable readinessProbe + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 5 + # -- Success threshold for readinessProbe + successThreshold: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + +# -- Custom startupProbe that overrides the default one +customStartupProbe: {} + +startupProbe: + # -- Enable startupProbe + enabled: true + # -- Initial delay seconds for startupProbe + initialDelaySeconds: 5 + # -- Period seconds for startupProbe + periodSeconds: 10 + # -- Timeout seconds for startupProbe + timeoutSeconds: 10 + # -- Success threshold for startupProbe + successThreshold: 1 + # -- Failure threshold for startupProbe + failureThreshold: 30 + +# -- Network policy defines who can access this application and who this applications has access to +# @default -- check `values.yaml` +networkpolicy: + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 192.168.0.0/16 + - 172.16.0.0/12 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-kafka-bootstrap' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-schema-registry' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-rest-sources-backend' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'management-portal' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + +# -- Maximum number of worker threads inside a connector pod. +maxTasks: 5 + +# -- The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. +kafkaVersion: 3.9.0 + +# -- URI of Kafka brokers of the cluster +kafka: SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094 +# -- URL of the Kafka schema registry +schema_registry: http://radar-kafka-schema-registry:8081 +# -- Base URL of the rest-sources-authorizer-backend service +radar_rest_sources_backend_url: http://radar-rest-sources-backend:8080/rest-sources/backend/ + +# -- Oura API client id. +oura_api_client: "" +# -- Oura API client secret. +oura_api_secret: "" + +# -- OAuth2 client id from Management Portal +oauthClientId: radar_oura_connector +# -- OAuth2 client secret from Management Portal +oauthClientSecret: secret +# -- OAuth2 Auth URL for connector client to get access tokens +auth_url: http://management-portal:8080/managementportal/oauth/token +# -- URL of Management Portal. This will be used to create URLs to access Management Portal +managementportal_url: http://management-portal:8080/managementportal +# -- Set to true, if intraday access data should be collected by the connector. This will be set in connector.properties. +includeIntradayData: true + +# -- Class name of the user repository. This should be the same as the one used in the connector. +user_repository_class: OuraServiceUserRepositoryLegacy + +# -- Log4j configuration +log4j: + # -- Root log level for the Kafka Connect instance + rootLogLevel: INFO + +# Sentry monitoring configuration +sentry: + # -- DSN (Data Source Name) of the sentry server + dsn: + # -- Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) + level: ERROR diff --git a/charts/radar-oura-connector/Chart.yaml b/charts/radar-oura-connector/Chart.yaml index e14944a0..2fd9e546 100644 --- a/charts/radar-oura-connector/Chart.yaml +++ b/charts/radar-oura-connector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.6.2 description: A Helm chart for RADAR-base oura connector. This application collects data from participants via the Oura Web API. name: radar-oura-connector -version: 0.4.1 +version: 0.4.2 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-oura-connector @@ -12,7 +12,7 @@ keywords: - remote-trial annotations: artifacthub.io/license: Apache-2.0 -deprecated: false +deprecated: true type: application home: "https://radar-base.org" maintainers: diff --git a/charts/radar-oura-connector/README.md b/charts/radar-oura-connector/README.md index a82de48b..8beb8bd8 100644 --- a/charts/radar-oura-connector/README.md +++ b/charts/radar-oura-connector/README.md @@ -2,8 +2,9 @@ # radar-oura-connector [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-oura-connector)](https://artifacthub.io/packages/helm/radar-base/radar-oura-connector) +> **:exclamation: This Helm Chart is deprecated!** -![Version: 0.4.1](https://img.shields.io/badge/Version-0.4.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) +![Version: 0.4.2](https://img.shields.io/badge/Version-0.4.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) A Helm chart for RADAR-base oura connector. This application collects data from participants via the Oura Web API. diff --git a/charts/radar-s3-connector-strimzi/.helmignore b/charts/radar-s3-connector-strimzi/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/radar-s3-connector-strimzi/Chart.lock b/charts/radar-s3-connector-strimzi/Chart.lock new file mode 100644 index 00000000..03e44cd7 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + version: 2.27.0 +digest: sha256:397fd12e07f1aa1a5280579406a75b169f8262ac4214584ea366460277288546 +generated: "2024-12-27T18:49:31.278906468+01:00" diff --git a/charts/radar-s3-connector-strimzi/Chart.yaml b/charts/radar-s3-connector-strimzi/Chart.yaml new file mode 100644 index 00000000..256000a2 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +appVersion: 3.9.0 +description: A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options +name: radar-s3-connector-strimzi +version: 0.7.1 +icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" +sources: +- https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-s3-connector +- https://github.com/RADAR-base/kafka-connect-transform-keyvalue +- https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options +keywords: + - radar-base + - remote-trial +annotations: + artifacthub.io/license: Apache-2.0 +type: application +home: "https://radar-base.org" +maintainers: + - email: pim@thehyve.nl + name: Pim van Nierop + url: https://www.thehyve.nl/experts/pim-van-nierop +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + tags: + - bitnami-common + version: 2.x.x diff --git a/charts/radar-s3-connector-strimzi/DOCS.md.gotmpl b/charts/radar-s3-connector-strimzi/DOCS.md.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-s3-connector-strimzi/README.md b/charts/radar-s3-connector-strimzi/README.md new file mode 100644 index 00000000..0c263db7 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/README.md @@ -0,0 +1,109 @@ + + +# radar-s3-connector-strimzi +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-s3-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-s3-connector-strimzi) + +![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.9.0](https://img.shields.io/badge/AppVersion-3.9.0-informational?style=flat-square) + +A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Pim van Nierop | | | + +## Source Code + +* +* +* + +## Prerequisites +* Kubernetes 1.28+ +* Kubectl 1.28+ +* Helm 3.1.0+ + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://radar-base.github.io/radar-helm-charts | common | 2.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | Number of radar-s3-connector replicas to deploy | +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"radarbase/kafka-connect-transform-s3"` | Image repository | +| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. | +| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets: - myRegistryKeySecretName | +| nameOverride | string | `""` | String to partially override common.names.fullname template with a string (will prepend the release name) | +| fullnameOverride | string | `""` | String to fully override common.names.fullname template with a string | +| podSecurityContext | object | `{}` | Configure radar-s3-connector pods' Security Context | +| securityContext | object | `{}` | Configure radar-s3-connector containers' Security Context | +| service.type | string | `"ClusterIP"` | Kubernetes Service type | +| service.port | int | `8083` | radar-s3-connector port | +| resources.requests | object | `{"cpu":"100m","memory":"3Gi"}` | CPU/Memory resource requests | +| jvmOptions.xmx | string | `"4g"` | | +| jvmOptions.xms | string | `"3g"` | | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| tolerations | list | `[]` | Toleration labels for pod assignment | +| affinity | object | `{}` | Affinity labels for pod assignment | +| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret for the Kafka SASL JAAS configuration | +| extraEnvVars | list | `[]` | Extra environment variables | +| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one | +| livenessProbe.enabled | bool | `true` | Enable livenessProbe | +| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe | +| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe | +| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe | +| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe | +| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe | +| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one | +| readinessProbe.enabled | bool | `true` | Enable readinessProbe | +| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe | +| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe | +| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe | +| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe | +| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe | +| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to | +| maxTasks | int | `5` | Maximum number of worker threads inside a connector pod. | +| kafkaVersion | string | `"3.9.0"` | The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. | +| kafka | string | `"SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094"` | URI of Kafka brokers of the cluster | +| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry | +| radar_rest_sources_backend_url | string | `"http://radar-rest-sources-backend:8080/rest-sources/backend/"` | Base URL of the rest-sources-authorizer-backend service | +| catalogServer.url | string | `"http://catalog-server:9010"` | Catalog server URL | +| topics | string | `"questionnaire_response"` | List of topics to be consumed by the sink connector separated by comma. Topics defined in the catalog server will automatically be loaded if `syncTopics.enabled` is true. May not be empty or the Strimzi operator will not start the connector. | +| s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint url | +| s3Tagging | bool | `false` | set to true, if S3 objects should be tagged with start and end offsets, as well as record count. | +| s3PartSize | int | `5242880` | The Part Size in S3 Multi-part Uploads. | +| s3Region | string | `nil` | The AWS region to be used the connector. Some compatibility layers require this. | +| flushSize | int | `10000` | Number of records written to store before invoking file commits. | +| rotateInterval | int | `900000` | The time interval in milliseconds to invoke file commits. | +| bucketAccessKey | string | `"access_key"` | Access key of the target S3 bucket | +| bucketSecretKey | string | `"secret"` | Secret key of the target S3 bucket | +| bucketName | string | `"radar_intermediate_storage"` | Bucket name of the target S3 bucket | +| cc.enabled | bool | `false` | Set to true, if Confluent Cloud is used | +| cc.bootstrapServerurl | string | `""` | Confluent cloud based Kafka broker URL (if Confluent Cloud based Kafka cluster is used) | +| cc.schemaRegistryUrl | string | `""` | Confluent cloud based Schema registry URL (if Confluent Cloud based Schema registry is used) | +| cc.apiKey | string | `"ccApikey"` | API Key of the Confluent Cloud cluster | +| cc.apiSecret | string | `"ccApiSecret"` | API secret of the Confluent Cloud cluster | +| cc.schemaRegistryApiKey | string | `"srApiKey"` | API Key of the Confluent Cloud Schema registry | +| cc.schemaRegistryApiSecret | string | `"srApiSecret"` | API Key of the Confluent Cloud Schema registry | +| syncTopics.enabled | bool | `true` | If true, fetch list of topics from catalog server | +| syncTopics.image.repository | string | `"badouralix/curl-jq"` | Image repository to fetch topics with | +| syncTopics.image.tag | string | `"latest"` | Image tag to fetch topics with | +| syncTopics.image.pullPolicy | string | `"IfNotPresent"` | Image pull policy to fetch topics with | +| syncTopics.job.backoffLimit | int | `100000` | How often to retry the job in case of failure during topics init. | +| syncTopics.cronjob.schedule | string | `"*/10 * * * *"` | Cron expression for the topics sync cronjob | +| syncTopics.cronjob.successfulJobsHistoryLimit | int | `1` | Number of successful jobs to keep in history | +| syncTopics.cronjob.failedJobsHistoryLimit | int | `1` | Number of failed jobs to keep in history | +| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration | +| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance | +| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server | +| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) | diff --git a/charts/radar-s3-connector-strimzi/README.md.gotmpl b/charts/radar-s3-connector-strimzi/README.md.gotmpl new file mode 100644 index 00000000..fab382f7 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/README.md.gotmpl @@ -0,0 +1,18 @@ +{{ template "common.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "common.prerequisites" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/radar-s3-connector-strimzi/charts/common-2.27.0.tgz b/charts/radar-s3-connector-strimzi/charts/common-2.27.0.tgz new file mode 100644 index 00000000..5c539c35 Binary files /dev/null and b/charts/radar-s3-connector-strimzi/charts/common-2.27.0.tgz differ diff --git a/charts/radar-s3-connector-strimzi/templates/NOTES.txt b/charts/radar-s3-connector-strimzi/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-s3-connector-strimzi/templates/_helpers.tpl b/charts/radar-s3-connector-strimzi/templates/_helpers.tpl new file mode 100644 index 00000000..39b263cc --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Return the proper image name +*/}} +{{- define "radar-s3-connector.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} diff --git a/charts/radar-s3-connector-strimzi/templates/configmap-log4j.yaml b/charts/radar-s3-connector-strimzi/templates/configmap-log4j.yaml new file mode 100644 index 00000000..2a0aa4b3 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/configmap-log4j.yaml @@ -0,0 +1,29 @@ +# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification. +# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }}-log4j + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + # Root logger configuration + # The sentryAppender is added to the rootLogger if SENTRY_DSN is set + log4j.rootLogger={{ .Values.log4j.rootLogLevel | default "INFO" }}, stdout{{- if .Values.sentry.dsn }}, sentryAppender{{- end }} + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Appender for Sentry monitoring + {{- if .Values.sentry.dsn }} + log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender + log4j.appender.sentryAppender.threshold={{ .Values.sentry.level | default "INFO" }} + {{- end }} + + # Default log levels + log4j.logger.org.reflections=ERROR + log4j.logger.org.I0Itec.zkclient=ERROR diff --git a/charts/radar-s3-connector-strimzi/templates/kafka-connect.yaml b/charts/radar-s3-connector-strimzi/templates/kafka-connect.yaml new file mode 100644 index 00000000..36ee8b94 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/kafka-connect.yaml @@ -0,0 +1,159 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + version: {{ .Values.kafkaVersion }} + replicas: {{ .Values.replicaCount }} + bootstrapServers: {{ .Values.kafka }} + image: {{ template "radar-s3-connector.image" . }} + {{- if .Values.secret.jaas.name }} + # Authentication-related props are handled by the template builder in https://github.com/strimzi/strimzi-kafka-operator/blob/main/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaConnectConfigurationBuilder.java + authentication: + type: scram-sha-512 + username: shared-service-user + passwordSecret: + secretName: {{ .Values.secret.jaas.name }} + password: password # The name of the key in the secret that contains the password + {{- else if .Values.cc.enabled }} + authentication: + type: scram-sha-512 + username: {{ .Values.cc.apiKey | quote }} + passwordSecret: + secretName: {{ template "common.names.fullname" . }} + password: ccJaasPassword + {{- end }} + autoRestart: + enabled: true # automatic restarts of failed connectors or their tasks + jvmOptions: + "-Xmx": {{ .Values.jvmOptions.xmx | quote }} + "-Xms": {{ .Values.jvmOptions.xms | quote }} + config: # Properties that configure the Kafka Connect cluster itself + group.id: {{ include "common.names.fullname" . }} + offset.storage.topic: {{ template "common.names.fullname" . }}.offset + config.storage.topic: {{ template "common.names.fullname" . }}.config + status.storage.topic: {{ template "common.names.fullname" . }}.status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + key.converter: "io.confluent.connect.avro.AvroConverter" + value.converter: "io.confluent.connect.avro.AvroConverter" + {{- if .Values.cc.enabled }} + key.converter.schema.registry.url: {{ .Values.cc.schemaRegistryUrl }} + key.converter.basic.auth.credentials.source: USER_INFO + key.converter.basic.auth.user.info: {{ .Values.cc.schemaRegistryApiKey }}:{{ .Values.cc.schemaRegistryApiSecret }} + value.converter.schema.registry.url: {{ .Values.cc.schemaRegistryUrl }} + value.converter.basic.auth.credentials.source: USER_INFO + value.converter.basic.auth.user.info: {{ .Values.cc.schemaRegistryApiKey }}:{{ .Values.cc.schemaRegistryApiSecret }} + {{- else }} + key.converter.schema.registry.url: {{ .Values.schema_registry }} + value.converter.schema.registry.url: {{ .Values.schema_registry }} + {{- end }} + template: + pod: + metadata: + annotations: + checksum/configmap-log4j: {{ include (print $.Template.BasePath "/configmap-log4j.yaml") . | sha256sum }} + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + affinity: + {{- if .Values.affinity }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - {{ template "common.names.name" . }} + - key: "app.kubernetes.io/instance" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + connectContainer: + env: + {{- if and .Values.bucketAccessKey .Values.bucketSecretKey }} + - name: AWS_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ template "common.names.fullname" . }} + key: awsAccessKey + - name: AWS_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ template "common.names.fullname" . }} + key: awsSecretKey + {{- end }} + - name: CONNECT_REST_ADVERTISED_HOST_NAME # Possibly replaced by ADVERTISED_HOSTNAME in kafka_connect_run.sh + value: {{ include "common.names.fullname" . }} + - name: CONNECT_LOG4J_ROOT_LOGLEVEL + value: {{ .Values.log4j.rootLogLevel | quote }} + {{- if .Values.sentry.dsn }} + - name: SENTRY_DSN + value: {{ .Values.sentry.dsn | quote }} + - name: SENTRY_LOG_LEVEL + value: {{ .Values.sentry.level | quote }} + {{- end }} + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 8 }} + {{- end }} + logging: + type: external + valueFrom: + configMapKeyRef: + name: {{ template "common.names.fullname" . }}-log4j + key: log4j.properties + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- .Values.customLivenessProbe | toYaml | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- .Values.customReadinessProbe | toYaml | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 4 }} diff --git a/charts/radar-s3-connector-strimzi/templates/kafka-connector.yaml b/charts/radar-s3-connector-strimzi/templates/kafka-connector.yaml new file mode 100644 index 00000000..0bf44de5 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/kafka-connector.yaml @@ -0,0 +1,29 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: {{ include "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: + strimzi.io/cluster: {{ template "common.names.fullname" . }} +spec: + class: io.confluent.connect.s3.S3SinkConnector + tasksMax: {{ .Values.maxTasks }} + config: + topics: {{ .Values.topics }} + s3.bucket.name: {{ .Values.bucketName }} + {{- if .Values.s3Region }} + s3.region: {{ .Values.s3Region }} + {{- end }} + s3.part.size: {{ .Values.s3PartSize | int }} + s3.object.tagging: {{ .Values.s3Tagging }} + {{- if .Values.s3Endpoint }} + store.url: {{ .Values.s3Endpoint }} + {{- end }} + storage.class: io.confluent.connect.s3.storage.S3Storage + format.class: io.confluent.connect.s3.format.avro.AvroFormat + transforms: combineKeyValue + transforms.combineKeyValue.type: org.radarbase.kafka.connect.transforms.CombineKeyValue + rotate.schedule.interval.ms: {{ .Values.rotateInterval | int }} + timezone: UTC + flush.size: {{ .Values.flushSize | int }} + connect.meta.data: false diff --git a/charts/radar-s3-connector-strimzi/templates/networkpolicy.yaml b/charts/radar-s3-connector-strimzi/templates/networkpolicy.yaml new file mode 100644 index 00000000..7c4edf91 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkpolicy }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: +{{ include "common.labels.standard" . | indent 6 }} + {{- tpl (toYaml .Values.networkpolicy) . | nindent 2 }} +{{- end -}} diff --git a/charts/radar-s3-connector-strimzi/templates/secrets.yaml b/charts/radar-s3-connector-strimzi/templates/secrets.yaml new file mode 100644 index 00000000..9db2bf40 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/secrets.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +type: Opaque +data: + {{ if .Values.cc.enabled -}} + ccJaasPassword: {{ .Values.cc.apiSecret | b64enc | quote }} + {{- end }} + {{- if and .Values.bucketAccessKey .Values.bucketSecretKey }} + awsAccessKey: {{ .Values.bucketAccessKey | b64enc | quote }} + awsSecretKey: {{ .Values.bucketSecretKey | b64enc | quote }} + {{- end }} diff --git a/charts/radar-s3-connector-strimzi/templates/service.yaml b/charts/radar-s3-connector-strimzi/templates/service.yaml new file mode 100644 index 00000000..d8c4f364 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} diff --git a/charts/radar-s3-connector-strimzi/templates/tests/test-connection.yaml b/charts/radar-s3-connector-strimzi/templates/tests/test-connection.yaml new file mode 100644 index 00000000..f4959a02 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + helm.sh/chart: {{ include "common.names.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/radar-s3-connector-strimzi/templates/topics-sync-job.yaml b/charts/radar-s3-connector-strimzi/templates/topics-sync-job.yaml new file mode 100644 index 00000000..68b883a4 --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/topics-sync-job.yaml @@ -0,0 +1,129 @@ +{{ if .Values.syncTopics.enabled }} +{{- $connectClusterName := printf "%s-%s" (include "common.names.fullname" .) "connect-api" }} +{{- $connectorName := include "common.names.fullname" . }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }}-topics-sync-job + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + sync-topics.sh: | + #!/bin/sh + set -u # Only unset variable errors + + sleep 5 # Needed for successful connection to the cluster when the job is run for the first time + + echo "Fetching {{ $connectorName }} configuration..." + current_config=$(curl -sf {{ $connectClusterName }}:8083/connectors/{{ $connectorName }}/config) + curl_status=$? + if [ $curl_status -ne 0 ]; then + echo "Cannot reach {{ $connectorName }} cluster; will retry in a moment." + exit $curl_status + fi + + echo "Fetching current topics from catalog-server..." + upstream_topics=$(curl -sf {{ .Values.catalogServer.url }}/source-types | jq -r '[to_entries | .[] | select(.key | test("^.*-source-types$")) | .value[].data[].topic] | sort | unique | join(",")') + curl_status=$? + if [ $curl_status -ne 0 ]; then + echo "Cannot reach catalog-server; will retry in a moment." + exit $curl_status + fi + + if [ -z "$upstream_topics" ]; then + echo "No topics found in catalog-server. Exiting because there is nothing to synchronize." + exit 0 + fi + + current_topics=$(echo "$current_config" | jq -r '.topics') + jq_status=$? + if [ $jq_status -ne 0 ]; then + echo "ERROR: Failed to parse current topics (exit code $jq_status)" + exit $jq_status + fi + + if [ "$upstream_topics" != "$current_topics" ]; then + echo "Updating topics in {{ $connectorName }} config..." + updated_config=$(echo "$current_config" | jq --arg topics "$upstream_topics" '.topics = $topics') + jq_status=$? + if [ $jq_status -ne 0 ]; then + echo "ERROR: Failed to update config with new topics (exit code $jq_status)" + exit $jq_status + fi + curl -X PUT -H "Content-Type: application/json" -d "$updated_config" -sf {{ $connectClusterName }}:8083/connectors/{{ $connectorName }}/config > /dev/null + put_status=$? + if [ $put_status -ne 0 ]; then + echo "ERROR: Failed to update connector config (exit code $put_status)" + exit $put_status + fi + echo "Topics updated successfully." + exit 0 + else + echo "No changes in topics; will not update." + exit 0 + fi +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "common.names.fullname" . }}-topics-sync + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + schedule: {{ default "*/10 * * * *" .Values.syncTopics.cronjob.schedule | quote }} + successfulJobsHistoryLimit: {{ .Values.syncTopics.cronjob.successfulJobsHistoryLimit | default 1 }} + failedJobsHistoryLimit: {{ .Values.syncTopics.cronjob.failedJobsHistoryLimit | default 1 }} + jobTemplate: + spec: + template: + metadata: + labels: + job-selector: {{ include "common.names.fullname" . }}-topics-sync + spec: + containers: + - name: topics-sync + image: {{ .Values.syncTopics.image.repository }}:{{ .Values.syncTopics.image.tag }} + command: ["/tmp/sync-topics.sh"] + volumeMounts: + - mountPath: /tmp/ + name: sync-topics-config + restartPolicy: OnFailure + volumes: + - name: sync-topics-config + configMap: + name: {{ include "common.names.fullname" . }}-topics-sync-job + defaultMode: 0777 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "common.names.fullname" . }}-topics-init + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + backoffLimit: {{ .Values.syncTopics.job.backoffLimit }} + template: + spec: + containers: + - name: topics-sync + image: {{ .Values.syncTopics.image.repository }}:{{ .Values.syncTopics.image.tag }} + command: ["/tmp/sync-topics.sh"] + volumeMounts: + - mountPath: /tmp/ + name: sync-topics-config + restartPolicy: OnFailure + volumes: + - name: sync-topics-config + configMap: + name: {{ include "common.names.fullname" . }}-topics-sync-job + defaultMode: 0777 +{{- end }} diff --git a/charts/radar-s3-connector-strimzi/templates/topics-sync-netpol.yaml b/charts/radar-s3-connector-strimzi/templates/topics-sync-netpol.yaml new file mode 100644 index 00000000..0d3af00f --- /dev/null +++ b/charts/radar-s3-connector-strimzi/templates/topics-sync-netpol.yaml @@ -0,0 +1,30 @@ +{{ if .Values.syncTopics.enabled }} +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ include "common.names.fullname" . }}-topics-sync + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + ingress: + - from: + - podSelector: + matchLabels: + job-name: {{ include "common.names.fullname" . }}-topics-init + - podSelector: + matchLabels: + job-selector: {{ include "common.names.fullname" . }}-topics-sync + ports: + - port: 8083 + protocol: TCP + podSelector: + matchLabels: + strimzi.io/cluster: {{ include "common.names.fullname" . }} + strimzi.io/kind: KafkaConnect + strimzi.io/name: {{ include "common.names.fullname" . }}-connect + policyTypes: + - Ingress + {{- end }} diff --git a/charts/radar-s3-connector-strimzi/values.yaml b/charts/radar-s3-connector-strimzi/values.yaml new file mode 100644 index 00000000..e24a5b6b --- /dev/null +++ b/charts/radar-s3-connector-strimzi/values.yaml @@ -0,0 +1,259 @@ +# Default values for radar-s3-connector-strimzi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of radar-s3-connector replicas to deploy +replicaCount: 1 + +image: + # -- Image registry + registry: docker.io + # -- Image repository + repository: radarbase/kafka-connect-transform-s3 + # -- Image tag (immutable tags are recommended) + # Overrides the image tag whose default is the chart appVersion. + tag: + # -- Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + digest: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # e.g: + # pullSecrets: + # - myRegistryKeySecretName + # + pullSecrets: [] + +# -- String to partially override common.names.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override common.names.fullname template with a string +fullnameOverride: "" + +# -- Configure radar-s3-connector pods' Security Context +podSecurityContext: {} + # fsGroup: 2000 + +# -- Configure radar-s3-connector containers' Security Context +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # -- Kubernetes Service type + type: ClusterIP + # -- radar-s3-connector port + port: 8083 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + + # -- CPU/Memory resource requests + requests: + cpu: 100m + memory: 3Gi + +jvmOptions: + xmx: "4g" + xms: "3g" + +# -- Node labels for pod assignment +nodeSelector: {} + +# -- Toleration labels for pod assignment +tolerations: [] + +# -- Affinity labels for pod assignment +affinity: {} + +secret: + # -- Secret for the Kafka SASL JAAS configuration + jaas: + name: shared-service-user + key: sasl.jaas.config + +# -- Extra environment variables +extraEnvVars: [] + +# -- Custom livenessProbe that overrides the default one +customLivenessProbe: {} + +livenessProbe: + # -- Enable livenessProbe + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 5 + # -- Success threshold for livenessProbe + successThreshold: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + +# -- Custom readinessProbe that overrides the default one +customReadinessProbe: {} + +readinessProbe: + # -- Enable readinessProbe + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 5 + # -- Success threshold for readinessProbe + successThreshold: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + +# -- Network policy defines who can access this application and who this applications has access to +# @default -- check `values.yaml` +networkpolicy: + policyTypes: + - Egress + egress: + - to: + - ipBlock: + cidr: 0.0.0.0/0 + except: + - 10.0.0.0/8 + - 192.168.0.0/16 + - 172.16.0.0/12 + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-kafka-bootstrap' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-schema-registry' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'catalog-server' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'minio' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + +# -- Maximum number of worker threads inside a connector pod. +maxTasks: 5 + +# -- The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. +kafkaVersion: 3.9.0 + +# -- URI of Kafka brokers of the cluster +kafka: SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094 +# -- URL of the Kafka schema registry +schema_registry: http://radar-kafka-schema-registry:8081 +# -- Base URL of the rest-sources-authorizer-backend service +radar_rest_sources_backend_url: http://radar-rest-sources-backend:8080/rest-sources/backend/ +catalogServer: + # -- Catalog server URL + url: http://catalog-server:9010 + +# -- List of topics to be consumed by the sink connector separated by comma. Topics defined in the catalog server +# will automatically be loaded if `syncTopics.enabled` is true. May not be empty or the Strimzi operator will not start the connector. +topics: questionnaire_response +# -- Target S3 endpoint url +s3Endpoint: http://minio:9000/ +# -- set to true, if S3 objects should be tagged with start and end offsets, as well as record count. +s3Tagging: false +# -- The Part Size in S3 Multi-part Uploads. +s3PartSize: 5242880 +# -- The AWS region to be used the connector. Some compatibility layers require this. +s3Region: +# -- Number of records written to store before invoking file commits. +flushSize: 10000 +# -- The time interval in milliseconds to invoke file commits. +rotateInterval: 900000 +# -- Access key of the target S3 bucket +bucketAccessKey: access_key +# -- Secret key of the target S3 bucket +bucketSecretKey: secret +# -- Bucket name of the target S3 bucket +bucketName: radar_intermediate_storage + +cc: + # -- Set to true, if Confluent Cloud is used + enabled: false + # -- Confluent cloud based Kafka broker URL (if Confluent Cloud based Kafka cluster is used) + bootstrapServerurl: "" + # -- Confluent cloud based Schema registry URL (if Confluent Cloud based Schema registry is used) + schemaRegistryUrl: "" + # -- API Key of the Confluent Cloud cluster + apiKey: ccApikey + # -- API secret of the Confluent Cloud cluster + apiSecret: ccApiSecret + # -- API Key of the Confluent Cloud Schema registry + schemaRegistryApiKey: srApiKey + # -- API Key of the Confluent Cloud Schema registry + schemaRegistryApiSecret: srApiSecret + +syncTopics: + # -- If true, fetch list of topics from catalog server + enabled: true + image: + # -- Image repository to fetch topics with + repository: badouralix/curl-jq + # -- Image tag to fetch topics with + tag: latest + # -- Image pull policy to fetch topics with + pullPolicy: IfNotPresent + job: + # -- How often to retry the job in case of failure during topics init. + backoffLimit: 100000 + cronjob: + # -- Cron expression for the topics sync cronjob + schedule: "*/10 * * * *" + # -- Number of successful jobs to keep in history + successfulJobsHistoryLimit: 1 + # -- Number of failed jobs to keep in history + failedJobsHistoryLimit: 1 + + +# -- Log4j configuration +log4j: + # -- Root log level for the Kafka Connect instance + rootLogLevel: INFO + +# Sentry monitoring configuration +sentry: + # -- DSN (Data Source Name) of the sentry server + dsn: + # -- Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) + level: ERROR diff --git a/charts/radar-s3-connector/Chart.yaml b/charts/radar-s3-connector/Chart.yaml index 2af64688..4e838724 100644 --- a/charts/radar-s3-connector/Chart.yaml +++ b/charts/radar-s3-connector/Chart.yaml @@ -2,12 +2,13 @@ apiVersion: v2 appVersion: 7.8.1 description: A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options name: radar-s3-connector -version: 0.7.0 +version: 0.7.1 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-s3-connector - https://github.com/RADAR-base/kafka-connect-transform-keyvalue - https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options +deprecated: true keywords: - radar-base - remote-trial diff --git a/charts/radar-s3-connector/README.md b/charts/radar-s3-connector/README.md index 1125f674..40db9d5e 100644 --- a/charts/radar-s3-connector/README.md +++ b/charts/radar-s3-connector/README.md @@ -2,8 +2,9 @@ # radar-s3-connector [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-s3-connector)](https://artifacthub.io/packages/helm/radar-base/radar-s3-connector) +> **:exclamation: This Helm Chart is deprecated!** -![Version: 0.7.0](https://img.shields.io/badge/Version-0.7.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 7.8.1](https://img.shields.io/badge/AppVersion-7.8.1-informational?style=flat-square) +![Version: 0.7.1](https://img.shields.io/badge/Version-0.7.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 7.8.1](https://img.shields.io/badge/AppVersion-7.8.1-informational?style=flat-square) A Helm chart for RADAR-base s3 connector. This connector uses Confluent s3 connector with a custom data transformers. These configurations enable a sink connector. See full list of properties here https://docs.confluent.io/kafka-connect-s3-sink/current/configuration_options.html#s3-configuration-options diff --git a/charts/radar-upload-connector-strimzi/.helmignore b/charts/radar-upload-connector-strimzi/.helmignore new file mode 100644 index 00000000..50af0317 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/radar-upload-connector-strimzi/Chart.lock b/charts/radar-upload-connector-strimzi/Chart.lock new file mode 100644 index 00000000..f4d129a7 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: common + repository: https://radar-base.github.io/radar-helm-charts + version: 2.27.0 +digest: sha256:397fd12e07f1aa1a5280579406a75b169f8262ac4214584ea366460277288546 +generated: "2025-07-08T17:04:49.580215201+02:00" diff --git a/charts/radar-upload-connector-strimzi/Chart.yaml b/charts/radar-upload-connector-strimzi/Chart.yaml new file mode 100644 index 00000000..a9e07efb --- /dev/null +++ b/charts/radar-upload-connector-strimzi/Chart.yaml @@ -0,0 +1,27 @@ +apiVersion: v2 +appVersion: 0.7.0 +description: A Helm chart for RADAR-base upload kafka connector. This is used for reading uploaded data from backend and sending them to Kafka cluster for later processing. +name: radar-upload-connector-strimzi +version: 0.0.1 +icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" +sources: + - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-upload-source-connector + - https://github.com/RADAR-base/radar-upload-source-connector +keywords: + - radar-base + - remote-trial +annotations: + artifacthub.io/license: Apache-2.0 +deprecated: false +type: application +home: "https://radar-base.org" +maintainers: + - email: pim@thehyve.nl + name: Pim van Nierop + url: https://www.thehyve.nl/experts/pim-van-nierop +dependencies: + - name: common + repository: https://radar-base.github.io/radar-helm-charts + tags: + - bitnami-common + version: 2.x.x diff --git a/charts/radar-upload-connector-strimzi/DOCS.md.gotmpl b/charts/radar-upload-connector-strimzi/DOCS.md.gotmpl new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-upload-connector-strimzi/README.md b/charts/radar-upload-connector-strimzi/README.md new file mode 100644 index 00000000..fbeefd01 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/README.md @@ -0,0 +1,97 @@ + + +# radar-upload-connector-strimzi +[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-upload-connector-strimzi)](https://artifacthub.io/packages/helm/radar-base/radar-upload-connector-strimzi) + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.7.0](https://img.shields.io/badge/AppVersion-0.7.0-informational?style=flat-square) + +A Helm chart for RADAR-base upload kafka connector. This is used for reading uploaded data from backend and sending them to Kafka cluster for later processing. + +**Homepage:** + +## Maintainers + +| Name | Email | Url | +| ---- | ------ | --- | +| Pim van Nierop | | | + +## Source Code + +* +* + +## Prerequisites +* Kubernetes 1.28+ +* Kubectl 1.28+ +* Helm 3.1.0+ +* PV provisioner support in the underlying infrastructure + +## Requirements + +| Repository | Name | Version | +|------------|------|---------| +| https://radar-base.github.io/radar-helm-charts | common | 2.x.x | + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| replicaCount | int | `1` | Number of radar-upload-connector-strimzi replicas to deploy | +| image.registry | string | `"docker.io"` | Image registry | +| image.repository | string | `"radarbase/radar-upload-connector"` | Image repository | +| image.tag | string | `nil` | Image tag (immutable tags are recommended) Overrides the image tag whose default is the chart appVersion. | +| image.digest | string | `""` | Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag | +| image.pullPolicy | string | `"IfNotPresent"` | Image pull policy | +| image.pullSecrets | list | `[]` | Optionally specify an array of imagePullSecrets. Secrets must be manually created in the namespace. e.g: pullSecrets: - myRegistryKeySecretName | +| nameOverride | string | `""` | String to partially override common.names.fullname template with a string (will prepend the release name) | +| fullnameOverride | string | `""` | String to fully override common.names.fullname template with a string | +| podSecurityContext | object | `{}` | Configure radar-upload-connector-strimzi pods' Security Context | +| securityContext | object | `{}` | Configure radar-upload-connector-strimzi containers' Security Context | +| service.type | string | `"ClusterIP"` | Kubernetes Service type | +| service.port | int | `8083` | radar-upload-connector-strimzi port | +| resources.requests | object | `{"cpu":"100m","memory":"1Gi"}` | CPU/Memory resource requests | +| nodeSelector | object | `{}` | Node labels for pod assignment | +| tolerations | list | `[]` | Toleration labels for pod assignment | +| affinity | object | `{}` | Affinity labels for pod assignment | +| secret.jaas | object | `{"key":"sasl.jaas.config","name":"shared-service-user"}` | Secret name for the JAAS configuration | +| customLivenessProbe | object | `{}` | Custom livenessProbe that overrides the default one | +| livenessProbe.enabled | bool | `true` | Enable livenessProbe | +| livenessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for livenessProbe | +| livenessProbe.periodSeconds | int | `30` | Period seconds for livenessProbe | +| livenessProbe.timeoutSeconds | int | `5` | Timeout seconds for livenessProbe | +| livenessProbe.successThreshold | int | `1` | Success threshold for livenessProbe | +| livenessProbe.failureThreshold | int | `3` | Failure threshold for livenessProbe | +| customReadinessProbe | object | `{}` | Custom readinessProbe that overrides the default one | +| readinessProbe.enabled | bool | `true` | Enable readinessProbe | +| readinessProbe.initialDelaySeconds | int | `5` | Initial delay seconds for readinessProbe | +| readinessProbe.periodSeconds | int | `30` | Period seconds for readinessProbe | +| readinessProbe.timeoutSeconds | int | `5` | Timeout seconds for readinessProbe | +| readinessProbe.successThreshold | int | `1` | Success threshold for readinessProbe | +| readinessProbe.failureThreshold | int | `3` | Failure threshold for readinessProbe | +| networkpolicy | object | check `values.yaml` | Network policy defines who can access this application and who this applications has access to | +| jvmOptions | object | `{"xms":"1500m","xmx":"2500m"}` | Java heap options | +| maxTasks | int | `5` | Maximum number of worker threads inside a connector pod. | +| kafkaVersion | string | `"3.9.0"` | The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. | +| kafka | string | `"SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094"` | URI of Kafka brokers of the cluster | +| schema_registry | string | `"http://radar-kafka-schema-registry:8081"` | URL of the Kafka schema registry | +| managementportal_url | string | `"http://management-portal:8080/managementportal"` | URL of the Management Portal | +| radar_upload_connect_backend | string | `"radar-upload-connect-backend"` | Host name of the upload connect backend application | +| client_id | string | `"radar_upload_connect"` | OAuth2 Client Id of the Upload connector | +| client_secret | string | `"upload_secret"` | OAuth2 Client secret of the Upload connector | +| poll_interval | int | `60000` | How often the connector should poll for new records from upload connect backend in milliseconds. | +| record_converter_classes | string | Check values.yaml | List of converter classes to be activated as comma separated values. | +| uploaderType | string | `"s3"` | Uploader type for converters which directly write the files bypassing the Kafka processing. e.g. images and binaries. | +| s3Endpoint | string | `"http://minio:9000/"` | Target S3 endpoint, if files should be written to a location bypassing the Kafka processing. | +| bucketAccessKey | string | `"access_key"` | Target S3 access key | +| bucketSecretKey | string | `"secret"` | Target S3 secret key | +| targetBucketName | string | `"radar-output-storage"` | Target S3 bucket name | +| task.queueSize | int | `10000` | Maximum number of source records that can be produced at a time, preventing out of memory errors. | +| connect.offsetFlushIntervalMs | int | `5000` | Interval at which to try committing offsets for tasks. See | +| producer | object | Check below | Override kafka producer configs. For more details see https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html | +| producer.compressionType | string | `"lz4"` | The compression type for all data generated by the producer. | +| producer.bufferMemory | int | `2000000` | The total bytes of memory the producer can use to buffer records waiting to be sent to the server. | +| producer.batchSize | int | `200000` | Batch size in bytes to batch records together into fewer requests when multiple records are being sent to the same partition. | +| log4j | object | `{"rootLogLevel":"INFO"}` | Log4j configuration | +| log4j.rootLogLevel | string | `"INFO"` | Root log level for the Kafka Connect instance | +| sentry.dsn | string | `nil` | DSN (Data Source Name) of the sentry server | +| sentry.level | string | `"ERROR"` | Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) | diff --git a/charts/radar-upload-connector-strimzi/README.md.gotmpl b/charts/radar-upload-connector-strimzi/README.md.gotmpl new file mode 100644 index 00000000..929d551e --- /dev/null +++ b/charts/radar-upload-connector-strimzi/README.md.gotmpl @@ -0,0 +1,18 @@ +{{ template "common.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "common.prerequisiteswithpv" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} diff --git a/charts/radar-upload-connector-strimzi/charts/common-2.27.0.tgz b/charts/radar-upload-connector-strimzi/charts/common-2.27.0.tgz new file mode 100644 index 00000000..5c539c35 Binary files /dev/null and b/charts/radar-upload-connector-strimzi/charts/common-2.27.0.tgz differ diff --git a/charts/radar-upload-connector-strimzi/templates/NOTES.txt b/charts/radar-upload-connector-strimzi/templates/NOTES.txt new file mode 100644 index 00000000..e69de29b diff --git a/charts/radar-upload-connector-strimzi/templates/_helpers.tpl b/charts/radar-upload-connector-strimzi/templates/_helpers.tpl new file mode 100644 index 00000000..d828fcb0 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/_helpers.tpl @@ -0,0 +1,6 @@ +{{/* +Return the proper image name +*/}} +{{- define "radar-upload-connector.image" -}} +{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global "chart" .Chart ) }} +{{- end -}} diff --git a/charts/radar-upload-connector-strimzi/templates/configmap-log4j.yaml b/charts/radar-upload-connector-strimzi/templates/configmap-log4j.yaml new file mode 100644 index 00000000..2a0aa4b3 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/configmap-log4j.yaml @@ -0,0 +1,29 @@ +# This approach allows to specify dynamic logging configuration for Kafka Connect through an external logging section of KafkaConnect resource specification. +# See: https://strimzi.io/docs/operators/0.45.1/configuring#property-kafka-connect-logging-reference +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "common.names.fullname" . }}-log4j + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +data: + log4j.properties: |- + # Root logger configuration + # The sentryAppender is added to the rootLogger if SENTRY_DSN is set + log4j.rootLogger={{ .Values.log4j.rootLogLevel | default "INFO" }}, stdout{{- if .Values.sentry.dsn }}, sentryAppender{{- end }} + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Appender for Sentry monitoring + {{- if .Values.sentry.dsn }} + log4j.appender.sentryAppender=io.sentry.log4j.SentryAppender + log4j.appender.sentryAppender.threshold={{ .Values.sentry.level | default "INFO" }} + {{- end }} + + # Default log levels + log4j.logger.org.reflections=ERROR + log4j.logger.org.I0Itec.zkclient=ERROR diff --git a/charts/radar-upload-connector-strimzi/templates/kafka-connect.yaml b/charts/radar-upload-connector-strimzi/templates/kafka-connect.yaml new file mode 100644 index 00000000..3df73329 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/kafka-connect.yaml @@ -0,0 +1,131 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnect +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + annotations: + strimzi.io/use-connector-resources: "true" + {{- if .Values.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + version: {{ .Values.kafkaVersion }} + replicas: {{ .Values.replicaCount }} + bootstrapServers: {{ .Values.kafka }} + image: {{ template "radar-upload-connector.image" . }} + {{- if .Values.secret.jaas.name }} + # Authentication-related props are handled by the template builder in https://github.com/strimzi/strimzi-kafka-operator/blob/main/cluster-operator/src/main/java/io/strimzi/operator/cluster/model/KafkaConnectConfigurationBuilder.java + authentication: + type: scram-sha-512 + username: shared-service-user + passwordSecret: + secretName: {{ .Values.secret.jaas.name }} + password: password # The name of the key in the secret that contains the password + {{- end }} + autoRestart: + enabled: true # automatic restarts of failed connectors or their tasks + jvmOptions: + "-Xmx": {{ .Values.jvmOptions.xmx | quote }} + "-Xms": {{ .Values.jvmOptions.xms | quote }} + config: # Properties that configure the Kafka Connect cluster itself + group.id: {{ include "common.names.fullname" . }} + offset.storage.topic: {{ template "common.names.fullname" . }}.offset + config.storage.topic: {{ template "common.names.fullname" . }}.config + status.storage.topic: {{ template "common.names.fullname" . }}.status + # -1 means it will use the default replication factor configured in the broker + config.storage.replication.factor: -1 + offset.storage.replication.factor: -1 + status.storage.replication.factor: -1 + key.converter: "io.confluent.connect.avro.AvroConverter" + value.converter: "io.confluent.connect.avro.AvroConverter" + key.converter.schema.registry.url: {{ .Values.schema_registry }} + value.converter.schema.registry.url: {{ .Values.schema_registry }} + template: + pod: + metadata: + annotations: + checksum/configmap-log4j: {{ include (print $.Template.BasePath "/configmap-log4j.yaml") . | sha256sum }} + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + {{- if .Values.image.pullSecrets }} + imagePullSecrets: + {{- toYaml .Values.image.pullSecrets | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + affinity: + {{- if .Values.affinity }} + {{- toYaml .Values.affinity | nindent 8 }} + {{- end }} + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app.kubernetes.io/name" + operator: In + values: + - {{ template "common.names.name" . }} + - key: "app.kubernetes.io/instance" + operator: In + values: + - {{ .Release.Name }} + topologyKey: "kubernetes.io/hostname" + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + connectContainer: + env: + - name: CONNECT_REST_ADVERTISED_HOST_NAME # Possibly replaced by ADVERTISED_HOSTNAME in kafka_connect_run.sh + value: {{ include "common.names.fullname" . }} + - name: KAFKA_LOG4J_OPTS + value: "-Dlog4j.configuration=file:/opt/kafka/custom-config/log4j.properties" + {{- if .Values.sentry.dsn }} + - name: SENTRY_DSN + value: {{ .Values.sentry.dsn | quote }} + - name: SENTRY_LOG_LEVEL + value: {{ .Values.sentry.level | quote }} + {{- end }} + {{- with .Values.extraEnvVars }} + {{- toYaml . | nindent 8 }} + {{- end }} + logging: + type: external + valueFrom: + configMapKeyRef: + name: {{ template "common.names.fullname" . }}-log4j + key: log4j.properties + {{- if .Values.customLivenessProbe }} + livenessProbe: {{- .Values.customLivenessProbe | toYaml | nindent 12 }} + {{- else if .Values.livenessProbe.enabled }} + livenessProbe: + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + {{- if .Values.customReadinessProbe }} + readinessProbe: {{- .Values.customReadinessProbe | toYaml | nindent 12 }} + {{- else if .Values.readinessProbe.enabled }} + readinessProbe: + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 4 }} diff --git a/charts/radar-upload-connector-strimzi/templates/kafka-connector.yaml b/charts/radar-upload-connector-strimzi/templates/kafka-connector.yaml new file mode 100644 index 00000000..9d40d011 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/kafka-connector.yaml @@ -0,0 +1,23 @@ +apiVersion: kafka.strimzi.io/v1beta2 +kind: KafkaConnector +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: + strimzi.io/cluster: {{ template "common.names.fullname" . }} +spec: + class: org.radarbase.connect.upload.UploadSourceConnector + tasksMax: {{ .Values.maxTasks }} + config: + upload.source.client.id: {{ .Values.client_id }} + upload.source.client.secret: {{ .Values.client_secret }} + upload.source.client.tokenUrl: {{ .Values.managementportal_url }}/oauth/token + upload.source.backend.baseUrl: http://{{ .Values.radar_upload_connect_backend }}:8085/upload/api/ + upload.source.poll.interval.ms: {{ .Values.poll_interval | int }} + upload.source.queue.size: {{ .Values.task.queueSize | int }} + upload.source.record.converter.classes: {{ .Values.record_converter_classes }} + upload.source.file.uploader.type: {{ .Values.uploaderType }} + upload.source.file.uploader.target.endpoint: {{ .Values.s3Endpoint }} + upload.source.file.uploader.target.root.directory: {{ .Values.targetBucketName }} + upload.source.file.uploader.username: {{ .Values.bucketAccessKey }} + upload.source.file.uploader.password: {{ .Values.bucketSecretKey }} diff --git a/charts/radar-upload-connector-strimzi/templates/networkpolicy.yaml b/charts/radar-upload-connector-strimzi/templates/networkpolicy.yaml new file mode 100644 index 00000000..7c4edf91 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/networkpolicy.yaml @@ -0,0 +1,16 @@ +{{- if .Values.networkpolicy }} +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + podSelector: + matchLabels: +{{ include "common.labels.standard" . | indent 6 }} + {{- tpl (toYaml .Values.networkpolicy) . | nindent 2 }} +{{- end -}} diff --git a/charts/radar-upload-connector-strimzi/templates/service.yaml b/charts/radar-upload-connector-strimzi/templates/service.yaml new file mode 100644 index 00000000..d8c4f364 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ template "common.names.fullname" . }} + namespace: {{ include "common.names.namespace" . | quote }} + labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} + {{- if .Values.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + {{- $podLabels := include "common.tplvalues.merge" ( dict "values" ( list .Values.podLabels .Values.commonLabels ) "context" . ) }} + selector: {{- include "common.labels.matchLabels" ( dict "customLabels" $podLabels "context" $ ) | nindent 4 }} diff --git a/charts/radar-upload-connector-strimzi/templates/tests/test-connection.yaml b/charts/radar-upload-connector-strimzi/templates/tests/test-connection.yaml new file mode 100644 index 00000000..f4959a02 --- /dev/null +++ b/charts/radar-upload-connector-strimzi/templates/tests/test-connection.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "common.names.fullname" . }}-test-connection" + labels: + app.kubernetes.io/name: {{ include "common.names.name" . }} + helm.sh/chart: {{ include "common.names.chart" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "common.names.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/radar-upload-connector-strimzi/values.yaml b/charts/radar-upload-connector-strimzi/values.yaml new file mode 100644 index 00000000..eb47a06f --- /dev/null +++ b/charts/radar-upload-connector-strimzi/values.yaml @@ -0,0 +1,233 @@ +# Default values for radar-upload-connector-strimzi. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# -- Number of radar-upload-connector-strimzi replicas to deploy +replicaCount: 1 + +image: + # -- Image registry + registry: docker.io + # -- Image repository + repository: radarbase/radar-upload-connector + # -- Image tag (immutable tags are recommended) + # Overrides the image tag whose default is the chart appVersion. + tag: + # -- Image digest in the way sha256:aa.... Please note this parameter, if set, will override the tag + digest: "" + # -- Image pull policy + pullPolicy: IfNotPresent + # -- Optionally specify an array of imagePullSecrets. + # Secrets must be manually created in the namespace. + # e.g: + # pullSecrets: + # - myRegistryKeySecretName + # + pullSecrets: [] + +# -- String to partially override common.names.fullname template with a string (will prepend the release name) +nameOverride: "" +# -- String to fully override common.names.fullname template with a string +fullnameOverride: "" + +# -- Configure radar-upload-connector-strimzi pods' Security Context +podSecurityContext: {} + # fsGroup: 2000 + +# -- Configure radar-upload-connector-strimzi containers' Security Context +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + # -- Kubernetes Service type + type: ClusterIP + # -- radar-upload-connector-strimzi port + port: 8083 + +resources: + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + + # -- CPU/Memory resource requests + requests: + cpu: 100m + memory: 1Gi + +# -- Node labels for pod assignment +nodeSelector: {} + +# -- Toleration labels for pod assignment +tolerations: [] + +# -- Affinity labels for pod assignment +affinity: {} + +secret: + # -- Secret name for the JAAS configuration + jaas: + name: shared-service-user + key: sasl.jaas.config + +# -- Custom livenessProbe that overrides the default one +customLivenessProbe: {} + +livenessProbe: + # -- Enable livenessProbe + enabled: true + # -- Initial delay seconds for livenessProbe + initialDelaySeconds: 5 + # -- Period seconds for livenessProbe + periodSeconds: 30 + # -- Timeout seconds for livenessProbe + timeoutSeconds: 5 + # -- Success threshold for livenessProbe + successThreshold: 1 + # -- Failure threshold for livenessProbe + failureThreshold: 3 + +# -- Custom readinessProbe that overrides the default one +customReadinessProbe: {} + +readinessProbe: + # -- Enable readinessProbe + enabled: true + # -- Initial delay seconds for readinessProbe + initialDelaySeconds: 5 + # -- Period seconds for readinessProbe + periodSeconds: 30 + # -- Timeout seconds for readinessProbe + timeoutSeconds: 5 + # -- Success threshold for readinessProbe + successThreshold: 1 + # -- Failure threshold for readinessProbe + failureThreshold: 3 + +# -- Network policy defines who can access this application and who this applications has access to +# @default -- check `values.yaml` +networkpolicy: + policyTypes: + - Egress + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-kafka-bootstrap' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-kafka-schema-registry' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'management-portal' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'minio' + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: '{{ .Release.Namespace }}' + podSelector: + matchLabels: + app.kubernetes.io/name: 'radar-upload-connect-backend' + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + +# -- Java heap options +jvmOptions: + xms: "1500m" + xmx: "2500m" + +# -- Maximum number of worker threads inside a connector pod. +maxTasks: 5 + +# -- The version of the Kafka deployed by the Strimzi operator. Must be consistent across the RADAR-base platform. +kafkaVersion: 3.9.0 + +# -- URI of Kafka brokers of the cluster +kafka: SASL_PLAINTEXT://radar-kafka-kafka-bootstrap:9094 +# -- URL of the Kafka schema registry +schema_registry: http://radar-kafka-schema-registry:8081 +# -- URL of the Management Portal +managementportal_url: http://management-portal:8080/managementportal +# -- Host name of the upload connect backend application +radar_upload_connect_backend: radar-upload-connect-backend + +# -- OAuth2 Client Id of the Upload connector +client_id: radar_upload_connect +# -- OAuth2 Client secret of the Upload connector +client_secret: upload_secret +# -- How often the connector should poll for new records from upload connect backend in milliseconds. +poll_interval: 60000 +# -- List of converter classes to be activated as comma separated values. +# @default -- Check values.yaml +record_converter_classes: org.radarbase.connect.upload.converter.altoida.AltoidaConverterFactory,org.radarbase.connect.upload.converter.axivity.AxivityConverterFactory,org.radarbase.connect.upload.converter.oxford.WearableCameraConverterFactory,org.radarbase.connect.upload.converter.gaitup.Physilog5ConverterFactory +# -- Uploader type for converters which directly write the files bypassing the Kafka processing. e.g. images and binaries. +uploaderType: s3 +# -- Target S3 endpoint, if files should be written to a location bypassing the Kafka processing. +s3Endpoint: http://minio:9000/ +# -- Target S3 access key +bucketAccessKey: access_key +# -- Target S3 secret key +bucketSecretKey: secret +# -- Target S3 bucket name +targetBucketName: radar-output-storage + +task: + # -- Maximum number of source records that can be produced at a time, preventing out of memory errors. + queueSize: 10000 + +connect: + # -- Interval at which to try committing offsets for tasks. See + offsetFlushIntervalMs: 5000 + +# -- Override kafka producer configs. For more details see https://docs.confluent.io/platform/current/installation/configuration/producer-configs.html +# @default -- Check below +producer: + # -- The compression type for all data generated by the producer. + compressionType: lz4 + # -- The total bytes of memory the producer can use to buffer records waiting to be sent to the server. + bufferMemory: 2000000 + # -- Batch size in bytes to batch records together into fewer requests when multiple records are being sent to the same partition. + batchSize: 200000 + +# -- Log4j configuration +log4j: + # -- Root log level for the Kafka Connect instance + rootLogLevel: INFO + +# Sentry monitoring configuration +sentry: + # -- DSN (Data Source Name) of the sentry server + dsn: + # -- Log level for sentry (TRACE, DEBUG, INFO, WARN, or ERROR) + level: ERROR diff --git a/charts/radar-upload-source-connector/Chart.yaml b/charts/radar-upload-source-connector/Chart.yaml index 951299c3..064d2cb1 100644 --- a/charts/radar-upload-source-connector/Chart.yaml +++ b/charts/radar-upload-source-connector/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 appVersion: 0.6.2 description: A Helm chart for RADAR-base upload kafka connector. This is used for reading uploaded data from backend and sending them to Kafka cluster for later processing. name: radar-upload-source-connector -version: 0.6.0 +version: 0.6.1 icon: "http://radar-base.org/wp-content/uploads/2022/09/Logo_RADAR-Base-RGB.png" sources: - https://github.com/RADAR-base/radar-helm-charts/tree/main/charts/radar-upload-source-connector @@ -12,7 +12,7 @@ keywords: - remote-trial annotations: artifacthub.io/license: Apache-2.0 -deprecated: false +deprecated: true type: application home: "https://radar-base.org" maintainers: diff --git a/charts/radar-upload-source-connector/README.md b/charts/radar-upload-source-connector/README.md index 74a8050a..3ce57918 100644 --- a/charts/radar-upload-source-connector/README.md +++ b/charts/radar-upload-source-connector/README.md @@ -2,8 +2,9 @@ # radar-upload-source-connector [![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/radar-upload-source-connector)](https://artifacthub.io/packages/helm/radar-base/radar-upload-source-connector) +> **:exclamation: This Helm Chart is deprecated!** -![Version: 0.6.0](https://img.shields.io/badge/Version-0.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) +![Version: 0.6.1](https://img.shields.io/badge/Version-0.6.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.6.2](https://img.shields.io/badge/AppVersion-0.6.2-informational?style=flat-square) A Helm chart for RADAR-base upload kafka connector. This is used for reading uploaded data from backend and sending them to Kafka cluster for later processing.