From 0f8dcd6155b6d4956329ad225549278a23f5e799 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:17:22 +0700 Subject: [PATCH 01/12] Introduce patches with external kafka --- patches/README.md | 8 ++ patches/_lib.sh | 16 +++ patches/external-kafka.sh | 227 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100644 patches/README.md create mode 100755 patches/_lib.sh create mode 100755 patches/external-kafka.sh diff --git a/patches/README.md b/patches/README.md new file mode 100644 index 00000000000..1dd5153a2a1 --- /dev/null +++ b/patches/README.md @@ -0,0 +1,8 @@ +# Self-Hosted Sentry Patches + +Other than the default self-hosted Sentry installation, sometimes users can leverage their existing infrastructure to help them +with limited resources. "Patches", or you might call this like a "plugin system", is a bunch of bash scripts that can be used +to modify the existing configuration to achieve the desired goal. + +Beware that this is very experimental and might not work as expected. +Use it at your own risk. diff --git a/patches/_lib.sh b/patches/_lib.sh new file mode 100755 index 00000000000..42555742c0b --- /dev/null +++ b/patches/_lib.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -euo pipefail +test "${DEBUG:-}" && set -x + +function patch_file() { + target="$1" + content="$2" + if [[ -f "$target" ]]; then + echo "🙈 Patching $target ..." + patch -p1 <"$content" + else + echo "🙊 Skipping $target ..." + fi +} + diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh new file mode 100755 index 00000000000..b5e192881f8 --- /dev/null +++ b/patches/external-kafka.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash + +# Kafka plays a very significant role on Sentry's infrastructure, from ingesting +# to processing events until they end up on ClickHouse or filesystem for permanent +# storage. Since Kafka may require a significant amount of resources on the server +# it may make sense to split it from the main Sentry installation. This can be +# particularly appealing if you already have a managed Kafka cluster set up. +# +# Sentry (the company) itself uses a Kafka cluster on production with a very +# tailored setup, especially for authentication. Some Kafka configuration options +# (such as `SASL_SSL` security protocol) might not be available for some services, +# but since everything is open source, you are encouraged to contribute to +# implement those missing things. +# +# If you are using authentication, make sure that the user is able to create +# new topics. As of now, there is no support for prefixed topic name. +# +# PLEASE NOTE: This patch will only modify the existing configuration files. +# You will have to run `./install.sh` again to apply the changes. + +source patches/_lib.sh + +# If `sentry/sentry.conf.py` exists, we'll modify it. +# Otherwise, we'll use `sentry/sentry.conf.example.py`. +SENTRY_CONFIG_PY="sentry/sentry.conf.py" +if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then + SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" +fi + +ENV_FILE=".env.custom" +if [[ -f "$ENV_FILE" ]]; then + ENV_FILE=".env" +fi + +RELAY_CONFIG_YML="relay/config.yml" +if [[ ! -f "$RELAY_CONFIG_YML" ]]; then + RELAY_CONFIG_YML="relay/config.example.yml" +fi + +patch -p1 $SENTRY_CONFIG_PY <<"EOF" +@@ -136,9 +136,17 @@ + SENTRY_CACHE = "sentry.cache.redis.RedisCache" + + DEFAULT_KAFKA_OPTIONS = { +- "bootstrap.servers": "kafka:9092", ++ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, ++ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++ # If you don't use any of these options below, you can remove them or set them to `None`. ++ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++ "sasl.username": env("KAFKA_SASL_USERNAME", None), ++ "sasl.password": env("KAFKA_SASL_PASSWORD", None), ++ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. + } + + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" +EOF + +# Add additional Kafka options to the ENV_FILE +cat <>"$ENV_FILE" +################################################################################ +## Additional External Kafka options +################################################################################ +KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 +# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL +KAFKA_SECURITY_PROTOCOL=PLAINTEXT +# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. +# KAFKA_SASL_MECHANISM=PLAIN +# KAFKA_SASL_USERNAME=username +# KAFKA_SASL_PASSWORD=password +# Put your certificates on the \`certificates/kafka\` directory. +# The certificates will be mounted as read-only volumes. +# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem +# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem +# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key +EOF + +#TODO: Modify relay config file + +COMPOSE_OVERRIDE_CONTENT=$(cat <<-EOF +x-snuba-defaults: &snuba_defaults + environment: + DEFAULT_BROKERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} + KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} + KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro +x-sentry-defaults: &sentry_defaults + environment: + KAFKA_BOOTSTRAP_SERVERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + KAFKA_SSL_CA_LOCATION: \${KAFKA_SSL_CA_LOCATION:-} + KAFKA_SSL_CERTIFICATE_LOCATION: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + KAFKA_SSL_KEY_LOCATION: \${KAFKA_SSL_KEY_LOCATION:-} + KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro + +services: + kafka: !reset null + vroom: + environment: + SENTRY_KAFKA_BROKERS_PROFILING: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_BROKERS_OCCURRENCES: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_BROKERS_SPANS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} + SENTRY_KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} + SENTRY_KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} + SENTRY_KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} + SENTRY_KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} + SENTRY_KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} + SENTRY_KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} + SENTRY_KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} + volumes: + - ./certificates/kafka:/kafka-certificates:ro + snuba-api: + <<: *snuba_defaults + snuba-errors-consumer: + <<: *snuba_defaults + snuba-outcomes-consumer: + <<: *snuba_defaults + snuba-outcomes-billing-consumer: + <<: *snuba_defaults + snuba-group-attributes-consumer: + <<: *snuba_defaults + snuba-replacer: + <<: *snuba_defaults + snuba-subscription-consumer-events: + <<: *snuba_defaults + snuba-transactions-consumer: + <<: *snuba_defaults + snuba-replays-consumer: + <<: *snuba_defaults + snuba-issue-occurrence-consumer: + <<: *snuba_defaults + snuba-metrics-consumer: + <<: *snuba_defaults + snuba-subscription-consumer-transactions: + <<: *snuba_defaults + snuba-subscription-consumer-metrics: + <<: *snuba_defaults + snuba-generic-metrics-distributions-consumer: + <<: *snuba_defaults + snuba-generic-metrics-sets-consumer: + <<: *snuba_defaults + snuba-generic-metrics-counters-consumer: + <<: *snuba_defaults + snuba-generic-metrics-gauges-consumer: + <<: *snuba_defaults + snuba-profiling-profiles-consumer: + <<: *snuba_defaults + snuba-profiling-functions-consumer: + <<: *snuba_defaults + snuba-spans-consumer: + <<: *snuba_defaults + web: + <<: *sentry_defaults + cron: + <<: *sentry_defaults + worker: + <<: *sentry_defaults + events-consumer: + <<: *sentry_defaults + attachments-consumer: + <<: *sentry_defaults + post-process-forwarder-errors: + <<: *sentry_defaults + subscription-consumer-events: + <<: *sentry_defaults + transactions-consumer: + <<: *sentry_defaults + metrics-consumer: + <<: *sentry_defaults + generic-metrics-consumer: + <<: *sentry_defaults + billing-metrics-consumer: + <<: *sentry_defaults + ingest-replay-recordings: + <<: *sentry_defaults + ingest-occurrences: + <<: *sentry_defaults + ingest-profiles: + <<: *sentry_defaults + ingest-monitors: + <<: *sentry_defaults + ingest-feedback-events: + <<: *sentry_defaults + monitors-clock-tick: + <<: *sentry_defaults + monitors-clock-tasks: + <<: *sentry_defaults + post-process-forwarder-transactions: + <<: *sentry_defaults + post-process-forwarder-issue-platform: + <<: *sentry_defaults + subscription-consumer-transactions: + <<: *sentry_defaults + subscription-consumer-metrics: + <<: *sentry_defaults + subscription-consumer-generic-metrics: + <<: *sentry_defaults +EOF +) +if [[ -f "docker-compose.override.yml" ]]; then + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "$COMPOSE_OVERRIDE_CONTENT" +else + echo "🚨 docker-compose.override.yml does not exist. Creating it now." + echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml +fi + +echo "" +echo "" +echo "--------------------------------------------------------------" +echo "- Finished patching external-kafka.sh" +echo "- Modify the Kafka credentials on your $ENV_FILE file" +echo "- Then, run ./install.sh" +echo "--------------------------------------------------------------" From 62d5a4f905735f327a24c91043c49107638a8450 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:21:55 +0700 Subject: [PATCH 02/12] Fix pre-commit hooks --- patches/external-kafka.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index b5e192881f8..08e797bab5b 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -24,23 +24,23 @@ source patches/_lib.sh # Otherwise, we'll use `sentry/sentry.conf.example.py`. SENTRY_CONFIG_PY="sentry/sentry.conf.py" if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then - SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" + SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" fi ENV_FILE=".env.custom" -if [[ -f "$ENV_FILE" ]]; then - ENV_FILE=".env" +if [[ ! -f "$ENV_FILE" ]]; then + ENV_FILE=".env" fi RELAY_CONFIG_YML="relay/config.yml" if [[ ! -f "$RELAY_CONFIG_YML" ]]; then - RELAY_CONFIG_YML="relay/config.example.yml" + RELAY_CONFIG_YML="relay/config.example.yml" fi patch -p1 $SENTRY_CONFIG_PY <<"EOF" @@ -136,9 +136,17 @@ SENTRY_CACHE = "sentry.cache.redis.RedisCache" - + DEFAULT_KAFKA_OPTIONS = { - "bootstrap.servers": "kafka:9092", + "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), @@ -55,12 +55,13 @@ patch -p1 $SENTRY_CONFIG_PY <<"EOF" + "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. + "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. } - + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" EOF # Add additional Kafka options to the ENV_FILE cat <>"$ENV_FILE" + ################################################################################ ## Additional External Kafka options ################################################################################ @@ -80,7 +81,8 @@ EOF #TODO: Modify relay config file -COMPOSE_OVERRIDE_CONTENT=$(cat <<-EOF +COMPOSE_OVERRIDE_CONTENT=$( + cat <<-EOF x-snuba-defaults: &snuba_defaults environment: DEFAULT_BROKERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} @@ -211,11 +213,11 @@ services: EOF ) if [[ -f "docker-compose.override.yml" ]]; then - echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" - echo "$COMPOSE_OVERRIDE_CONTENT" + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "$COMPOSE_OVERRIDE_CONTENT" else - echo "🚨 docker-compose.override.yml does not exist. Creating it now." - echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml + echo "🚨 docker-compose.override.yml does not exist. Creating it now." + echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml fi echo "" From f9946894901aca54dbcc114b8e250540b4b75b8b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sat, 11 Jan 2025 20:31:23 +0700 Subject: [PATCH 03/12] Patch relay config file --- patches/_lib.sh | 1 - patches/external-kafka.sh | 31 ++++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/patches/_lib.sh b/patches/_lib.sh index 42555742c0b..46e55d3e257 100755 --- a/patches/_lib.sh +++ b/patches/_lib.sh @@ -13,4 +13,3 @@ function patch_file() { echo "🙊 Skipping $target ..." fi } - diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 08e797bab5b..3c4802f6ca6 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -79,7 +79,24 @@ KAFKA_SECURITY_PROTOCOL=PLAINTEXT # KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key EOF -#TODO: Modify relay config file +patch -p1 $RELAY_CONFIG_YML <<"EOF" +@@ -7,8 +7,15 @@ + processing: + enabled: true + kafka_config: +- - {name: "bootstrap.servers", value: "kafka:9092"} ++ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} + - {name: "message.max.bytes", value: 50000000} # 50MB ++ - {name: "security.protocol", value: "PLAINTEXT"} ++ - {name: "sasl.mechanism", value: "PLAIN"} ++ - {name: "sasl.username", value: "username"} ++ - {name: "sasl.password", value: "password"} ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} + redis: redis://redis:6379 + geoip_path: "/geoip/GeoLite2-City.mmdb" +EOF COMPOSE_OVERRIDE_CONTENT=$( cat <<-EOF @@ -124,6 +141,9 @@ services: SENTRY_KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} volumes: - ./certificates/kafka:/kafka-certificates:ro + relay: + volumes: + - ./certificates/kafka:/kafka-certificates:ro snuba-api: <<: *snuba_defaults snuba-errors-consumer: @@ -222,8 +242,9 @@ fi echo "" echo "" -echo "--------------------------------------------------------------" -echo "- Finished patching external-kafka.sh" -echo "- Modify the Kafka credentials on your $ENV_FILE file" +echo "------------------------------------------------------------------------" +echo "- Finished patching external-kafka.sh. Some things you'll need to do:" +echo "- Modify the Kafka credentials on your $ENV_FILE file." +echo "- Modify the Kafka credentials on your $RELAY_CONFIG_YML file." echo "- Then, run ./install.sh" -echo "--------------------------------------------------------------" +echo "------------------------------------------------------------------------" From 8e5aa7d0f5bea074ace92e1beb85a251acefd03b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 07:56:18 +0700 Subject: [PATCH 04/12] Documentation for patches stuff --- patches/README.md | 24 +++++++++++++++++++++--- patches/external-kafka.sh | 30 ++++++++++++++++++++---------- 2 files changed, 41 insertions(+), 13 deletions(-) diff --git a/patches/README.md b/patches/README.md index 1dd5153a2a1..0319a3729db 100644 --- a/patches/README.md +++ b/patches/README.md @@ -1,8 +1,26 @@ # Self-Hosted Sentry Patches Other than the default self-hosted Sentry installation, sometimes users can leverage their existing infrastructure to help them -with limited resources. "Patches", or you might call this like a "plugin system", is a bunch of bash scripts that can be used +with limited resources. "Patches", or you might call this like a "plugin system", is a collection of bash scripts that can be used to modify the existing configuration to achieve the desired goal. -Beware that this is very experimental and might not work as expected. -Use it at your own risk. +> [!WARNING] +> Beware that this is very experimental and might not work as expected. +> +> **Use it at your own risk!** + +## How to use patches + +The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. +They should be run from the root directory. For example, the `external-kafka.sh` patch should be run as: +```bash +./patches/external-kafka.sh +``` + +Some patches might require additional steps to be taken, like providing credentials or additional TLS certificates. + +## Official support + +Sentry employees are not obliged to provide dedicated support for patches, but they can help by providing information to move us forward. We encourage the community to contribute for any bug fixes or improvements. + +See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information. diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 3c4802f6ca6..00f4883868d 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -22,6 +22,7 @@ source patches/_lib.sh # If `sentry/sentry.conf.py` exists, we'll modify it. # Otherwise, we'll use `sentry/sentry.conf.example.py`. +# This kind of conditional logic will be used with other files. SENTRY_CONFIG_PY="sentry/sentry.conf.py" if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" @@ -37,6 +38,8 @@ if [[ ! -f "$RELAY_CONFIG_YML" ]]; then RELAY_CONFIG_YML="relay/config.example.yml" fi +## XXX(aldy505): Create the diff by running `diff -u sentry/sentry.conf.py sentry/sentry.conf.example.py`. +## But you'll need to have your own `sentry/sentry.conf.py` file with the changes already set. patch -p1 $SENTRY_CONFIG_PY <<"EOF" @@ -136,9 +136,17 @@ SENTRY_CACHE = "sentry.cache.redis.RedisCache" @@ -60,7 +63,11 @@ patch -p1 $SENTRY_CONFIG_PY <<"EOF" EOF # Add additional Kafka options to the ENV_FILE -cat <>"$ENV_FILE" +# Only patch this when "KAFKA_BOOTSTRAP_SERVERS" is not set. +if [[ grep -q "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}" ]]; then + echo "🚨 Skipping patching of ${ENV_FILE}" +else + cat <>"$ENV_FILE" ################################################################################ ## Additional External Kafka options @@ -78,6 +85,7 @@ KAFKA_SECURITY_PROTOCOL=PLAINTEXT # KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem # KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key EOF +fi patch -p1 $RELAY_CONFIG_YML <<"EOF" @@ -7,8 +7,15 @@ @@ -88,12 +96,12 @@ patch -p1 $RELAY_CONFIG_YML <<"EOF" + - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} - {name: "message.max.bytes", value: 50000000} # 50MB + - {name: "security.protocol", value: "PLAINTEXT"} -+ - {name: "sasl.mechanism", value: "PLAIN"} -+ - {name: "sasl.username", value: "username"} -+ - {name: "sasl.password", value: "password"} -+ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} -+ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} -+ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} ++ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used. ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used. redis: redis://redis:6379 geoip_path: "/geoip/GeoLite2-City.mmdb" EOF @@ -244,7 +252,9 @@ echo "" echo "" echo "------------------------------------------------------------------------" echo "- Finished patching external-kafka.sh. Some things you'll need to do:" -echo "- Modify the Kafka credentials on your $ENV_FILE file." -echo "- Modify the Kafka credentials on your $RELAY_CONFIG_YML file." -echo "- Then, run ./install.sh" +echo "- 1. Modify the Kafka credentials on your $ENV_FILE file." +echo "- 2. Modify the Kafka credentials on your $RELAY_CONFIG_YML file." +echo "- 3. Run ./install.sh" +echo "-" +echo "- NOTE: Remove or comment the corresponding line if you don't use it." echo "------------------------------------------------------------------------" From 4135ee6139fd6d6915a95bdcaa5b875bd80ce885 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 08:08:15 +0700 Subject: [PATCH 05/12] Provide more helpful information for Docker Compose Override file --- patches/README.md | 2 +- patches/external-kafka.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/patches/README.md b/patches/README.md index 0319a3729db..dfdf48f9126 100644 --- a/patches/README.md +++ b/patches/README.md @@ -11,7 +11,7 @@ to modify the existing configuration to achieve the desired goal. ## How to use patches -The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. +The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. They should be run from the root directory. For example, the `external-kafka.sh` patch should be run as: ```bash ./patches/external-kafka.sh diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index 00f4883868d..bbf17a3a436 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -241,7 +241,9 @@ services: EOF ) if [[ -f "docker-compose.override.yml" ]]; then - echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually:" + echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually with the following content." + echo "Changes: Reset the 'kafka' service block, add additional Kafka-related environment variables to 'vroom', 'relay', 'sentry'-related and 'snuba'-related services." + echo "" echo "$COMPOSE_OVERRIDE_CONTENT" else echo "🚨 docker-compose.override.yml does not exist. Creating it now." @@ -257,4 +259,6 @@ echo "- 2. Modify the Kafka credentials on your $RELAY_CONFIG_YML file." echo "- 3. Run ./install.sh" echo "-" echo "- NOTE: Remove or comment the corresponding line if you don't use it." +echo "- It is also safe to prune/delete 'sentry-kafka'" +echo "- and 'sentry-kafka-log' Docker volumes." echo "------------------------------------------------------------------------" From 9a204dda9214094a4a11e6082451d97ee5a7d467 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Sun, 12 Jan 2025 08:23:34 +0700 Subject: [PATCH 06/12] Fix grep command --- patches/external-kafka.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh index bbf17a3a436..a024dcee283 100755 --- a/patches/external-kafka.sh +++ b/patches/external-kafka.sh @@ -64,8 +64,9 @@ EOF # Add additional Kafka options to the ENV_FILE # Only patch this when "KAFKA_BOOTSTRAP_SERVERS" is not set. -if [[ grep -q "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}" ]]; then - echo "🚨 Skipping patching of ${ENV_FILE}" +kafka_config_exists_on_env=$(grep -c "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}") +if [[ "$kafka_config_exists_on_env" -ne 0 ]]; then + echo "🚨 Skipping patching of ${ENV_FILE}. You may already have Kafka-related environment variables set." else cat <>"$ENV_FILE" From 9c501e04d385b92ec27194872bab943c4ba3afa7 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 15 May 2025 11:08:50 +0700 Subject: [PATCH 07/12] ref: rename to 'optional-modifications' --- optional-modifications/README.md | 41 +++ {patches => optional-modifications}/_lib.sh | 0 .../patches/external-kafka/.env.patch | 22 ++ .../external-kafka/config.example.yml.patch | 19 ++ .../external-kafka/docker-compose.yml.patch | 142 ++++++++++ .../sentry.conf.example.py.patch | 21 ++ patches/README.md | 26 -- patches/external-kafka.sh | 265 ------------------ 8 files changed, 245 insertions(+), 291 deletions(-) create mode 100644 optional-modifications/README.md rename {patches => optional-modifications}/_lib.sh (100%) create mode 100644 optional-modifications/patches/external-kafka/.env.patch create mode 100644 optional-modifications/patches/external-kafka/config.example.yml.patch create mode 100644 optional-modifications/patches/external-kafka/docker-compose.yml.patch create mode 100644 optional-modifications/patches/external-kafka/sentry.conf.example.py.patch delete mode 100644 patches/README.md delete mode 100755 patches/external-kafka.sh diff --git a/optional-modifications/README.md b/optional-modifications/README.md new file mode 100644 index 00000000000..84cf7e0e2b9 --- /dev/null +++ b/optional-modifications/README.md @@ -0,0 +1,41 @@ +# Optional Modifications + +Other than the default self-hosted Sentry installation, sometimes users +can leverage their existing infrastructure to help them with limited +resources. "Patches", or you might call this like a "plugin system", is +a collection of patch files (see [man patch(1)](https://man7.org/linux/man-pages/man1/patch.1.html)) +that can be used with to modify the existing configuration to achieve +the desired goal. + +> [!WARNING] +> Beware that this is very experimental and might not work as expected. +> +> **Use it at your own risk!** + +## How to use patches + +The patches are designed mostly to help modify the existing +configuration files. You will need to run the `install.sh` script +afterwards. + +They should be run from the root directory. For example, the +`external-kafka` patches should be run as: + +```bash +patch < optional-modifications/patches/external-kafka/.env.patch +patch < optional-modifications/patches/external-kafka/config.example.yml.patch +patch < optional-modifications/patches/external-kafka/sentry.conf.example.py.patch +patch < optional-modifications/patches/external-kafka/docker-compose.yml.patch +``` + +Some patches might require additional steps to be taken, like providing +credentials or additional TLS certificates. + +## Official support + +Sentry employees are not obliged to provide dedicated support for +patches, but they can help by providing information to move us forward. +We encourage the community to contribute for any bug fixes or +improvements. + +See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information. diff --git a/patches/_lib.sh b/optional-modifications/_lib.sh similarity index 100% rename from patches/_lib.sh rename to optional-modifications/_lib.sh diff --git a/optional-modifications/patches/external-kafka/.env.patch b/optional-modifications/patches/external-kafka/.env.patch new file mode 100644 index 00000000000..36e45f7785f --- /dev/null +++ b/optional-modifications/patches/external-kafka/.env.patch @@ -0,0 +1,22 @@ +--- .env 2025-02-04 07:31:54.868049984 +0700 ++++ .env.external-kafka 2025-05-15 08:33:15.442361105 +0700 +@@ -22,3 +22,19 @@ + POSTGRES_MAX_CONNECTIONS=100 + # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets + # SETUP_JS_SDK_ASSETS=1 ++ ++################################################################################ ++## Additional External Kafka options ++################################################################################ ++KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 ++# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++KAFKA_SECURITY_PROTOCOL=PLAINTEXT ++# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++# KAFKA_SASL_MECHANISM=PLAIN ++# KAFKA_SASL_USERNAME=username ++# KAFKA_SASL_PASSWORD=password ++# Put your certificates on the \`certificates/kafka\` directory. ++# The certificates will be mounted as read-only volumes. ++# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem ++# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem ++# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key diff --git a/optional-modifications/patches/external-kafka/config.example.yml.patch b/optional-modifications/patches/external-kafka/config.example.yml.patch new file mode 100644 index 00000000000..a0c1aab04b8 --- /dev/null +++ b/optional-modifications/patches/external-kafka/config.example.yml.patch @@ -0,0 +1,19 @@ +--- relay/config.example.yml 2025-05-15 08:27:40.426876887 +0700 ++++ relay/config.example.external-kafka.yml 2025-05-15 08:34:21.113311217 +0700 +@@ -7,8 +7,15 @@ + processing: + enabled: true + kafka_config: +- - {name: "bootstrap.servers", value: "kafka:9092"} ++ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} + - {name: "message.max.bytes", value: 50000000} # 50MB ++ - {name: "security.protocol", value: "PLAINTEXT"} ++ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used. ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used. + redis: redis://redis:6379 + geoip_path: "/geoip/GeoLite2-City.mmdb" + diff --git a/optional-modifications/patches/external-kafka/docker-compose.yml.patch b/optional-modifications/patches/external-kafka/docker-compose.yml.patch new file mode 100644 index 00000000000..ad0328410b7 --- /dev/null +++ b/optional-modifications/patches/external-kafka/docker-compose.yml.patch @@ -0,0 +1,142 @@ +--- docker-compose.yml 2025-03-17 13:32:15.120328412 +0700 ++++ docker-compose.external-kafka.yml 2025-05-15 08:39:05.509951068 +0700 +@@ -26,8 +26,6 @@ + depends_on: + redis: + <<: *depends_on-healthy +- kafka: +- <<: *depends_on-healthy + postgres: + <<: *depends_on-healthy + memcached: +@@ -59,6 +57,14 @@ + SENTRY_EVENT_RETENTION_DAYS: + SENTRY_MAIL_HOST: + SENTRY_MAX_EXTERNAL_SOURCEMAP_SIZE: ++ KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-} ++ KAFKA_SSL_CERTIFICATE_LOCATION: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-} ++ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + volumes: + - "sentry-data:/data" + - "./sentry:/etc/sentry" +@@ -69,15 +75,20 @@ + depends_on: + clickhouse: + <<: *depends_on-healthy +- kafka: +- <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + image: "$SNUBA_IMAGE" + environment: + SNUBA_SETTINGS: self_hosted + CLICKHOUSE_HOST: clickhouse +- DEFAULT_BROKERS: "kafka:9092" ++ DEFAULT_BROKERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-} ++ KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-} ++ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + REDIS_HOST: redis + UWSGI_MAX_REQUESTS: "10000" + UWSGI_DISABLE_LOGGING: "true" +@@ -140,43 +151,7 @@ + POSTGRES_HOST_AUTH_METHOD: "trust" + volumes: + - "sentry-postgres:/var/lib/postgresql/data" +- kafka: +- <<: *restart_policy +- image: "confluentinc/cp-kafka:7.6.1" +- environment: +- # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example +- KAFKA_PROCESS_ROLES: "broker,controller" +- KAFKA_CONTROLLER_QUORUM_VOTERS: "1001@127.0.0.1:29093" +- KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" +- KAFKA_NODE_ID: "1001" +- CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" +- KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093" +- KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092" +- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT" +- KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" +- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" +- KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" +- KAFKA_LOG_RETENTION_HOURS: "24" +- KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust +- KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too +- CONFLUENT_SUPPORT_METRICS_ENABLE: "false" +- KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN" +- KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" +- KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" +- ulimits: +- nofile: +- soft: 4096 +- hard: 4096 +- volumes: +- - "sentry-kafka:/var/lib/kafka/data" +- - "sentry-kafka-log:/var/lib/kafka/log" +- - "sentry-secrets:/etc/kafka/secrets" +- healthcheck: +- <<: *healthcheck_defaults +- test: ["CMD-SHELL", "nc -z localhost 9092"] +- interval: 10s +- timeout: 10s +- retries: 30 ++ kafka: !reset null + clickhouse: + <<: *restart_policy + image: clickhouse-self-hosted-local +@@ -475,9 +450,8 @@ + read_only: true + source: ./geoip + target: /geoip ++ - ./certificates/kafka:/kafka-certificates:ro + depends_on: +- kafka: +- <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + web: +@@ -486,15 +460,21 @@ + <<: *restart_policy + image: "$VROOM_IMAGE" + environment: +- SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092" +- SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092" ++ SENTRY_KAFKA_BROKERS_PROFILING: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_BROKERS_OCCURRENCES: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_BROKERS_SPANS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ SENTRY_KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-} ++ SENTRY_KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ SENTRY_KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-} ++ SENTRY_KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ SENTRY_KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ SENTRY_KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles + SENTRY_SNUBA_HOST: "http://snuba-api:1218" + volumes: + - sentry-vroom:/var/lib/sentry-profiles +- depends_on: +- kafka: +- <<: *depends_on-healthy ++ - ./certificates/kafka:/kafka-certificates:ro + profiles: + - feature-complete + vroom-cleanup: +@@ -523,8 +503,6 @@ + external: true + sentry-redis: + external: true +- sentry-kafka: +- external: true + sentry-clickhouse: + external: true + sentry-symbolicator: diff --git a/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch b/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch new file mode 100644 index 00000000000..abc755c00a0 --- /dev/null +++ b/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch @@ -0,0 +1,21 @@ +--- sentry/sentry.conf.example.py 2025-05-15 08:27:40.427876868 +0700 ++++ sentry/sentry.conf.example.external-kafka.py 2025-05-15 08:32:44.845127931 +0700 +@@ -132,9 +132,17 @@ + SENTRY_CACHE = "sentry.cache.redis.RedisCache" + + DEFAULT_KAFKA_OPTIONS = { +- "bootstrap.servers": "kafka:9092", ++ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, ++ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++ # If you don't use any of these options below, you can remove them or set them to `None`. ++ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++ "sasl.username": env("KAFKA_SASL_USERNAME", None), ++ "sasl.password": env("KAFKA_SASL_PASSWORD", None), ++ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. + } + + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" diff --git a/patches/README.md b/patches/README.md deleted file mode 100644 index dfdf48f9126..00000000000 --- a/patches/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Self-Hosted Sentry Patches - -Other than the default self-hosted Sentry installation, sometimes users can leverage their existing infrastructure to help them -with limited resources. "Patches", or you might call this like a "plugin system", is a collection of bash scripts that can be used -to modify the existing configuration to achieve the desired goal. - -> [!WARNING] -> Beware that this is very experimental and might not work as expected. -> -> **Use it at your own risk!** - -## How to use patches - -The patches are designed mostly to help modify the existing configuration files. You will need to run the `install.sh` script afterwards. -They should be run from the root directory. For example, the `external-kafka.sh` patch should be run as: -```bash -./patches/external-kafka.sh -``` - -Some patches might require additional steps to be taken, like providing credentials or additional TLS certificates. - -## Official support - -Sentry employees are not obliged to provide dedicated support for patches, but they can help by providing information to move us forward. We encourage the community to contribute for any bug fixes or improvements. - -See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information. diff --git a/patches/external-kafka.sh b/patches/external-kafka.sh deleted file mode 100755 index a024dcee283..00000000000 --- a/patches/external-kafka.sh +++ /dev/null @@ -1,265 +0,0 @@ -#!/usr/bin/env bash - -# Kafka plays a very significant role on Sentry's infrastructure, from ingesting -# to processing events until they end up on ClickHouse or filesystem for permanent -# storage. Since Kafka may require a significant amount of resources on the server -# it may make sense to split it from the main Sentry installation. This can be -# particularly appealing if you already have a managed Kafka cluster set up. -# -# Sentry (the company) itself uses a Kafka cluster on production with a very -# tailored setup, especially for authentication. Some Kafka configuration options -# (such as `SASL_SSL` security protocol) might not be available for some services, -# but since everything is open source, you are encouraged to contribute to -# implement those missing things. -# -# If you are using authentication, make sure that the user is able to create -# new topics. As of now, there is no support for prefixed topic name. -# -# PLEASE NOTE: This patch will only modify the existing configuration files. -# You will have to run `./install.sh` again to apply the changes. - -source patches/_lib.sh - -# If `sentry/sentry.conf.py` exists, we'll modify it. -# Otherwise, we'll use `sentry/sentry.conf.example.py`. -# This kind of conditional logic will be used with other files. -SENTRY_CONFIG_PY="sentry/sentry.conf.py" -if [[ ! -f "$SENTRY_CONFIG_PY" ]]; then - SENTRY_CONFIG_PY="sentry/sentry.conf.example.py" -fi - -ENV_FILE=".env.custom" -if [[ ! -f "$ENV_FILE" ]]; then - ENV_FILE=".env" -fi - -RELAY_CONFIG_YML="relay/config.yml" -if [[ ! -f "$RELAY_CONFIG_YML" ]]; then - RELAY_CONFIG_YML="relay/config.example.yml" -fi - -## XXX(aldy505): Create the diff by running `diff -u sentry/sentry.conf.py sentry/sentry.conf.example.py`. -## But you'll need to have your own `sentry/sentry.conf.py` file with the changes already set. -patch -p1 $SENTRY_CONFIG_PY <<"EOF" -@@ -136,9 +136,17 @@ - SENTRY_CACHE = "sentry.cache.redis.RedisCache" - - DEFAULT_KAFKA_OPTIONS = { -- "bootstrap.servers": "kafka:9092", -+ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), - "message.max.bytes": 50000000, - "socket.timeout.ms": 1000, -+ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL -+ # If you don't use any of these options below, you can remove them or set them to `None`. -+ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. -+ "sasl.username": env("KAFKA_SASL_USERNAME", None), -+ "sasl.password": env("KAFKA_SASL_PASSWORD", None), -+ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used. -+ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. -+ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. - } - - SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" -EOF - -# Add additional Kafka options to the ENV_FILE -# Only patch this when "KAFKA_BOOTSTRAP_SERVERS" is not set. -kafka_config_exists_on_env=$(grep -c "KAFKA_BOOTSTRAP_SERVERS" "${ENV_FILE}") -if [[ "$kafka_config_exists_on_env" -ne 0 ]]; then - echo "🚨 Skipping patching of ${ENV_FILE}. You may already have Kafka-related environment variables set." -else - cat <>"$ENV_FILE" - -################################################################################ -## Additional External Kafka options -################################################################################ -KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 -# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL -KAFKA_SECURITY_PROTOCOL=PLAINTEXT -# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. -# KAFKA_SASL_MECHANISM=PLAIN -# KAFKA_SASL_USERNAME=username -# KAFKA_SASL_PASSWORD=password -# Put your certificates on the \`certificates/kafka\` directory. -# The certificates will be mounted as read-only volumes. -# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem -# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem -# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key -EOF -fi - -patch -p1 $RELAY_CONFIG_YML <<"EOF" -@@ -7,8 +7,15 @@ - processing: - enabled: true - kafka_config: -- - {name: "bootstrap.servers", value: "kafka:9092"} -+ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} - - {name: "message.max.bytes", value: 50000000} # 50MB -+ - {name: "security.protocol", value: "PLAINTEXT"} -+ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used. -+ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used. -+ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used. -+ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used. -+ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used. -+ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used. - redis: redis://redis:6379 - geoip_path: "/geoip/GeoLite2-City.mmdb" -EOF - -COMPOSE_OVERRIDE_CONTENT=$( - cat <<-EOF -x-snuba-defaults: &snuba_defaults - environment: - DEFAULT_BROKERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} - KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} - KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} - KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} - KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} - KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} - KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} - volumes: - - ./certificates/kafka:/kafka-certificates:ro -x-sentry-defaults: &sentry_defaults - environment: - KAFKA_BOOTSTRAP_SERVERS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} - KAFKA_SSL_CA_LOCATION: \${KAFKA_SSL_CA_LOCATION:-} - KAFKA_SSL_CERTIFICATE_LOCATION: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} - KAFKA_SSL_KEY_LOCATION: \${KAFKA_SSL_KEY_LOCATION:-} - KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} - KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} - KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} - volumes: - - ./certificates/kafka:/kafka-certificates:ro - -services: - kafka: !reset null - vroom: - environment: - SENTRY_KAFKA_BROKERS_PROFILING: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - SENTRY_KAFKA_BROKERS_OCCURRENCES: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - SENTRY_KAFKA_BROKERS_SPANS: \${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} - SENTRY_KAFKA_SECURITY_PROTOCOL: \${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} - SENTRY_KAFKA_SSL_CA_PATH: \${KAFKA_SSL_CA_LOCATION:-} - SENTRY_KAFKA_SSL_CERT_PATH: \${KAFKA_SSL_CERTIFICATE_LOCATION:-} - SENTRY_KAFKA_SSL_KEY_PATH: \${KAFKA_SSL_KEY_LOCATION:-} - SENTRY_KAFKA_SASL_MECHANISM: \${KAFKA_SASL_MECHANISM:-} - SENTRY_KAFKA_SASL_USERNAME: \${KAFKA_SASL_USERNAME:-} - SENTRY_KAFKA_SASL_PASSWORD: \${KAFKA_SASL_PASSWORD:-} - volumes: - - ./certificates/kafka:/kafka-certificates:ro - relay: - volumes: - - ./certificates/kafka:/kafka-certificates:ro - snuba-api: - <<: *snuba_defaults - snuba-errors-consumer: - <<: *snuba_defaults - snuba-outcomes-consumer: - <<: *snuba_defaults - snuba-outcomes-billing-consumer: - <<: *snuba_defaults - snuba-group-attributes-consumer: - <<: *snuba_defaults - snuba-replacer: - <<: *snuba_defaults - snuba-subscription-consumer-events: - <<: *snuba_defaults - snuba-transactions-consumer: - <<: *snuba_defaults - snuba-replays-consumer: - <<: *snuba_defaults - snuba-issue-occurrence-consumer: - <<: *snuba_defaults - snuba-metrics-consumer: - <<: *snuba_defaults - snuba-subscription-consumer-transactions: - <<: *snuba_defaults - snuba-subscription-consumer-metrics: - <<: *snuba_defaults - snuba-generic-metrics-distributions-consumer: - <<: *snuba_defaults - snuba-generic-metrics-sets-consumer: - <<: *snuba_defaults - snuba-generic-metrics-counters-consumer: - <<: *snuba_defaults - snuba-generic-metrics-gauges-consumer: - <<: *snuba_defaults - snuba-profiling-profiles-consumer: - <<: *snuba_defaults - snuba-profiling-functions-consumer: - <<: *snuba_defaults - snuba-spans-consumer: - <<: *snuba_defaults - web: - <<: *sentry_defaults - cron: - <<: *sentry_defaults - worker: - <<: *sentry_defaults - events-consumer: - <<: *sentry_defaults - attachments-consumer: - <<: *sentry_defaults - post-process-forwarder-errors: - <<: *sentry_defaults - subscription-consumer-events: - <<: *sentry_defaults - transactions-consumer: - <<: *sentry_defaults - metrics-consumer: - <<: *sentry_defaults - generic-metrics-consumer: - <<: *sentry_defaults - billing-metrics-consumer: - <<: *sentry_defaults - ingest-replay-recordings: - <<: *sentry_defaults - ingest-occurrences: - <<: *sentry_defaults - ingest-profiles: - <<: *sentry_defaults - ingest-monitors: - <<: *sentry_defaults - ingest-feedback-events: - <<: *sentry_defaults - monitors-clock-tick: - <<: *sentry_defaults - monitors-clock-tasks: - <<: *sentry_defaults - post-process-forwarder-transactions: - <<: *sentry_defaults - post-process-forwarder-issue-platform: - <<: *sentry_defaults - subscription-consumer-transactions: - <<: *sentry_defaults - subscription-consumer-metrics: - <<: *sentry_defaults - subscription-consumer-generic-metrics: - <<: *sentry_defaults -EOF -) -if [[ -f "docker-compose.override.yml" ]]; then - echo "🚨 docker-compose.override.yml already exists. You will need to modify it manually with the following content." - echo "Changes: Reset the 'kafka' service block, add additional Kafka-related environment variables to 'vroom', 'relay', 'sentry'-related and 'snuba'-related services." - echo "" - echo "$COMPOSE_OVERRIDE_CONTENT" -else - echo "🚨 docker-compose.override.yml does not exist. Creating it now." - echo "$COMPOSE_OVERRIDE_CONTENT" >docker-compose.override.yml -fi - -echo "" -echo "" -echo "------------------------------------------------------------------------" -echo "- Finished patching external-kafka.sh. Some things you'll need to do:" -echo "- 1. Modify the Kafka credentials on your $ENV_FILE file." -echo "- 2. Modify the Kafka credentials on your $RELAY_CONFIG_YML file." -echo "- 3. Run ./install.sh" -echo "-" -echo "- NOTE: Remove or comment the corresponding line if you don't use it." -echo "- It is also safe to prune/delete 'sentry-kafka'" -echo "- and 'sentry-kafka-log' Docker volumes." -echo "------------------------------------------------------------------------" From 90f0f671b9347211fffa884e53db66533f6690ad Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 15 May 2025 11:13:54 +0700 Subject: [PATCH 08/12] chore(pre-commit): exclude .patch extension --- .pre-commit-config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d995372865..2e8781674ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: args: [-w, -d] files: .*\.sh stages: [commit, merge-commit, push, manual] - + exclude: "\.patch$" - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: @@ -22,3 +22,4 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-yaml + exclude: "\.patch$" From 9d87804e480e527a80d5bce2b0d682ccd001b3b5 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 15 May 2025 11:15:14 +0700 Subject: [PATCH 09/12] chore(pre-commit): escape backslash --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2e8781674ab..04c029c856c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: args: [-w, -d] files: .*\.sh stages: [commit, merge-commit, push, manual] - exclude: "\.patch$" + exclude: '\\.patch$' - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: @@ -22,4 +22,4 @@ repos: - id: end-of-file-fixer - id: trailing-whitespace - id: check-yaml - exclude: "\.patch$" + exclude: '\\.patch$' From c4a358abb19c803873646f4b20608b63228c6942 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 15 May 2025 12:24:30 +0700 Subject: [PATCH 10/12] chore(pre-commit): put exclude field on hooks --- .pre-commit-config.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04c029c856c..a9f2bbbbeb9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,6 @@ repos: args: [-w, -d] files: .*\.sh stages: [commit, merge-commit, push, manual] - exclude: '\\.patch$' - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: @@ -21,5 +20,5 @@ repos: - id: check-symlinks - id: end-of-file-fixer - id: trailing-whitespace + exclude: '\\.patch$' - id: check-yaml - exclude: '\\.patch$' From d17430fd7cb2dfd56636ceff521a6b4bef38f925 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Wed, 4 Jun 2025 07:54:27 +0700 Subject: [PATCH 11/12] chore(pre-commit): put exclude field on top level Based on https://pre-commit.com/#top_level-exclude --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a9f2bbbbeb9..fa127361176 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: files: .*\.sh stages: [commit, merge-commit, push, manual] - repo: https://github.com/pre-commit/pre-commit-hooks + exclude: '\\.patch$' rev: v4.3.0 hooks: - id: check-case-conflict @@ -20,5 +21,4 @@ repos: - id: check-symlinks - id: end-of-file-fixer - id: trailing-whitespace - exclude: '\\.patch$' - id: check-yaml From 87fe898592d87ab0de61174aa3e1660bad243944 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Wed, 4 Jun 2025 08:00:36 +0700 Subject: [PATCH 12/12] chore(pre-commit): move to even more top level --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fa127361176..9d544410bab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: '\.patch$' repos: - repo: local hooks: @@ -12,7 +13,6 @@ repos: files: .*\.sh stages: [commit, merge-commit, push, manual] - repo: https://github.com/pre-commit/pre-commit-hooks - exclude: '\\.patch$' rev: v4.3.0 hooks: - id: check-case-conflict