diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d995372865..9d544410bab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +exclude: '\.patch$' repos: - repo: local hooks: @@ -11,7 +12,6 @@ repos: args: [-w, -d] files: .*\.sh stages: [commit, merge-commit, push, manual] - - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: diff --git a/optional-modifications/README.md b/optional-modifications/README.md new file mode 100644 index 00000000000..84cf7e0e2b9 --- /dev/null +++ b/optional-modifications/README.md @@ -0,0 +1,41 @@ +# Optional Modifications + +Other than the default self-hosted Sentry installation, sometimes users +can leverage their existing infrastructure to help them with limited +resources. "Patches", or you might call this like a "plugin system", is +a collection of patch files (see [man patch(1)](https://man7.org/linux/man-pages/man1/patch.1.html)) +that can be used with to modify the existing configuration to achieve +the desired goal. + +> [!WARNING] +> Beware that this is very experimental and might not work as expected. +> +> **Use it at your own risk!** + +## How to use patches + +The patches are designed mostly to help modify the existing +configuration files. You will need to run the `install.sh` script +afterwards. + +They should be run from the root directory. For example, the +`external-kafka` patches should be run as: + +```bash +patch < optional-modifications/patches/external-kafka/.env.patch +patch < optional-modifications/patches/external-kafka/config.example.yml.patch +patch < optional-modifications/patches/external-kafka/sentry.conf.example.py.patch +patch < optional-modifications/patches/external-kafka/docker-compose.yml.patch +``` + +Some patches might require additional steps to be taken, like providing +credentials or additional TLS certificates. + +## Official support + +Sentry employees are not obliged to provide dedicated support for +patches, but they can help by providing information to move us forward. +We encourage the community to contribute for any bug fixes or +improvements. + +See the [support policy for self-hosted Sentry](https://develop.sentry.dev/self-hosted/support/) for more information. diff --git a/optional-modifications/_lib.sh b/optional-modifications/_lib.sh new file mode 100755 index 00000000000..46e55d3e257 --- /dev/null +++ b/optional-modifications/_lib.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +set -euo pipefail +test "${DEBUG:-}" && set -x + +function patch_file() { + target="$1" + content="$2" + if [[ -f "$target" ]]; then + echo "🙈 Patching $target ..." + patch -p1 <"$content" + else + echo "🙊 Skipping $target ..." + fi +} diff --git a/optional-modifications/patches/external-kafka/.env.patch b/optional-modifications/patches/external-kafka/.env.patch new file mode 100644 index 00000000000..36e45f7785f --- /dev/null +++ b/optional-modifications/patches/external-kafka/.env.patch @@ -0,0 +1,22 @@ +--- .env 2025-02-04 07:31:54.868049984 +0700 ++++ .env.external-kafka 2025-05-15 08:33:15.442361105 +0700 +@@ -22,3 +22,19 @@ + POSTGRES_MAX_CONNECTIONS=100 + # Set SETUP_JS_SDK_ASSETS to 1 to enable the setup of JS SDK assets + # SETUP_JS_SDK_ASSETS=1 ++ ++################################################################################ ++## Additional External Kafka options ++################################################################################ ++KAFKA_BOOTSTRAP_SERVERS=kafka-node1:9092,kafka-node2:9092,kafka-node3:9092 ++# Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++KAFKA_SECURITY_PROTOCOL=PLAINTEXT ++# Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++# KAFKA_SASL_MECHANISM=PLAIN ++# KAFKA_SASL_USERNAME=username ++# KAFKA_SASL_PASSWORD=password ++# Put your certificates on the \`certificates/kafka\` directory. ++# The certificates will be mounted as read-only volumes. ++# KAFKA_SSL_CA_LOCATION=/kafka-certificates/ca.pem ++# KAFKA_SSL_CERTIFICATE_LOCATION=/kafka-certificates/client.pem ++# KAFKA_SSL_KEY_LOCATION=/kafka-certificates/client.key diff --git a/optional-modifications/patches/external-kafka/config.example.yml.patch b/optional-modifications/patches/external-kafka/config.example.yml.patch new file mode 100644 index 00000000000..a0c1aab04b8 --- /dev/null +++ b/optional-modifications/patches/external-kafka/config.example.yml.patch @@ -0,0 +1,19 @@ +--- relay/config.example.yml 2025-05-15 08:27:40.426876887 +0700 ++++ relay/config.example.external-kafka.yml 2025-05-15 08:34:21.113311217 +0700 +@@ -7,8 +7,15 @@ + processing: + enabled: true + kafka_config: +- - {name: "bootstrap.servers", value: "kafka:9092"} ++ - {name: "bootstrap.servers", value: "kafka-node1:9092,kafka-node2:9092,kafka-node3:9092"} + - {name: "message.max.bytes", value: 50000000} # 50MB ++ - {name: "security.protocol", value: "PLAINTEXT"} ++ - {name: "sasl.mechanism", value: "PLAIN"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.username", value: "username"} # Remove or comment this line if SASL is not used. ++ - {name: "sasl.password", value: "password"} # Remove or comment this line if SASL is not used. ++ - {name: "ssl.ca.location", value: "/kafka-certificates/ca.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.certificate.location", value: "/kafka-certificates/client.pem"} # Remove or comment this line if SSL is not used. ++ - {name: "ssl.key.location", value: "/kafka-certificates/client.key"} # Remove or comment this line if SSL is not used. + redis: redis://redis:6379 + geoip_path: "/geoip/GeoLite2-City.mmdb" + diff --git a/optional-modifications/patches/external-kafka/docker-compose.yml.patch b/optional-modifications/patches/external-kafka/docker-compose.yml.patch new file mode 100644 index 00000000000..ad0328410b7 --- /dev/null +++ b/optional-modifications/patches/external-kafka/docker-compose.yml.patch @@ -0,0 +1,142 @@ +--- docker-compose.yml 2025-03-17 13:32:15.120328412 +0700 ++++ docker-compose.external-kafka.yml 2025-05-15 08:39:05.509951068 +0700 +@@ -26,8 +26,6 @@ + depends_on: + redis: + <<: *depends_on-healthy +- kafka: +- <<: *depends_on-healthy + postgres: + <<: *depends_on-healthy + memcached: +@@ -59,6 +57,14 @@ + SENTRY_EVENT_RETENTION_DAYS: + SENTRY_MAIL_HOST: + SENTRY_MAX_EXTERNAL_SOURCEMAP_SIZE: ++ KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ KAFKA_SSL_CA_LOCATION: ${KAFKA_SSL_CA_LOCATION:-} ++ KAFKA_SSL_CERTIFICATE_LOCATION: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ KAFKA_SSL_KEY_LOCATION: ${KAFKA_SSL_KEY_LOCATION:-} ++ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + volumes: + - "sentry-data:/data" + - "./sentry:/etc/sentry" +@@ -69,15 +75,20 @@ + depends_on: + clickhouse: + <<: *depends_on-healthy +- kafka: +- <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + image: "$SNUBA_IMAGE" + environment: + SNUBA_SETTINGS: self_hosted + CLICKHOUSE_HOST: clickhouse +- DEFAULT_BROKERS: "kafka:9092" ++ DEFAULT_BROKERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-} ++ KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-} ++ KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + REDIS_HOST: redis + UWSGI_MAX_REQUESTS: "10000" + UWSGI_DISABLE_LOGGING: "true" +@@ -140,43 +151,7 @@ + POSTGRES_HOST_AUTH_METHOD: "trust" + volumes: + - "sentry-postgres:/var/lib/postgresql/data" +- kafka: +- <<: *restart_policy +- image: "confluentinc/cp-kafka:7.6.1" +- environment: +- # https://docs.confluent.io/platform/current/installation/docker/config-reference.html#cp-kakfa-example +- KAFKA_PROCESS_ROLES: "broker,controller" +- KAFKA_CONTROLLER_QUORUM_VOTERS: "1001@127.0.0.1:29093" +- KAFKA_CONTROLLER_LISTENER_NAMES: "CONTROLLER" +- KAFKA_NODE_ID: "1001" +- CLUSTER_ID: "MkU3OEVBNTcwNTJENDM2Qk" +- KAFKA_LISTENERS: "PLAINTEXT://0.0.0.0:29092,INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29093" +- KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://127.0.0.1:29092,INTERNAL://kafka:9093,EXTERNAL://kafka:9092" +- KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: "PLAINTEXT:PLAINTEXT,INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT" +- KAFKA_INTER_BROKER_LISTENER_NAME: "PLAINTEXT" +- KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" +- KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" +- KAFKA_LOG_RETENTION_HOURS: "24" +- KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust +- KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too +- CONFLUENT_SUPPORT_METRICS_ENABLE: "false" +- KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,state.change.logger=WARN" +- KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" +- KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" +- ulimits: +- nofile: +- soft: 4096 +- hard: 4096 +- volumes: +- - "sentry-kafka:/var/lib/kafka/data" +- - "sentry-kafka-log:/var/lib/kafka/log" +- - "sentry-secrets:/etc/kafka/secrets" +- healthcheck: +- <<: *healthcheck_defaults +- test: ["CMD-SHELL", "nc -z localhost 9092"] +- interval: 10s +- timeout: 10s +- retries: 30 ++ kafka: !reset null + clickhouse: + <<: *restart_policy + image: clickhouse-self-hosted-local +@@ -475,9 +450,8 @@ + read_only: true + source: ./geoip + target: /geoip ++ - ./certificates/kafka:/kafka-certificates:ro + depends_on: +- kafka: +- <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + web: +@@ -486,15 +460,21 @@ + <<: *restart_policy + image: "$VROOM_IMAGE" + environment: +- SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092" +- SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092" ++ SENTRY_KAFKA_BROKERS_PROFILING: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_BROKERS_OCCURRENCES: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_BROKERS_SPANS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092} ++ SENTRY_KAFKA_SECURITY_PROTOCOL: ${KAFKA_SECURITY_PROTOCOL:-PLAINTEXT} ++ SENTRY_KAFKA_SSL_CA_PATH: ${KAFKA_SSL_CA_LOCATION:-} ++ SENTRY_KAFKA_SSL_CERT_PATH: ${KAFKA_SSL_CERTIFICATE_LOCATION:-} ++ SENTRY_KAFKA_SSL_KEY_PATH: ${KAFKA_SSL_KEY_LOCATION:-} ++ SENTRY_KAFKA_SASL_MECHANISM: ${KAFKA_SASL_MECHANISM:-} ++ SENTRY_KAFKA_SASL_USERNAME: ${KAFKA_SASL_USERNAME:-} ++ SENTRY_KAFKA_SASL_PASSWORD: ${KAFKA_SASL_PASSWORD:-} + SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles + SENTRY_SNUBA_HOST: "http://snuba-api:1218" + volumes: + - sentry-vroom:/var/lib/sentry-profiles +- depends_on: +- kafka: +- <<: *depends_on-healthy ++ - ./certificates/kafka:/kafka-certificates:ro + profiles: + - feature-complete + vroom-cleanup: +@@ -523,8 +503,6 @@ + external: true + sentry-redis: + external: true +- sentry-kafka: +- external: true + sentry-clickhouse: + external: true + sentry-symbolicator: diff --git a/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch b/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch new file mode 100644 index 00000000000..abc755c00a0 --- /dev/null +++ b/optional-modifications/patches/external-kafka/sentry.conf.example.py.patch @@ -0,0 +1,21 @@ +--- sentry/sentry.conf.example.py 2025-05-15 08:27:40.427876868 +0700 ++++ sentry/sentry.conf.example.external-kafka.py 2025-05-15 08:32:44.845127931 +0700 +@@ -132,9 +132,17 @@ + SENTRY_CACHE = "sentry.cache.redis.RedisCache" + + DEFAULT_KAFKA_OPTIONS = { +- "bootstrap.servers": "kafka:9092", ++ "bootstrap.servers": env("KAFKA_BOOTSTRAP_SERVERS", "kafka:9092"), + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, ++ "security.protocol": env("KAFKA_SECURITY_PROTOCOL", "PLAINTEXT"), # Valid options are PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL ++ # If you don't use any of these options below, you can remove them or set them to `None`. ++ "sasl.mechanism": env("KAFKA_SASL_MECHANISM", None), # Valid options are PLAIN, SCRAM-SHA-256, SCRAM-SHA-512. Other mechanism might be unavailable. ++ "sasl.username": env("KAFKA_SASL_USERNAME", None), ++ "sasl.password": env("KAFKA_SASL_PASSWORD", None), ++ "ssl.ca.location": env("KAFKA_SSL_CA_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.certificate.location": env("KAFKA_SSL_CERTIFICATE_LOCATION", None), # Remove this line if SSL is not used. ++ "ssl.key.location": env("KAFKA_SSL_KEY_LOCATION", None), # Remove this line if SSL is not used. + } + + SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream"