From 2c50ba52aadcf92eaa9fa8714e3b219a9b629148 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 11:06:28 +0200 Subject: [PATCH 01/14] feat: add snuba healthcheck --- .env | 5 ++++ docker-compose.yml | 72 ++++++++++++++++++++++++++++------------------ 2 files changed, 49 insertions(+), 28 deletions(-) diff --git a/.env b/.env index ee11f6bd67..2fe1796def 100644 --- a/.env +++ b/.env @@ -19,6 +19,11 @@ UPTIME_CHECKER_IMAGE=ghcr.io/getsentry/uptime-checker:nightly HEALTHCHECK_INTERVAL=30s HEALTHCHECK_TIMEOUT=1m30s HEALTHCHECK_RETRIES=10 +HEATLHCHECK_START_PERIOD=10s +HEALTHCHECK_FILE_INTERVAL=320s +HEALTHCHECK_FILE_TIMEOUT=10s +HEALTHCHECK_FILE_RETRIES=1 +HEALTHCHECK_FILE_START_PERIOD=60s # Caution: Raising max connections of postgres increases CPU and RAM usage # see https://github.com/getsentry/self-hosted/pull/2740 for more information POSTGRES_MAX_CONNECTIONS=100 diff --git a/docker-compose.yml b/docker-compose.yml index f29a9bfd56..ed1eb37daa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,13 @@ x-healthcheck-defaults: &healthcheck_defaults interval: "$HEALTHCHECK_INTERVAL" timeout: "$HEALTHCHECK_TIMEOUT" retries: $HEALTHCHECK_RETRIES - start_period: 10s + start_period: "$HEATLHCHECK_START_PERIOD" +x-file-healthcheck: &file_healthcheck_defaults + test: ["CMD-SHELL", "rm /tmp/health.txt"] + interval: "$HEALTHCHECK_FILE_INTERVAL" + timeout: "$HEALTHCHECK_FILE_TIMEOUT" + retries: $HEALTHCHECK_FILE_RETRIES + start_period: "$HEALTHCHECK_FILE_START_PERIOD" x-sentry-defaults: &sentry_defaults <<: [*restart_policy, *pull_policy] image: sentry-self-hosted-local @@ -76,6 +82,8 @@ x-snuba-defaults: &snuba_defaults redis: <<: *depends_on-healthy image: "$SNUBA_IMAGE" + healthcheck: + <<: *file_healthcheck_defaults environment: SNUBA_SETTINGS: self_hosted CLICKHOUSE_HOST: clickhouse @@ -211,133 +219,141 @@ services: retries: 30 snuba-api: <<: *snuba_defaults + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - "-c" + # Courtesy of https://unix.stackexchange.com/a/234089/108960 + - 'exec 3<>/dev/tcp/127.0.0.1/1218 && echo -e "GET /health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' # Kafka consumer responsible for feeding events into Clickhouse snuba-errors-consumer: <<: *snuba_defaults - command: rust-consumer --storage errors --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage errors --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults - command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt snuba-outcomes-billing-consumer: <<: *snuba_defaults - command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing + command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing --health-check-file /tmp/health.txt snuba-group-attributes-consumer: <<: *snuba_defaults - command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt snuba-replacer: <<: *snuba_defaults - command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset + command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset --health-check-file /tmp/health.txt snuba-subscription-consumer-events: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt ############################################# ## Feature Complete Sentry Snuba Consumers ## ############################################# # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: <<: *snuba_defaults - command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-replays-consumer: <<: *snuba_defaults - command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-issue-occurrence-consumer: <<: *snuba_defaults - command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-metrics-consumer: <<: *snuba_defaults - command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-transactions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-metrics: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-generic-metrics-distributions: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-generic-metrics-sets: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-generic-metrics-counters: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-generic-metrics-gauges: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt profiles: - feature-complete snuba-generic-metrics-distributions-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-generic-metrics-sets-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-generic-metrics-counters-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-generic-metrics-gauges-consumer: <<: *snuba_defaults - command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-profiling-profiles-consumer: <<: *snuba_defaults - command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-profiling-functions-consumer: <<: *snuba_defaults - command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-profiling-profile-chunks-consumer: <<: *snuba_defaults - command: rust-consumer --storage profile_chunks --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + command: rust-consumer --storage profile_chunks --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-spans-consumer: <<: *snuba_defaults - command: rust-consumer --storage spans --consumer-group snuba-spans-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset + command: rust-consumer --storage spans --consumer-group snuba-spans-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete snuba-eap-items-consumer: <<: *snuba_defaults - command: rust-consumer --storage eap_items --consumer-group eap_items_group --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --use-rust-processor + command: rust-consumer --storage eap_items --consumer-group eap_items_group --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --use-rust-processor --health-check-file /tmp/health.txt profiles: - feature-complete snuba-subscription-consumer-eap-items: <<: *snuba_defaults - command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 + command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt snuba-uptime-results-consumer: <<: *snuba_defaults - command: rust-consumer --storage uptime_monitor_checks --consumer-group snuba-uptime-results --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset + command: rust-consumer --storage uptime_monitor_checks --consumer-group snuba-uptime-results --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt profiles: - feature-complete symbolicator: From 4ccd82f8c4c74681781ee769d76a6a18ee8f2925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 11:53:08 +0200 Subject: [PATCH 02/14] fix: snuba healthchecks --- docker-compose.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ed1eb37daa..257a56dbd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -226,7 +226,7 @@ services: - "/bin/bash" - "-c" # Courtesy of https://unix.stackexchange.com/a/234089/108960 - - 'exec 3<>/dev/tcp/127.0.0.1/1218 && echo -e "GET /health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' + - 'exec 3<>/dev/tcp/127.0.0.1/1218 && echo -e "GET /health HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' # Kafka consumer responsible for feeding events into Clickhouse snuba-errors-consumer: <<: *snuba_defaults @@ -245,6 +245,8 @@ services: command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt snuba-replacer: <<: *snuba_defaults + healthcheck: + disable: true command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset --health-check-file /tmp/health.txt snuba-subscription-consumer-events: <<: *snuba_defaults From ecb3757103d2e329d96d87c47779295efb6673a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 11:56:11 +0200 Subject: [PATCH 03/14] fix: remove healthcheck from replacer --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 257a56dbd8..15d2d522c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -247,7 +247,7 @@ services: <<: *snuba_defaults healthcheck: disable: true - command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset --health-check-file /tmp/health.txt + command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt From cbb71fa3bfbf2e7a570a0d0dbbacf7d3caae05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 12:06:49 +0200 Subject: [PATCH 04/14] feat: add healthchecks to sentry --- docker-compose.yml | 52 +++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 15d2d522c2..3d15408fa6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,8 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] + healthcheck: + <<: *file_healthcheck_defaults environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" @@ -400,117 +402,117 @@ services: command: run worker events-consumer: <<: *sentry_defaults - command: run consumer ingest-events --consumer-group ingest-consumer + command: run consumer ingest-events --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt attachments-consumer: <<: *sentry_defaults - command: run consumer ingest-attachments --consumer-group ingest-consumer + command: run consumer ingest-attachments --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt post-process-forwarder-errors: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers + command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers --healthcheck-file-path /tmp/health.txt subscription-consumer-events: <<: *sentry_defaults - command: run consumer events-subscription-results --consumer-group query-subscription-consumer + command: run consumer events-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt ############################################## ## Feature Complete Sentry Ingest Consumers ## ############################################## transactions-consumer: <<: *sentry_defaults - command: run consumer ingest-transactions --consumer-group ingest-consumer + command: run consumer ingest-transactions --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete metrics-consumer: <<: *sentry_defaults - command: run consumer ingest-metrics --consumer-group metrics-consumer + command: run consumer ingest-metrics --consumer-group metrics-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete generic-metrics-consumer: <<: *sentry_defaults - command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer + command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete billing-metrics-consumer: <<: *sentry_defaults - command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer + command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete ingest-replay-recordings: <<: *sentry_defaults - command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings + command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings --healthcheck-file-path /tmp/health.txt profiles: - feature-complete ingest-occurrences: <<: *sentry_defaults - command: run consumer ingest-occurrences --consumer-group ingest-occurrences + command: run consumer ingest-occurrences --consumer-group ingest-occurrences --healthcheck-file-path /tmp/health.txt profiles: - feature-complete ingest-profiles: <<: *sentry_defaults - command: run consumer ingest-profiles --consumer-group ingest-profiles + command: run consumer ingest-profiles --consumer-group ingest-profiles --healthcheck-file-path /tmp/health.txt profiles: - feature-complete ingest-monitors: <<: *sentry_defaults - command: run consumer ingest-monitors --consumer-group ingest-monitors + command: run consumer ingest-monitors --consumer-group ingest-monitors --healthcheck-file-path /tmp/health.txt profiles: - feature-complete ingest-feedback-events: <<: *sentry_defaults - command: run consumer ingest-feedback-events --consumer-group ingest-feedback + command: run consumer ingest-feedback-events --consumer-group ingest-feedback --healthcheck-file-path /tmp/health.txt profiles: - feature-complete process-spans: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset process-spans --consumer-group process-spans + command: run consumer --no-strict-offset-reset process-spans --consumer-group process-spans --healthcheck-file-path /tmp/health.txt profiles: - feature-complete process-segments: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset process-segments --consumer-group process-segments + command: run consumer --no-strict-offset-reset process-segments --consumer-group process-segments --healthcheck-file-path /tmp/health.txt profiles: - feature-complete monitors-clock-tick: <<: *sentry_defaults - command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick + command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick --healthcheck-file-path /tmp/health.txt profiles: - feature-complete monitors-clock-tasks: <<: *sentry_defaults - command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks + command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks --healthcheck-file-path /tmp/health.txt profiles: - feature-complete uptime-results: <<: *sentry_defaults - command: run consumer uptime-results --consumer-group uptime-results + command: run consumer uptime-results --consumer-group uptime-results --healthcheck-file-path /tmp/health.txt profiles: - feature-complete post-process-forwarder-transactions: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group + command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group --healthcheck-file-path /tmp/health.txt profiles: - feature-complete post-process-forwarder-issue-platform: <<: *sentry_defaults - command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group + command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group --healthcheck-file-path /tmp/health.txt profiles: - feature-complete subscription-consumer-transactions: <<: *sentry_defaults - command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer + command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete subscription-consumer-eap-items: <<: *sentry_defaults - command: run consumer subscription-results-eap-items --consumer-group subscription-results-eap-items + command: run consumer subscription-results-eap-items --consumer-group subscription-results-eap-items --healthcheck-file-path /tmp/health.txt profiles: - feature-complete subscription-consumer-metrics: <<: *sentry_defaults - command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer + command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete subscription-consumer-generic-metrics: <<: *sentry_defaults - command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer + command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt profiles: - feature-complete sentry-cleanup: @@ -521,6 +523,8 @@ services: args: BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" + healthcheck: + disable: true command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy From ed2efbb9c5296237cc02f87181cf6e59aed33741 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 12:21:43 +0200 Subject: [PATCH 05/14] fix: small changes to sentry healthchecks --- .env | 2 +- docker-compose.yml | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 2fe1796def..c6d2b90416 100644 --- a/.env +++ b/.env @@ -20,7 +20,7 @@ HEALTHCHECK_INTERVAL=30s HEALTHCHECK_TIMEOUT=1m30s HEALTHCHECK_RETRIES=10 HEATLHCHECK_START_PERIOD=10s -HEALTHCHECK_FILE_INTERVAL=320s +HEALTHCHECK_FILE_INTERVAL=60s HEALTHCHECK_FILE_TIMEOUT=10s HEALTHCHECK_FILE_RETRIES=1 HEALTHCHECK_FILE_START_PERIOD=60s diff --git a/docker-compose.yml b/docker-compose.yml index 3d15408fa6..dd73639e76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -397,9 +397,18 @@ services: cron: <<: *sentry_defaults command: run cron + healthcheck: + disable: true worker: <<: *sentry_defaults command: run worker + healthcheck: + command: + - CMD + - sentry + - exec + - -c + - 'from sentry.celery import app; import os; dest="celery@{}".format(os.environ["HOSTNAME"]); print(app.control.ping(destination=[dest], timeout=5)[0][dest]["ok"])' events-consumer: <<: *sentry_defaults command: run consumer ingest-events --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt @@ -575,9 +584,13 @@ services: taskscheduler: <<: *sentry_defaults command: run taskworker-scheduler + healthcheck: + disable: true taskworker: <<: *sentry_defaults command: run taskworker --concurrency=4 --rpc-host=taskbroker:50051 + healthcheck: + disable: true vroom: <<: *restart_policy image: "$VROOM_IMAGE" From 32ad250195b0346f8e70908d42007631b490a6ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 12:23:20 +0200 Subject: [PATCH 06/14] fix: worker healthcheck --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index dd73639e76..e74ee70a4e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -403,7 +403,7 @@ services: <<: *sentry_defaults command: run worker healthcheck: - command: + test: - CMD - sentry - exec From 7e9edcb9fab101203f4f1fda69e418e12df9315f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 12:52:21 +0200 Subject: [PATCH 07/14] feat: vroom healthcheck --- .env | 2 +- docker-compose.yml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.env b/.env index c6d2b90416..4c0e1de958 100644 --- a/.env +++ b/.env @@ -23,7 +23,7 @@ HEATLHCHECK_START_PERIOD=10s HEALTHCHECK_FILE_INTERVAL=60s HEALTHCHECK_FILE_TIMEOUT=10s HEALTHCHECK_FILE_RETRIES=1 -HEALTHCHECK_FILE_START_PERIOD=60s +HEALTHCHECK_FILE_START_PERIOD=180s # Caution: Raising max connections of postgres increases CPU and RAM usage # see https://github.com/getsentry/self-hosted/pull/2740 for more information POSTGRES_MAX_CONNECTIONS=100 diff --git a/docker-compose.yml b/docker-compose.yml index e74ee70a4e..b7ec558a14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -403,6 +403,7 @@ services: <<: *sentry_defaults command: run worker healthcheck: + <<: *healthcheck_defaults test: - CMD - sentry @@ -601,6 +602,14 @@ services: SENTRY_SNUBA_HOST: "http://snuba-api:1218" volumes: - sentry-vroom:/var/vroom/sentry-profiles + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - "-c" + # Courtesy of https://unix.stackexchange.com/a/234089/108960 + - 'exec 3<>/dev/tcp/127.0.0.1/8085 && echo -e "GET /health HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep OK -s -m 1 <&3' depends_on: kafka: <<: *depends_on-healthy From 828bb3544ac8ac737f9a34508bb21bd071c69ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 12:56:56 +0200 Subject: [PATCH 08/14] feat: symbolicator healthcheck --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b7ec558a14..e5301c66a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -369,6 +369,14 @@ services: read_only: true source: ./symbolicator target: /etc/symbolicator + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - "-c" + # Courtesy of https://unix.stackexchange.com/a/234089/108960 + - 'exec 3<>/dev/tcp/127.0.0.1/3021 && echo -e "GET /healthcheck HTTP/1.1\r\nhost: 127.0.0.1\r\nConnection: close\r\n\r\n" >&3 && grep OK -s -m 1 <&3' command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: <<: [*restart_policy, *pull_policy] From e1a400aa1a347128fde42c14ad8b57c072629372 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 13:09:51 +0200 Subject: [PATCH 09/14] feat: add nginx healthcheck --- docker-compose.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index e5301c66a6..555e3a81e8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -556,6 +556,12 @@ services: target: /etc/nginx/nginx.conf - sentry-nginx-cache:/var/cache/nginx - sentry-nginx-www:/var/www + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/usr/bin/curl" + - http://localhost depends_on: - web - relay From 6bed0eb796aba0511a8ed966de50c6692f16db4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 17:55:49 +0200 Subject: [PATCH 10/14] fix: typo in .env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 4c0e1de958..6570bf8c28 100644 --- a/.env +++ b/.env @@ -19,7 +19,7 @@ UPTIME_CHECKER_IMAGE=ghcr.io/getsentry/uptime-checker:nightly HEALTHCHECK_INTERVAL=30s HEALTHCHECK_TIMEOUT=1m30s HEALTHCHECK_RETRIES=10 -HEATLHCHECK_START_PERIOD=10s +HEALTHCHECK_START_PERIOD=10s HEALTHCHECK_FILE_INTERVAL=60s HEALTHCHECK_FILE_TIMEOUT=10s HEALTHCHECK_FILE_RETRIES=1 From 80fa06fde71bbc97c4cdd99685efe5e2539a1c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 17:58:58 +0200 Subject: [PATCH 11/14] fix: typo in docker-compose.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 555e3a81e8..619cc8356c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ x-healthcheck-defaults: &healthcheck_defaults interval: "$HEALTHCHECK_INTERVAL" timeout: "$HEALTHCHECK_TIMEOUT" retries: $HEALTHCHECK_RETRIES - start_period: "$HEATLHCHECK_START_PERIOD" + start_period: "$HEALTHCHECK_START_PERIOD" x-file-healthcheck: &file_healthcheck_defaults test: ["CMD-SHELL", "rm /tmp/health.txt"] interval: "$HEALTHCHECK_FILE_INTERVAL" From fc43389793017f8041fc1cdae0a5cb5e2fb759f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 18:12:48 +0200 Subject: [PATCH 12/14] test: increase docker compose --wait-timeout --- _integration-test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_integration-test/conftest.py b/_integration-test/conftest.py index e73a7f286c..6051ff799d 100644 --- a/_integration-test/conftest.py +++ b/_integration-test/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture(scope="session", autouse=True) def configure_self_hosted_environment(request): subprocess.run( - ["docker", "compose", "--ansi", "never", "up", "--wait"], + ["docker", "compose", "--ansi", "never", "up", "--wait", "--wait-timeout", "600"], check=True, capture_output=True, ) From dcb496df0a53fbb5b9d4c1cec017b1ef53a3f7fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 18:52:36 +0200 Subject: [PATCH 13/14] Revert "test: increase docker compose --wait-timeout" This reverts commit fc43389793017f8041fc1cdae0a5cb5e2fb759f2. --- _integration-test/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_integration-test/conftest.py b/_integration-test/conftest.py index 6051ff799d..e73a7f286c 100644 --- a/_integration-test/conftest.py +++ b/_integration-test/conftest.py @@ -13,7 +13,7 @@ @pytest.fixture(scope="session", autouse=True) def configure_self_hosted_environment(request): subprocess.run( - ["docker", "compose", "--ansi", "never", "up", "--wait", "--wait-timeout", "600"], + ["docker", "compose", "--ansi", "never", "up", "--wait"], check=True, capture_output=True, ) From 14eff39038e0057f69e6f7798a94cd4e2ce40242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Zgli=C5=84ski?= Date: Thu, 7 Aug 2025 18:59:45 +0200 Subject: [PATCH 14/14] fix: do not use healthcheck: [disabled: true], since it breaks docker compose wait command --- docker-compose.yml | 114 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 100 insertions(+), 14 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 619cc8356c..437bdc7694 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,8 +48,6 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] - healthcheck: - <<: *file_healthcheck_defaults environment: PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" @@ -84,8 +82,6 @@ x-snuba-defaults: &snuba_defaults redis: <<: *depends_on-healthy image: "$SNUBA_IMAGE" - healthcheck: - <<: *file_healthcheck_defaults environment: SNUBA_SETTINGS: self_hosted CLICKHOUSE_HOST: clickhouse @@ -233,26 +229,34 @@ services: snuba-errors-consumer: <<: *snuba_defaults command: rust-consumer --storage errors --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: <<: *snuba_defaults command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults snuba-outcomes-billing-consumer: <<: *snuba_defaults command: rust-consumer --storage outcomes_raw --consumer-group snuba-consumers --auto-offset-reset=earliest --max-batch-time-ms 750 --no-strict-offset-reset --raw-events-topic outcomes-billing --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults snuba-group-attributes-consumer: <<: *snuba_defaults command: rust-consumer --storage group_attributes --consumer-group snuba-group-attributes-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults snuba-replacer: <<: *snuba_defaults - healthcheck: - disable: true command: replacer --storage errors --auto-offset-reset=latest --no-strict-offset-reset snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset events --entity events --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-events-subscriptions-consumers --followed-consumer-group=snuba-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults ############################################# ## Feature Complete Sentry Snuba Consumers ## ############################################# @@ -260,104 +264,146 @@ services: snuba-transactions-consumer: <<: *snuba_defaults command: rust-consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-replays-consumer: <<: *snuba_defaults command: rust-consumer --storage replays --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-issue-occurrence-consumer: <<: *snuba_defaults command: rust-consumer --storage search_issues --consumer-group generic_events_group --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-metrics-consumer: <<: *snuba_defaults command: rust-consumer --storage metrics_raw --consumer-group snuba-metrics-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-transactions: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-metrics: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset metrics --entity metrics_sets --entity metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-metrics-subscriptions-consumers --followed-consumer-group=snuba-metrics-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-distributions: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_distributions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-distributions-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-distributions-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-sets: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_sets --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-sets-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-sets-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-counters: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_counters --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-counters-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-counters-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-generic-metrics-gauges: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset generic_metrics --entity=generic_metrics_gauges --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers --followed-consumer-group=snuba-gen-metrics-gauges-consumers --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-distributions-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_distributions_raw --consumer-group snuba-gen-metrics-distributions-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-sets-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_sets_raw --consumer-group snuba-gen-metrics-sets-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-counters-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_counters_raw --consumer-group snuba-gen-metrics-counters-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-generic-metrics-gauges-consumer: <<: *snuba_defaults command: rust-consumer --storage generic_metrics_gauges_raw --consumer-group snuba-gen-metrics-gauges-consumers --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-profiles-consumer: <<: *snuba_defaults command: rust-consumer --storage profiles --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-functions-consumer: <<: *snuba_defaults command: rust-consumer --storage functions_raw --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-profiling-profile-chunks-consumer: <<: *snuba_defaults command: rust-consumer --storage profile_chunks --consumer-group snuba-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-spans-consumer: <<: *snuba_defaults command: rust-consumer --storage spans --consumer-group snuba-spans-consumers --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-eap-items-consumer: <<: *snuba_defaults command: rust-consumer --storage eap_items --consumer-group eap_items_group --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset --use-rust-processor --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete snuba-subscription-consumer-eap-items: <<: *snuba_defaults command: subscriptions-scheduler-executor --dataset events_analytics_platform --entity eap_items --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-eap-items-subscriptions-consumers --followed-consumer-group=eap_items_group --schedule-ttl=60 --stale-threshold-seconds=900 --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults snuba-uptime-results-consumer: <<: *snuba_defaults command: rust-consumer --storage uptime_monitor_checks --consumer-group snuba-uptime-results --auto-offset-reset=latest --max-batch-time-ms 750 --no-strict-offset-reset --health-check-file /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete symbolicator: @@ -405,8 +451,6 @@ services: cron: <<: *sentry_defaults command: run cron - healthcheck: - disable: true worker: <<: *sentry_defaults command: run worker @@ -421,116 +465,164 @@ services: events-consumer: <<: *sentry_defaults command: run consumer ingest-events --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults attachments-consumer: <<: *sentry_defaults command: run consumer ingest-attachments --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults post-process-forwarder-errors: <<: *sentry_defaults command: run consumer --no-strict-offset-reset post-process-forwarder-errors --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-commit-log --synchronize-commit-group=snuba-consumers --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults subscription-consumer-events: <<: *sentry_defaults command: run consumer events-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults ############################################## ## Feature Complete Sentry Ingest Consumers ## ############################################## transactions-consumer: <<: *sentry_defaults command: run consumer ingest-transactions --consumer-group ingest-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete metrics-consumer: <<: *sentry_defaults command: run consumer ingest-metrics --consumer-group metrics-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete generic-metrics-consumer: <<: *sentry_defaults command: run consumer ingest-generic-metrics --consumer-group generic-metrics-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete billing-metrics-consumer: <<: *sentry_defaults command: run consumer billing-metrics-consumer --consumer-group billing-metrics-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete ingest-replay-recordings: <<: *sentry_defaults command: run consumer ingest-replay-recordings --consumer-group ingest-replay-recordings --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete ingest-occurrences: <<: *sentry_defaults command: run consumer ingest-occurrences --consumer-group ingest-occurrences --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete ingest-profiles: <<: *sentry_defaults command: run consumer ingest-profiles --consumer-group ingest-profiles --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete ingest-monitors: <<: *sentry_defaults command: run consumer ingest-monitors --consumer-group ingest-monitors --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete ingest-feedback-events: <<: *sentry_defaults command: run consumer ingest-feedback-events --consumer-group ingest-feedback --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete process-spans: <<: *sentry_defaults command: run consumer --no-strict-offset-reset process-spans --consumer-group process-spans --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete process-segments: <<: *sentry_defaults command: run consumer --no-strict-offset-reset process-segments --consumer-group process-segments --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete monitors-clock-tick: <<: *sentry_defaults command: run consumer monitors-clock-tick --consumer-group monitors-clock-tick --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete monitors-clock-tasks: <<: *sentry_defaults command: run consumer monitors-clock-tasks --consumer-group monitors-clock-tasks --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete uptime-results: <<: *sentry_defaults command: run consumer uptime-results --consumer-group uptime-results --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete post-process-forwarder-transactions: <<: *sentry_defaults command: run consumer --no-strict-offset-reset post-process-forwarder-transactions --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete post-process-forwarder-issue-platform: <<: *sentry_defaults command: run consumer --no-strict-offset-reset post-process-forwarder-issue-platform --consumer-group post-process-forwarder --synchronize-commit-log-topic=snuba-generic-events-commit-log --synchronize-commit-group generic_events_group --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-transactions: <<: *sentry_defaults command: run consumer transactions-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-eap-items: <<: *sentry_defaults command: run consumer subscription-results-eap-items --consumer-group subscription-results-eap-items --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-metrics: <<: *sentry_defaults command: run consumer metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete subscription-consumer-generic-metrics: <<: *sentry_defaults command: run consumer generic-metrics-subscription-results --consumer-group query-subscription-consumer --healthcheck-file-path /tmp/health.txt + healthcheck: + <<: *file_healthcheck_defaults profiles: - feature-complete sentry-cleanup: @@ -541,8 +633,6 @@ services: args: BASE_IMAGE: sentry-self-hosted-local entrypoint: "/entrypoint.sh" - healthcheck: - disable: true command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy @@ -599,13 +689,9 @@ services: taskscheduler: <<: *sentry_defaults command: run taskworker-scheduler - healthcheck: - disable: true taskworker: <<: *sentry_defaults command: run taskworker --concurrency=4 --rpc-host=taskbroker:50051 - healthcheck: - disable: true vroom: <<: *restart_policy image: "$VROOM_IMAGE"