feat: Add taskbroker + worker + scheduler (#3738)

Author: markstory

.env

@@ -13,6 +13,7 @@ SENTRY_IMAGE=getsentry/sentry:nightly
 SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
+TASKBROKER_IMAGE=getsentry/taskbroker:nightly
 VROOM_IMAGE=getsentry/vroom:nightly
 HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=1m30s

.github/ISSUE_TEMPLATE/release.yml

@@ -15,6 +15,7 @@ body:
           - [ ] [`snuba`](https://github.com/getsentry/snuba/actions/workflows/release.yml)
           - [ ] [`symbolicator`](https://github.com/getsentry/symbolicator/actions/workflows/release.yml)
           - [ ] [`vroom`](https://github.com/getsentry/vroom/actions/workflows/release.yaml)
+          - [ ] [`taskbroker`](https://github.com/getsentry/taskbroker/actions/workflows/release.yml)
         - [ ] Release self-hosted.
           - [ ] [Prepare the `self-hosted` release](https://github.com/getsentry/self-hosted/actions/workflows/release.yml) (_replace with publish issue repo link_).
           - [ ] Check to make sure the new release branch in self-hosted includes the appropriate CalVer images.

docker-compose.yml

@@ -487,6 +487,24 @@ services:
         <<: *depends_on-healthy
       web:
         <<: *depends_on-healthy
+  taskbroker:
+    <<: *restart_policy
+    image: "$TASKBROKER_IMAGE"
+    environment:
+      TASKBROKER_KAFKA_CLUSTER: "kafka:9092"
+      TASKBROKER_KAFKA_DEADLETTER_CLUSTER: "kafka:9092"
+      TASKBROKER_DB_PATH: "/opt/sqlite/taskbroker-activations.sqlite"
+    volumes:
+      - sentry-taskbroker:/opt/sqlite
+    depends_on:
+      kafka:
+        <<: *depends_on-healthy
+  taskscheduler:
+    <<: *sentry_defaults
+    command: run taskworker-scheduler
+  taskworker:
+    <<: *sentry_defaults
+    command: run taskworker --concurrency=4 --rpc-host=taskbroker:50051 --num-brokers=1
   vroom:
     <<: *restart_policy
     image: "$VROOM_IMAGE"
@@ -541,6 +559,10 @@ volumes:
   # Not being external will still persist data across restarts.
   # It won't persist if someone does a docker compose down -v.
   sentry-vroom:
+  # This volume stores task data that is inflight
+  # It should persist across restarts. If this volume is
+  # deleted, up to ~2048 tasks will be lost.
+  sentry-taskbroker:
   # These store ephemeral data that needn't persist across restarts.
   # That said, volumes will be persisted across restarts until they are deleted.
   sentry-secrets:

scripts/bump-version.sh

@@ -4,7 +4,7 @@ set -eu
 OLD_VERSION="$1"
 NEW_VERSION="$2"
 
-sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
+sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\|TASKBROKER\|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
 sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md
 
 echo "New version: $NEW_VERSION"