feat(profiling): Run profiling on self-hosted (#2154)

Author: Pierre Massat

.env

@@ -9,6 +9,7 @@ SENTRY_IMAGE=getsentry/sentry:nightly
 SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
+VROOM_IMAGE=getsentry/vroom:nightly
 WAL2JSON_VERSION=latest
 HEALTHCHECK_INTERVAL=30s
 HEALTHCHECK_TIMEOUT=60s

_integration-test/fixtures/envelope-with-profile

@@ -0,0 +1,5 @@
+{"event_id":"66578634d48d433db0ad52882d1efe5b","sent_at":"2023-05-17T14:54:31.057Z","sdk":{"name":"sentry.javascript.node","version":"7.46.0"},"trace":{"environment":"production","transaction":"fib: sourcemaps here","public_key":"05ab86aebbe14a24bcab62caa839cf27","trace_id":"33321bfbd5304bcc9663d1b53b08f84e","sample_rate":"1"}}
+{"type":"transaction"}
+{"contexts":{"profile":{"profile_id":"e73aaf1f29b24812be60132f32d09f92"},"trace":{"op":"test","span_id":"b38f2b24537c3858","trace_id":"33321bfbd5304bcc9663d1b53b08f84e"},"runtime":{"name":"node","version":"v16.16.0"},"app":{"app_start_time":"2023-05-17T14:54:27.678Z","app_memory":57966592},"os":{"kernel_version":"22.3.0","name":"macOS","version":"13.2","build":"22D49"},"device":{"boot_time":"2023-05-12T15:08:41.047Z","arch":"arm64","memory_size":34359738368,"free_memory":6861651968,"processor_count":10,"cpu_description":"Apple M1 Pro","processor_frequency":24},"culture":{"locale":"en-US","timezone":"America/New_York"}},"spans":[],"start_timestamp":1684335267.744,"tags":{},"timestamp":1684335271.033,"transaction":"fib: sourcemaps here","type":"transaction","transaction_info":{"source":"custom"},"platform":"node","server_name":"TK6G745PW1.local","event_id":"66578634d48d433db0ad52882d1efe5b","environment":"production","sdk":{"integrations":["InboundFilters","FunctionToString","Console","Http","OnUncaughtException","OnUnhandledRejection","ContextLines","LocalVariables","Context","Modules","RequestData","LinkedErrors","ProfilingIntegration"],"name":"sentry.javascript.node","version":"7.46.0","packages":[{"name":"npm:@sentry/node","version":"7.46.0"}]},"debug_meta":{"images":[]},"modules":{}}
+{"type":"profile"}
+{"event_id":"e73aaf1f29b24812be60132f32d09f92","timestamp":"2023-05-17T14:54:27.744Z","platform":"node","version":"1","release":"","environment":"production","runtime":{"name":"node","version":"16.16.0"},"os":{"name":"darwin","version":"22.3.0","build_number":"Darwin Kernel Version 22.3.0: Thu Jan  5 20:48:54 PST 2023; root:xnu-8792.81.2~2/RELEASE_ARM64_T6000"},"device":{"locale":"en_US.UTF-8","model":"arm64","manufacturer":"Darwin","architecture":"arm64","is_emulator":false},"debug_meta":{"images":[]},"profile":{"samples":[{"stack_id":0,"thread_id":"0","elapsed_since_start_ns":125000},{"stack_id":0,"thread_id":"0","elapsed_since_start_ns":13958000}],"frames":[{"lineno":14129,"colno":17,"function":"startProfiling","abs_path":"/Users/jonasbadalic/code/node-profiler/lib/index.js"}],"stacks":[[0]],"thread_metadata":{"0":{"name":"main"}}},"transaction":{"name":"fib: sourcemaps here","id":"66578634d48d433db0ad52882d1efe5b","trace_id":"33321bfbd5304bcc9663d1b53b08f84e","active_thread_id":"0"}}

_integration-test/run.sh

@@ -135,6 +135,18 @@ $dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py
 source _integration-test/custom-ca-roots/teardown.sh
 echo "${_endgroup}"
 
+echo "${_group}Test that profiling work ..."
+echo "Sending a test profile..."
+PROFILE_FIXTURE_PATH="$(git rev-parse --show-toplevel)/_integration-test/fixtures/envelope-with-profile"
+curl -sf --data-binary @$PROFILE_FIXTURE_PATH -H 'Content-Type: application/x-sentry-envelope' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/envelope/" -o /dev/null
+
+printf "Getting the test profile back"
+PROFILE_ID="$(jq -r -n --slurpfile profile $PROFILE_FIXTURE_PATH '$profile[4].event_id')"
+PROFILE_PATH="api/0/projects/sentry/sentry/profiling/raw_profiles/$PROFILE_ID/"
+timeout 60 bash -c 'until $(sentry_api_request "$PROFILE_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
+echo " got it!"
+echo "${_endgroup}"
+
 # Table formatting based on https://stackoverflow.com/a/39144364
 COMPOSE_PS_OUTPUT=$(docker compose ps --format json | jq -r \
   '.[] |

_unit-test/create-docker-volumes-test.sh

@@ -21,8 +21,6 @@ before=$(get_volumes)
 
 test "$before" == "" || test "$before" == "$expected_volumes"
 
-source install/create-docker-volumes.sh
-source install/create-docker-volumes.sh
 source install/create-docker-volumes.sh
 
 after=$(get_volumes)

docker-compose.yml

@@ -52,12 +52,15 @@ x-sentry-defaults: &sentry_defaults
       <<: *depends_on-default
     symbolicator:
       <<: *depends_on-default
+    vroom:
+      <<: *depends_on-default
   entrypoint: "/etc/sentry/entrypoint.sh"
   command: ["run", "web"]
   environment:
     PYTHONUSERBASE: "/data/custom-packages"
     SENTRY_CONF: "/etc/sentry"
     SNUBA: "http://snuba-api:1218"
+    VROOM: "http://vroom:8085"
     # Force everything to use the system CA bundle
     # This is mostly needed to support installing custom CA certs
     # This one is used by botocore
@@ -283,6 +286,12 @@ services:
   snuba-subscription-consumer-transactions:
     <<: *snuba_defaults
     command: subscriptions-scheduler-executor --dataset transactions --entity transactions --auto-offset-reset=latest --no-strict-offset-reset --consumer-group=snuba-transactions-subscriptions-consumers --followed-consumer-group=transactions_group --delay-seconds=60 --schedule-ttl=60 --stale-threshold-seconds=900
+  snuba-profiling-profiles-consumer:
+    <<: *snuba_defaults
+    command: consumer --storage profiles --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset
+  snuba-profiling-functions-consumer:
+    <<: *snuba_defaults
+    command: consumer --storage functions_raw --auto-offset-reset=latest --max-batch-time-ms 1000 --no-strict-offset-reset
   symbolicator:
     <<: *restart_policy
     image: "$SYMBOLICATOR_IMAGE"
@@ -329,10 +338,12 @@ services:
   ingest-replay-recordings:
     <<: *sentry_defaults
     command: run ingest-replay-recordings
+  ingest-profiles:
+    <<: *sentry_defaults
+    command: run ingest-profiles --no-strict-offset-reset
   post-process-forwarder-errors:
     <<: *sentry_defaults
     command: run post-process-forwarder --entity errors
-
   post-process-forwarder-transactions:
     <<: *sentry_defaults
     command: run post-process-forwarder --entity transactions --commit-log-topic=snuba-transactions-commit-log --synchronize-commit-group transactions_group
@@ -384,6 +395,19 @@ services:
         <<: *depends_on-healthy
       web:
         <<: *depends_on-healthy
+  vroom:
+    <<: *restart_policy
+    image: "$VROOM_IMAGE"
+    environment:
+      SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092"
+      SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092"
+      SENTRY_BUCKET_PROFILES: file://localhost//var/lib/sentry-profiles
+      SENTRY_SNUBA_HOST: "http://snuba-api:1218"
+    volumes:
+      - sentry-vroom:/var/lib/sentry-profiles
+    depends_on:
+      kafka:
+        <<: *depends_on-healthy
 volumes:
   # These store application data that should persist across restarts.
   sentry-data:
@@ -400,8 +424,12 @@ volumes:
     external: true
   sentry-symbolicator:
     external: true
-
+  # This volume stores profiles and should be persisted.
+  # Not being external will still persist data across restarts.
+  # It won't persist if someone does a docker compose down -v.
+  sentry-vroom:
   # These store ephemeral data that needn't persist across restarts.
+  # That said, volumes will be persisted across restarts until they are deleted.
   sentry-secrets:
   sentry-smtp:
   sentry-nginx-cache:

install/create-kafka-topics.sh

@@ -3,7 +3,7 @@ echo "${_group}Creating additional Kafka topics ..."
 # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step
 # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable)
 EXISTING_KAFKA_TOPICS=$($dcr -T kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null)
-NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings"
+NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events ingest-replay-recordings profiles"
 for topic in $NEEDED_KAFKA_TOPICS; do
   if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then
     $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092

scripts/bump-version.sh

@@ -7,7 +7,7 @@ NEW_VERSION="$2"
 WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')}
 
 sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env
-sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
+sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\|SYMBOLICATOR|VROOM\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env
 sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md
 sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE
 

sentry/sentry.conf.example.py

@@ -269,6 +269,7 @@ def get_internal_network():
             "organizations:performance-view",
             "organizations:advanced-search",
             "organizations:session-replay",
+            "organizations:profiling",
             "projects:custom-inbound-filters",
             "projects:data-forwarding",
             "projects:discard-groups",