From dd76ac61520cd7e888f88413a165bd080ca11925 Mon Sep 17 00:00:00 2001 From: Tony Xiao Date: Mon, 24 Nov 2025 12:08:45 -0500 Subject: [PATCH 01/18] fix(profiling): Ingest profile file path `ingest-profiles` is now using vroomrs to ingest profiles instead of writing through vroom. For self-hosted, we need to make sure filestore for profiles is properly configured so vroom can find the ingested profiles. --- docker-compose.yml | 1 + sentry/config.example.yml | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index bcc858c0b96..d8dadae2e41 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,6 +72,7 @@ x-sentry-defaults: &sentry_defaults SENTRY_STATSD_ADDR: "${STATSD_ADDR:-}" volumes: - "sentry-data:/data" + - sentry-vroom:/var/vroom/sentry-profiles - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index fefe9511b81..7888a49e85f 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -96,6 +96,10 @@ releasefile.cache-path: '/data/releasefile-cache' # secret_key: 'XXXXXXX' # bucket_name: 's3-bucket-name' +filestore.profiles-backend: 'filesystem' +filestore.profiles-options: + location: '/var/vroom/sentry-profiles' + symbolicator.enabled: true symbolicator.options: url: "http://symbolicator:3021" From 240eb92f7d38a89084742b5677770e4fc2d1114b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Tue, 25 Nov 2025 08:32:28 +0700 Subject: [PATCH 02/18] feat: move profiling data to seaweedfs --- docker-compose.yml | 4 +- install.sh | 1 + install/bootstrap-s3-nodestore.sh | 2 +- install/bootstrap-s3-profiles.sh | 113 ++++++++++++++++++++++++++++++ sentry/config.example.yml | 12 +++- 5 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 install/bootstrap-s3-profiles.sh diff --git a/docker-compose.yml b/docker-compose.yml index d8dadae2e41..f2f99427497 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,7 @@ x-sentry-defaults: &sentry_defaults SENTRY_STATSD_ADDR: "${STATSD_ADDR:-}" volumes: - "sentry-data:/data" - - sentry-vroom:/var/vroom/sentry-profiles + - "sentry-vroom:/var/vroom/sentry-profiles" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" @@ -753,7 +753,7 @@ services: environment: SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092" SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092" - SENTRY_BUCKET_PROFILES: file:///var/vroom/sentry-profiles + SENTRY_BUCKET_PROFILES: "s3://profiles?region=us-east-1&endpoint=seaweedfs:8333&s3ForcePathStyle=true&disableSSL=true" SENTRY_SNUBA_HOST: "http://snuba-api:1218" volumes: - sentry-vroom:/var/vroom/sentry-profiles diff --git a/install.sh b/install.sh index b31c9d9ee33..a4ef3d3a87a 100755 --- a/install.sh +++ b/install.sh @@ -37,6 +37,7 @@ source install/generate-secret-key.sh source install/update-docker-images.sh source install/build-docker-images.sh source install/bootstrap-s3-nodestore.sh +source install/bootstrap-s3-profiles.sh source install/bootstrap-snuba.sh source install/upgrade-postgres.sh source install/ensure-correct-permissions-profiles-dir.sh diff --git a/install/bootstrap-s3-nodestore.sh b/install/bootstrap-s3-nodestore.sh index c2cf91669f5..a1e273a00c2 100644 --- a/install/bootstrap-s3-nodestore.sh +++ b/install/bootstrap-s3-nodestore.sh @@ -1,6 +1,6 @@ echo "${_group}Bootstrapping seaweedfs (node store)..." -$dc up --wait seaweedfs postgres +start_service_and_wait_ready seaweedfs postgres $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd $dc exec seaweedfs mkdir -p /data/idx/ s3cmd="$dc exec seaweedfs s3cmd" diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh new file mode 100644 index 00000000000..ff4db2277eb --- /dev/null +++ b/install/bootstrap-s3-profiles.sh @@ -0,0 +1,113 @@ +echo "${_group}Bootstrapping seaweedfs (profiles)..." + +# The purpose of this file is to have both `sentry`-based containers and `vroom` use the same bucket for profiling. +# Currently, we have a `sentry-vroom` volume which stores the profiling data. However, since version 25.10.0, +# the behavior changed, and `vroomrs` now ingests profiles directly. Both services must share the same bucket, +# but at the time of this writing, it's not possible because the `sentry-vroom` volume has ownership set to `vroom:vroom`. +# This prevents the `sentry`-based containers from performing read/write operations on that volume. +# +# Therefore, this script should do the following: +# 1. Check if there are any files inside the `sentry-vroom` volume. +# 2. If (1) finds files, copy those files into a "profiles" bucket on SeaweedFS. +# 3. Point `filestore-profiles` and vroom to the SeaweedFS "profiles" bucket. + +start_service_and_wait_ready seaweedfs +$dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd + +bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) + +if [[ $(echo "$bucket_list" | tail -1 | awk '{print $3}') != 's3://profiles' ]]; then + apply_config_changes_profiles=0 + # Only touch if no existing profiles config is found + if ! grep -q "filestore.profiles-backend" $SENTRY_CONFIG_YML; then + if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then + echo + echo "We are migrating the Profiles data directory from the 'sentry-vroom' volume to S3." + echo "This migration will ensure profiles ingestion works correctly with the new 'vroomrs'" + echo "and allows both 'sentry' and 'vroom' to transition smoothly." + echo "To complete this, your sentry/config.yml file needs to be modified." + echo "Would you like us to perform this modification automatically?" + echo + + yn="" + until [ ! -z "$yn" ]; do + read -p "y or n? " yn + case $yn in + y | yes | 1) + export apply_config_changes_profiles=1 + echo + echo -n "Thank you." + ;; + n | no | 0) + export apply_config_changes_profiles=0 + echo + echo -n "Alright, you will need to update your sentry/config.yml file manually before running 'docker compose up'." + ;; + *) yn="" ;; + esac + done + + echo + echo "To avoid this prompt in the future, use one of these flags:" + echo + echo " --apply-automatic-config-updates" + echo " --no-apply-automatic-config-updates" + echo + echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" + echo + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" + echo + sleep 5 + fi + + if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_profiles" == 1 ]]; then + profiles_config=$(sed -n '/filestore.profiles-backend/,/s3v4"/{p}' sentry/config.example.yml) + echo "$profiles_config" >>$SENTRY_CONFIG_YML + fi + fi + + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://profiles + + # Check if there are files in the sentry-vroom volume + vroom_files_count=$($dc exec vroom sh -c "find /var/vroom/sentry-profiles -type f | wc -l") + if [[ "$vroom_files_count" -gt 0 ]]; then + echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." + + # Use a temporary container to copy files from the volume to SeaweedFS + $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs sh -c ' + apk add --no-cache s3cmd && + s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" sync /source/ s3://profiles/ + ' + + echo "Migration completed." + else + echo "No files found in 'sentry-vroom' volume. Skipping files migration." + fi +else + echo "'profiles' bucket already exists on SeaweedFS. Skipping creation." +fi + +if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" || "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 ]]; then + lifecycle_policy=$( + cat < + + + Sentry-Profiles-Rule + Enabled + + + $SENTRY_EVENT_RETENTION_DAYS + + + +EOF + ) + + $dc exec seaweedfs sh -c "printf '%s' '$lifecycle_policy' > /tmp/profiles-lifecycle-policy.xml" + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' setlifecycle /tmp/profiles-lifecycle-policy.xml s3://profiles + + echo "Making sure the bucket lifecycle policy is all set up correctly..." + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' getlifecycle s3://profiles +fi diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 7888a49e85f..f1c3669d4f0 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -96,9 +96,17 @@ releasefile.cache-path: '/data/releasefile-cache' # secret_key: 'XXXXXXX' # bucket_name: 's3-bucket-name' -filestore.profiles-backend: 'filesystem' +filestore.profiles-backend: 's3' filestore.profiles-options: - location: '/var/vroom/sentry-profiles' + bucket_acl: "private" + default_acl: "private" + access_key: "sentry" + secret_key: "sentry" + bucket_name: "profiles" + region_name: "us-east-1" + endpoint_url: "http://seaweedfs:8333" + addressing_style: "path" + signature_version: "s3v4" symbolicator.enabled: true symbolicator.options: From dae44368f1b1e191410f267db7ac19bc318f12d0 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Tue, 25 Nov 2025 08:37:11 +0700 Subject: [PATCH 03/18] feat: review from Sentry --- install/bootstrap-s3-profiles.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index ff4db2277eb..6b118901fac 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -13,6 +13,7 @@ echo "${_group}Bootstrapping seaweedfs (profiles)..." start_service_and_wait_ready seaweedfs $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd +s3cmd="$dc exec seaweedfs s3cmd" bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) @@ -70,6 +71,7 @@ if [[ $(echo "$bucket_list" | tail -1 | awk '{print $3}') != 's3://profiles' ]]; $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://profiles # Check if there are files in the sentry-vroom volume + start_service_and_wait_ready vroom vroom_files_count=$($dc exec vroom sh -c "find /var/vroom/sentry-profiles -type f | wc -l") if [[ "$vroom_files_count" -gt 0 ]]; then echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." From 52f1c907faccf2a68c0f28080832a4e3d0b8db9b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Wed, 26 Nov 2025 14:16:01 +0700 Subject: [PATCH 04/18] Apply suggestions from code review Co-authored-by: Burak Yigit Kaya --- docker-compose.yml | 1 - install/bootstrap-s3-profiles.sh | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f2f99427497..27ba2d075cc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -72,7 +72,6 @@ x-sentry-defaults: &sentry_defaults SENTRY_STATSD_ADDR: "${STATSD_ADDR:-}" volumes: - "sentry-data:/data" - - "sentry-vroom:/var/vroom/sentry-profiles" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" - "./certificates:/usr/local/share/ca-certificates:ro" diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index 6b118901fac..d606e7a5f72 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -1,7 +1,7 @@ echo "${_group}Bootstrapping seaweedfs (profiles)..." # The purpose of this file is to have both `sentry`-based containers and `vroom` use the same bucket for profiling. -# Currently, we have a `sentry-vroom` volume which stores the profiling data. However, since version 25.10.0, +# On pre-25.10.0, we have a `sentry-vroom` volume which stores the profiling data however, since this version, # the behavior changed, and `vroomrs` now ingests profiles directly. Both services must share the same bucket, # but at the time of this writing, it's not possible because the `sentry-vroom` volume has ownership set to `vroom:vroom`. # This prevents the `sentry`-based containers from performing read/write operations on that volume. @@ -23,7 +23,7 @@ if [[ $(echo "$bucket_list" | tail -1 | awk '{print $3}') != 's3://profiles' ]]; if ! grep -q "filestore.profiles-backend" $SENTRY_CONFIG_YML; then if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then echo - echo "We are migrating the Profiles data directory from the 'sentry-vroom' volume to S3." + echo "We are migrating the Profiles data directory from the 'sentry-vroom' volume to SeaweedFS." echo "This migration will ensure profiles ingestion works correctly with the new 'vroomrs'" echo "and allows both 'sentry' and 'vroom' to transition smoothly." echo "To complete this, your sentry/config.yml file needs to be modified." From f0d3d29a518a70564a3aaba79a1cc2e87c87f51a Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 14:47:28 +0700 Subject: [PATCH 05/18] ref: volume migration tests --- ...test.sh => bootstrap-s3-nodestore-test.sh} | 0 _unit-test/bootstrap-s3-profiles-test.sh | 33 +++++++++++++++++++ docker-compose.yml | 2 ++ 3 files changed, 35 insertions(+) rename _unit-test/{bootstrap-s3-seaweed-test.sh => bootstrap-s3-nodestore-test.sh} (100%) create mode 100755 _unit-test/bootstrap-s3-profiles-test.sh diff --git a/_unit-test/bootstrap-s3-seaweed-test.sh b/_unit-test/bootstrap-s3-nodestore-test.sh similarity index 100% rename from _unit-test/bootstrap-s3-seaweed-test.sh rename to _unit-test/bootstrap-s3-nodestore-test.sh diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh new file mode 100755 index 00000000000..a3060c43369 --- /dev/null +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +source _unit-test/_test_setup.sh +source install/dc-detect-version.sh +source install/create-docker-volumes.sh + +# Generate some random files on `sentry-vroom` volume for testing +$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles busybox sh -c ' + for i in $(seq 1 1000); do + echo "This is test file $i" > /var/vroom/sentry-profiles/test_file_$i.txt + done +' + +# Set the flag to apply automatic updates +export APPLY_AUTOMATIC_CONFIG_UPDATES=1 + +# Here we're just gonna test to run it multiple times +# Only to make sure it doesn't break +for i in $(seq 1 5); do + source install/bootstrap-s3-vroom.sh +done + +# Ensure that the files have been migrated to SeaweedFS +migrated_files_count=$($dc run --rm --no-deps seaweedfs sh -c ' + apk add --no-cache s3cmd && + s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" ls s3://profiles/ | wc -l +') +if [[ "$migrated_files_count" -ne 1000 ]]; then + echo "Error: Expected 1000 migrated files, but found $migrated_files_count" + exit 1 +fi + +report_success diff --git a/docker-compose.yml b/docker-compose.yml index 27ba2d075cc..10cb21716b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -753,6 +753,8 @@ services: SENTRY_KAFKA_BROKERS_PROFILING: "kafka:9092" SENTRY_KAFKA_BROKERS_OCCURRENCES: "kafka:9092" SENTRY_BUCKET_PROFILES: "s3://profiles?region=us-east-1&endpoint=seaweedfs:8333&s3ForcePathStyle=true&disableSSL=true" + AWS_ACCESS_KEY: "sentry" + AWS_SECRET_KEY: "sentry" SENTRY_SNUBA_HOST: "http://snuba-api:1218" volumes: - sentry-vroom:/var/vroom/sentry-profiles From b9547fc305484dd36565f472eb87b85ef3936729 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 14:56:45 +0700 Subject: [PATCH 06/18] ref: execute file creation from vroom container --- _unit-test/bootstrap-s3-profiles-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index a3060c43369..1cb1cc9789a 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -5,7 +5,7 @@ source install/dc-detect-version.sh source install/create-docker-volumes.sh # Generate some random files on `sentry-vroom` volume for testing -$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles busybox sh -c ' +$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles vroom sh -c ' for i in $(seq 1 1000); do echo "This is test file $i" > /var/vroom/sentry-profiles/test_file_$i.txt done From 4d228d9202fd40055810c4487e19b5a74a0406b8 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 16:32:42 +0700 Subject: [PATCH 07/18] fix: brainfart --- _unit-test/bootstrap-s3-profiles-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index 1cb1cc9789a..a09c7d966da 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -5,7 +5,7 @@ source install/dc-detect-version.sh source install/create-docker-volumes.sh # Generate some random files on `sentry-vroom` volume for testing -$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles vroom sh -c ' +$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/sh vroom -c ' for i in $(seq 1 1000); do echo "This is test file $i" > /var/vroom/sentry-profiles/test_file_$i.txt done From 7570995035b3a409c859fc4a23b52ad5401e84f9 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 16:53:54 +0700 Subject: [PATCH 08/18] debug --- _unit-test/bootstrap-s3-profiles-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index a09c7d966da..c509828c7e1 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -5,7 +5,8 @@ source install/dc-detect-version.sh source install/create-docker-volumes.sh # Generate some random files on `sentry-vroom` volume for testing -$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/sh vroom -c ' +$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/bash vroom su - vroom -c ' + ls -lah /var/vroom/sentry-profiles/ for i in $(seq 1 1000); do echo "This is test file $i" > /var/vroom/sentry-profiles/test_file_$i.txt done From 3cc8e38ef11ca949f3af9e8440925cf0f5c28f0c Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 17:14:46 +0700 Subject: [PATCH 09/18] hack --- _unit-test/bootstrap-s3-profiles-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index c509828c7e1..7f4b6ba90d9 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -5,10 +5,10 @@ source install/dc-detect-version.sh source install/create-docker-volumes.sh # Generate some random files on `sentry-vroom` volume for testing -$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/bash vroom su - vroom -c ' +$dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/bash vroom -c ' ls -lah /var/vroom/sentry-profiles/ for i in $(seq 1 1000); do - echo "This is test file $i" > /var/vroom/sentry-profiles/test_file_$i.txt + su vroom -c "echo This is test file $i > /var/vroom/sentry-profiles/test_file_$i.txt" done ' From e61c180c16f7b9e3fb6bc1110127e9ce46ea8d0e Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 17:22:09 +0700 Subject: [PATCH 10/18] more debug --- _unit-test/bootstrap-s3-profiles-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index 7f4b6ba90d9..434f8307847 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -7,8 +7,9 @@ source install/create-docker-volumes.sh # Generate some random files on `sentry-vroom` volume for testing $dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/bash vroom -c ' ls -lah /var/vroom/sentry-profiles/ + whoami for i in $(seq 1 1000); do - su vroom -c "echo This is test file $i > /var/vroom/sentry-profiles/test_file_$i.txt" + echo This is test file $i > /var/vroom/sentry-profiles/test_file_$i.txt done ' From 0a2838f6e2c543e9fb96ff30fd79e13acda5943a Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:01:30 +0700 Subject: [PATCH 11/18] now I know what I'm missing out --- _unit-test/bootstrap-s3-profiles-test.sh | 3 +-- install.sh | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index 434f8307847..ab2a337b3a6 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -3,11 +3,10 @@ source _unit-test/_test_setup.sh source install/dc-detect-version.sh source install/create-docker-volumes.sh +source install/ensure-correct-permissions-profiles-dir.sh # Generate some random files on `sentry-vroom` volume for testing $dc run --rm --no-deps -v sentry-vroom:/var/vroom/sentry-profiles --entrypoint /bin/bash vroom -c ' - ls -lah /var/vroom/sentry-profiles/ - whoami for i in $(seq 1 1000); do echo This is test file $i > /var/vroom/sentry-profiles/test_file_$i.txt done diff --git a/install.sh b/install.sh index a4ef3d3a87a..648f8184543 100755 --- a/install.sh +++ b/install.sh @@ -37,10 +37,10 @@ source install/generate-secret-key.sh source install/update-docker-images.sh source install/build-docker-images.sh source install/bootstrap-s3-nodestore.sh -source install/bootstrap-s3-profiles.sh source install/bootstrap-snuba.sh source install/upgrade-postgres.sh source install/ensure-correct-permissions-profiles-dir.sh +source install/bootstrap-s3-profiles.sh source install/set-up-and-migrate-database.sh source install/migrate-pgbouncer.sh source install/geoip.sh From 9f3f8af2a2684a640e74492a0d99bc6bd581575e Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:20:03 +0700 Subject: [PATCH 12/18] explicitly state feature complete --- _unit-test/bootstrap-s3-profiles-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index ab2a337b3a6..a097736481a 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -3,6 +3,7 @@ source _unit-test/_test_setup.sh source install/dc-detect-version.sh source install/create-docker-volumes.sh +export COMPOSE_PROFILES="feature-complete" source install/ensure-correct-permissions-profiles-dir.sh # Generate some random files on `sentry-vroom` volume for testing From 048d061370695181799dd766d281652759cb6b2b Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:23:25 +0700 Subject: [PATCH 13/18] try to pull vroom image --- _unit-test/bootstrap-s3-profiles-test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index a097736481a..4a020ed4514 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -4,6 +4,7 @@ source _unit-test/_test_setup.sh source install/dc-detect-version.sh source install/create-docker-volumes.sh export COMPOSE_PROFILES="feature-complete" +$dc pull vroom source install/ensure-correct-permissions-profiles-dir.sh # Generate some random files on `sentry-vroom` volume for testing From 786345561ba7bf54b498f208a04b2b8b46917f60 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:29:53 +0700 Subject: [PATCH 14/18] should only run when COMPOSE_PROFILES is feature complete --- _unit-test/bootstrap-s3-profiles-test.sh | 2 +- install/bootstrap-s3-profiles.sh | 155 ++++++++++++----------- 2 files changed, 80 insertions(+), 77 deletions(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index 4a020ed4514..65a6dbf25d3 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -20,7 +20,7 @@ export APPLY_AUTOMATIC_CONFIG_UPDATES=1 # Here we're just gonna test to run it multiple times # Only to make sure it doesn't break for i in $(seq 1 5); do - source install/bootstrap-s3-vroom.sh + source install/bootstrap-s3-profiles.sh done # Ensure that the files have been migrated to SeaweedFS diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index d606e7a5f72..3ebcb260c4a 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -1,5 +1,3 @@ -echo "${_group}Bootstrapping seaweedfs (profiles)..." - # The purpose of this file is to have both `sentry`-based containers and `vroom` use the same bucket for profiling. # On pre-25.10.0, we have a `sentry-vroom` volume which stores the profiling data however, since this version, # the behavior changed, and `vroomrs` now ingests profiles directly. Both services must share the same bucket, @@ -11,88 +9,92 @@ echo "${_group}Bootstrapping seaweedfs (profiles)..." # 2. If (1) finds files, copy those files into a "profiles" bucket on SeaweedFS. # 3. Point `filestore-profiles` and vroom to the SeaweedFS "profiles" bucket. -start_service_and_wait_ready seaweedfs -$dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd -s3cmd="$dc exec seaweedfs s3cmd" +# Should only run when `$COMPOSE_PROFILES` is set to `feature-complete` +if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then + echo "${_group}Bootstrapping seaweedfs (profiles)..." -bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) + start_service_and_wait_ready seaweedfs + $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs apk add --no-cache s3cmd + s3cmd="$dc exec seaweedfs s3cmd" -if [[ $(echo "$bucket_list" | tail -1 | awk '{print $3}') != 's3://profiles' ]]; then - apply_config_changes_profiles=0 - # Only touch if no existing profiles config is found - if ! grep -q "filestore.profiles-backend" $SENTRY_CONFIG_YML; then - if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then - echo - echo "We are migrating the Profiles data directory from the 'sentry-vroom' volume to SeaweedFS." - echo "This migration will ensure profiles ingestion works correctly with the new 'vroomrs'" - echo "and allows both 'sentry' and 'vroom' to transition smoothly." - echo "To complete this, your sentry/config.yml file needs to be modified." - echo "Would you like us to perform this modification automatically?" - echo + bucket_list=$($s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' ls) - yn="" - until [ ! -z "$yn" ]; do - read -p "y or n? " yn - case $yn in - y | yes | 1) - export apply_config_changes_profiles=1 - echo - echo -n "Thank you." - ;; - n | no | 0) - export apply_config_changes_profiles=0 - echo - echo -n "Alright, you will need to update your sentry/config.yml file manually before running 'docker compose up'." - ;; - *) yn="" ;; - esac - done + if [[ $(echo "$bucket_list" | tail -1 | awk '{print $3}') != 's3://profiles' ]]; then + apply_config_changes_profiles=0 + # Only touch if no existing profiles config is found + if ! grep -q "filestore.profiles-backend" $SENTRY_CONFIG_YML; then + if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" ]]; then + echo + echo "We are migrating the Profiles data directory from the 'sentry-vroom' volume to SeaweedFS." + echo "This migration will ensure profiles ingestion works correctly with the new 'vroomrs'" + echo "and allows both 'sentry' and 'vroom' to transition smoothly." + echo "To complete this, your sentry/config.yml file needs to be modified." + echo "Would you like us to perform this modification automatically?" + echo - echo - echo "To avoid this prompt in the future, use one of these flags:" - echo - echo " --apply-automatic-config-updates" - echo " --no-apply-automatic-config-updates" - echo - echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" - echo - echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" - echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" - echo - sleep 5 - fi + yn="" + until [ ! -z "$yn" ]; do + read -p "y or n? " yn + case $yn in + y | yes | 1) + export apply_config_changes_profiles=1 + echo + echo -n "Thank you." + ;; + n | no | 0) + export apply_config_changes_profiles=0 + echo + echo -n "Alright, you will need to update your sentry/config.yml file manually before running 'docker compose up'." + ;; + *) yn="" ;; + esac + done - if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_profiles" == 1 ]]; then - profiles_config=$(sed -n '/filestore.profiles-backend/,/s3v4"/{p}' sentry/config.example.yml) - echo "$profiles_config" >>$SENTRY_CONFIG_YML + echo + echo "To avoid this prompt in the future, use one of these flags:" + echo + echo " --apply-automatic-config-updates" + echo " --no-apply-automatic-config-updates" + echo + echo "or set the APPLY_AUTOMATIC_CONFIG_UPDATES environment variable:" + echo + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=1 to apply automatic updates" + echo " APPLY_AUTOMATIC_CONFIG_UPDATES=0 to not apply automatic updates" + echo + sleep 5 + fi + + if [[ "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 || "$apply_config_changes_profiles" == 1 ]]; then + profiles_config=$(sed -n '/filestore.profiles-backend/,/s3v4"/{p}' sentry/config.example.yml) + echo "$profiles_config" >>$SENTRY_CONFIG_YML + fi fi - fi - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://profiles + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' mb s3://profiles - # Check if there are files in the sentry-vroom volume - start_service_and_wait_ready vroom - vroom_files_count=$($dc exec vroom sh -c "find /var/vroom/sentry-profiles -type f | wc -l") - if [[ "$vroom_files_count" -gt 0 ]]; then - echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." + # Check if there are files in the sentry-vroom volume + start_service_and_wait_ready vroom + vroom_files_count=$($dc exec vroom sh -c "find /var/vroom/sentry-profiles -type f | wc -l") + if [[ "$vroom_files_count" -gt 0 ]]; then + echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." - # Use a temporary container to copy files from the volume to SeaweedFS - $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs sh -c ' - apk add --no-cache s3cmd && - s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" sync /source/ s3://profiles/ - ' + # Use a temporary container to copy files from the volume to SeaweedFS + $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs sh -c ' + apk add --no-cache s3cmd && + s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" sync /source/ s3://profiles/ + ' - echo "Migration completed." + echo "Migration completed." + else + echo "No files found in 'sentry-vroom' volume. Skipping files migration." + fi else - echo "No files found in 'sentry-vroom' volume. Skipping files migration." + echo "'profiles' bucket already exists on SeaweedFS. Skipping creation." fi -else - echo "'profiles' bucket already exists on SeaweedFS. Skipping creation." -fi -if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" || "$APPLY_AUTOMATIC_CONFIG_UPDATES" == 1 ]]; then - lifecycle_policy=$( - cat < @@ -105,11 +107,12 @@ if [[ -z "${APPLY_AUTOMATIC_CONFIG_UPDATES:-}" || "$APPLY_AUTOMATIC_CONFIG_UPDAT EOF - ) + ) - $dc exec seaweedfs sh -c "printf '%s' '$lifecycle_policy' > /tmp/profiles-lifecycle-policy.xml" - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' setlifecycle /tmp/profiles-lifecycle-policy.xml s3://profiles + $dc exec seaweedfs sh -c "printf '%s' '$lifecycle_policy' > /tmp/profiles-lifecycle-policy.xml" + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' setlifecycle /tmp/profiles-lifecycle-policy.xml s3://profiles - echo "Making sure the bucket lifecycle policy is all set up correctly..." - $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' getlifecycle s3://profiles + echo "Making sure the bucket lifecycle policy is all set up correctly..." + $s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket='localhost:8333/%(bucket)' getlifecycle s3://profiles + fi fi From cd085916ee5d3e680820943aba76f723bdfe7624 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:36:05 +0700 Subject: [PATCH 15/18] using run invoked weed instead of empty shell --- _unit-test/bootstrap-s3-profiles-test.sh | 3 ++- install/bootstrap-s3-profiles.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index 65a6dbf25d3..a3e37c599ce 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -3,6 +3,7 @@ source _unit-test/_test_setup.sh source install/dc-detect-version.sh source install/create-docker-volumes.sh +source install/ensure-files-from-examples.sh export COMPOSE_PROFILES="feature-complete" $dc pull vroom source install/ensure-correct-permissions-profiles-dir.sh @@ -24,7 +25,7 @@ for i in $(seq 1 5); do done # Ensure that the files have been migrated to SeaweedFS -migrated_files_count=$($dc run --rm --no-deps seaweedfs sh -c ' +migrated_files_count=$($dc run --rm --no-deps --entrypoint /bin/sh seaweedfs -c ' apk add --no-cache s3cmd && s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" ls s3://profiles/ | wc -l ') diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index 3ebcb260c4a..a850708288e 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -79,7 +79,7 @@ if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." # Use a temporary container to copy files from the volume to SeaweedFS - $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" seaweedfs sh -c ' + $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" --entrypoint /bin/sh seaweedfs -c ' apk add --no-cache s3cmd && s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" sync /source/ s3://profiles/ ' From aaa2d706193f1b5d0e8d4f93166bba8bcea2c6f3 Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:43:24 +0700 Subject: [PATCH 16/18] execute the upload script from vroom container --- install/bootstrap-s3-profiles.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index a850708288e..7ca686102f4 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -79,9 +79,10 @@ if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then echo "Migrating $vroom_files_count files from 'sentry-vroom' volume to 'profiles' bucket on SeaweedFS..." # Use a temporary container to copy files from the volume to SeaweedFS - $dc run --rm --no-deps -v sentry-vroom:/source -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" --entrypoint /bin/sh seaweedfs -c ' - apk add --no-cache s3cmd && - s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" sync /source/ s3://profiles/ + + $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" vroom sh -c ' + apt-get update && apt-get install -y --no-install-recommends s3cmd && + s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" sync /var/vroom/sentry-profiles/ s3://profiles/ ' echo "Migration completed." From 8a14ed5bb382a4350aea4c3bb83428b74447c21f Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:52:48 +0700 Subject: [PATCH 17/18] execute apt command as root --- install/bootstrap-s3-profiles.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/install/bootstrap-s3-profiles.sh b/install/bootstrap-s3-profiles.sh index 7ca686102f4..607f867db69 100644 --- a/install/bootstrap-s3-profiles.sh +++ b/install/bootstrap-s3-profiles.sh @@ -80,10 +80,8 @@ if [[ "$COMPOSE_PROFILES" == "feature-complete" ]]; then # Use a temporary container to copy files from the volume to SeaweedFS - $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" vroom sh -c ' - apt-get update && apt-get install -y --no-install-recommends s3cmd && - s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" sync /var/vroom/sentry-profiles/ s3://profiles/ - ' + $dc exec -e "HTTP_PROXY=${HTTP_PROXY:-}" -e "HTTPS_PROXY=${HTTPS_PROXY:-}" -e "NO_PROXY=${NO_PROXY:-}" -e "http_proxy=${http_proxy:-}" -e "https_proxy=${https_proxy:-}" -e "no_proxy=${no_proxy:-}" -u root vroom sh -c 'mkdir -p /var/lib/apt/lists/partial && apt-get update && apt-get install -y --no-install-recommends s3cmd' + $dc exec vroom sh -c 's3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" sync /var/vroom/sentry-profiles/ s3://profiles/' echo "Migration completed." else From 74d0195c3fd241a066e530a92807322985b1686f Mon Sep 17 00:00:00 2001 From: Reinaldy Rafli Date: Thu, 27 Nov 2025 21:58:20 +0700 Subject: [PATCH 18/18] gonna sleep --- _unit-test/bootstrap-s3-profiles-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_unit-test/bootstrap-s3-profiles-test.sh b/_unit-test/bootstrap-s3-profiles-test.sh index a3e37c599ce..9db387fa3fd 100755 --- a/_unit-test/bootstrap-s3-profiles-test.sh +++ b/_unit-test/bootstrap-s3-profiles-test.sh @@ -25,9 +25,9 @@ for i in $(seq 1 5); do done # Ensure that the files have been migrated to SeaweedFS -migrated_files_count=$($dc run --rm --no-deps --entrypoint /bin/sh seaweedfs -c ' +migrated_files_count=$($dc exec seaweedfs -c ' apk add --no-cache s3cmd && - s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=localhost:8333 --host-bucket="localhost:8333/%(bucket)" ls s3://profiles/ | wc -l + s3cmd --access_key=sentry --secret_key=sentry --no-ssl --region=us-east-1 --host=seaweedfs:8333 --host-bucket="seaweedfs:8333/%(bucket)" ls s3://profiles/ | wc -l ') if [[ "$migrated_files_count" -ne 1000 ]]; then echo "Error: Expected 1000 migrated files, but found $migrated_files_count"